Merge pull request #15 from PDOK/PDOK-17796-WMS-reconcile

WMS Reconcile

Author: damienmulder

api/v3/shared_types.go

@@ -158,3 +158,8 @@ func Sha1HashOfName[O WMSWFS](obj O) string {
 
 	return hex.EncodeToString(s.Sum(nil))
 }
+
+func (o *Options) UseWebserviceProxy() bool {
+	// options.DisableWebserviceProxy not set or false
+	return o != nil && (o.DisableWebserviceProxy == nil || !*o.DisableWebserviceProxy)
+}

cmd/main.go

@@ -48,11 +48,12 @@ import (
 )
 
 const (
-	defaultMapserverImage             = "acrpdokprodman.azurecr.io/mirror/docker.io/pdok/mapserver:8.4.0-4"
 	defaultMultitoolImage             = "acrpdokprodman.azurecr.io/pdok/docker-multitool:0.9.4"
 	defaultMapfileGeneratorImage      = "acrpdokprodman.azurecr.io/pdok/mapfile-generator:1.9.5"
+	defaultMapserverImage             = "acrpdokprodman.azurecr.io/mirror/docker.io/pdok/mapserver:8.4.0-4-nl"
 	defaultCapabilitiesGeneratorImage = "acrpdokprodman.azurecr.io/mirror/docker.io/pdok/ogc-capabilities-generator:1.0.0-beta5"
 	defaultFeatureinfoGeneratorImage  = "acrpdokprodman.azurecr.io/mirror/docker.io/pdok/featureinfo-generator:v1.4.0-beta1"
+	defaultOgcWebserviceProxyImage    = "acrpdokprodman.azurecr.io/pdok/ogc-webservice-proxy:0.1.8"
 )
 
 var (
@@ -80,11 +81,7 @@ func main() {
 	var enableHTTP2 bool
 	var tlsOpts []func(*tls.Config)
 	var host string
-	var mapserverImage string
-	var multitoolImage string
-	var mapfileGeneratorImage string
-	var capabilitiesGeneratorImage string
-	var featureinfoGeneratorImage string
+	var multitoolImage, mapfileGeneratorImage, mapserverImage, capabilitiesGeneratorImage, featureinfoGeneratorImage, ogcWebserviceProxyImage string
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
@@ -103,11 +100,12 @@ func main() {
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	flag.StringVar(&host, "baseurl", "", "The host which is used in the mapserver service.")
-	flag.StringVar(&mapserverImage, "mapserver-image", defaultMapserverImage, "The image to use in the blob download init-container.")
 	flag.StringVar(&multitoolImage, "multitool-image", defaultMultitoolImage, "The image to use in the blob download init-container.")
 	flag.StringVar(&mapfileGeneratorImage, "mapfile-generator-image", defaultMapfileGeneratorImage, "The image to use in the mapfile generator init-container.")
+	flag.StringVar(&mapserverImage, "mapserver-image", defaultMapserverImage, "The image to use in the mapserver container.")
 	flag.StringVar(&capabilitiesGeneratorImage, "capabilities-generator-image", defaultCapabilitiesGeneratorImage, "The image to use in the capabilities generator init-container.")
 	flag.StringVar(&featureinfoGeneratorImage, "featureinfo-generator-image", defaultFeatureinfoGeneratorImage, "The image to use in the featureinfo generator init-container.")
+	flag.StringVar(&ogcWebserviceProxyImage, "ogc-webservice-proxy-image", defaultOgcWebserviceProxyImage, "The image to use in the ogc webservice proxy container.")
 
 	opts := zap.Options{
 		Development: true,
@@ -237,20 +235,29 @@ func main() {
 	}
 
 	if err = (&controller.WMSReconciler{
-		Client:                    mgr.GetClient(),
-		Scheme:                    mgr.GetScheme(),
-		FeatureinfoGeneratorImage: featureinfoGeneratorImage,
+		Client: mgr.GetClient(),
+		Scheme: mgr.GetScheme(),
+		Images: controller.Images{
+			MultitoolImage:             multitoolImage,
+			MapfileGeneratorImage:      mapfileGeneratorImage,
+			MapserverImage:             mapserverImage,
+			CapabilitiesGeneratorImage: capabilitiesGeneratorImage,
+			FeatureinfoGeneratorImage:  featureinfoGeneratorImage,
+			OgcWebserviceProxyImage:    ogcWebserviceProxyImage,
+		},
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "WMS")
 		os.Exit(1)
 	}
 	if err = (&controller.WFSReconciler{
-		Client:                     mgr.GetClient(),
-		Scheme:                     mgr.GetScheme(),
-		MapserverImage:             mapserverImage,
-		MultitoolImage:             multitoolImage,
-		MapfileGeneratorImage:      mapfileGeneratorImage,
-		CapabilitiesGeneratorImage: capabilitiesGeneratorImage,
+		Client: mgr.GetClient(),
+		Scheme: mgr.GetScheme(),
+		Images: controller.Images{
+			MultitoolImage:             multitoolImage,
+			MapfileGeneratorImage:      mapfileGeneratorImage,
+			MapserverImage:             mapserverImage,
+			CapabilitiesGeneratorImage: capabilitiesGeneratorImage,
+		},
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "WFS")
 		os.Exit(1)

config/samples/v3_wms.yaml

@@ -6,7 +6,7 @@ metadata:
     app.kubernetes.io/managed-by: kustomize
     dataset: dataset
     dataset-owner: owner
-    service-type: wfs
+    service-type: wms
     service-version: 1.0.0
   name: sample-v3
 spec:

internal/controller/blobdownload/blob_download.go

@@ -139,6 +139,10 @@ func downloadTiffs(sb *strings.Builder, wms *pdoknlv3.WMS) error {
 }
 
 func downloadStylingAssets(sb *strings.Builder, wms *pdoknlv3.WMS) error {
+	if wms.Spec.Service.StylingAssets == nil { // TODO Is StylingAssets required and should this return an error?
+		return nil
+	}
+
 	re := regexp.MustCompile(".*\\.(ttf)$")
 	for _, blobKey := range wms.Spec.Service.StylingAssets.BlobKeys {
 		fileName, err := getFilenameFromBlobKey(blobKey)

internal/controller/featureinfogenerator/featureinfo_generator.go

@@ -4,12 +4,12 @@ import (
 	"encoding/json"
 	"fmt"
 	pdoknlv3 "github.com/pdok/mapserver-operator/api/v3"
+	"github.com/pdok/mapserver-operator/internal/controller/mapserver"
 	corev1 "k8s.io/api/core/v1"
 )
 
 const (
-	htmlTemplatesPath                       = "/srv/data/config/templates"
-	ConfigMapFeatureinfoGeneratorVolumeName = "featureinfo-generator-config"
+	htmlTemplatesPath = "/srv/data/config/templates"
 )
 
 func GetFeatureinfoGeneratorInitContainer(image string, srvDir string) (*corev1.Container, error) {
@@ -28,7 +28,7 @@ func GetFeatureinfoGeneratorInitContainer(image string, srvDir string) (*corev1.
 		},
 		VolumeMounts: []corev1.VolumeMount{
 			{Name: "base", MountPath: srvDir + "/data", ReadOnly: false},
-			{Name: ConfigMapFeatureinfoGeneratorVolumeName, MountPath: "/input", ReadOnly: true},
+			{Name: mapserver.ConfigMapFeatureinfoGeneratorVolumeName, MountPath: "/input", ReadOnly: true},
 		},
 	}
 

internal/controller/legendgenerator/legend_generator.go

@@ -0,0 +1,78 @@
+package legendgenerator
+
+import (
+	pdoknlv3 "github.com/pdok/mapserver-operator/api/v3"
+	"github.com/pdok/mapserver-operator/internal/controller/mapserver"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func GetLegendGeneratorInitContainer(wms *pdoknlv3.WMS, image string, srvDir string) (*corev1.Container, error) {
+	initContainer := corev1.Container{
+		Name:            "legend-generator",
+		Image:           image,
+		ImagePullPolicy: corev1.PullIfNotPresent,
+		Env:             []corev1.EnvVar{mapserver.GetMapfileEnvVar(wms)},
+		Command: []string{
+			"bash",
+			"-c",
+			`set -eu;
+cat /input/input | xargs -n 2 echo | while read layer style; do
+echo Generating legend for layer: $layer, style: $style;
+mkdir -p /var/www/legend/$layer;
+mapserv -nh 'QUERY_STRING=SERVICE=WMS&language=dut&version=1.3.0&service=WMS&request=GetLegendGraphic&sld_version=1.1.0&layer='$layer'&format=image/png&STYLE='$style'' > /var/www/legend/$layer/${style}.png;
+done
+`,
+		},
+		VolumeMounts: []corev1.VolumeMount{
+			{Name: "base", MountPath: srvDir + "/data", ReadOnly: false},
+			getDataVolumeMount(),
+			getConfigVolumeMount(),
+		},
+	}
+
+	if wms.Spec.Service.Mapfile != nil {
+		volumeMount := corev1.VolumeMount{
+			Name:      "mapfile",
+			MountPath: "/srv/data/config/mapfile",
+		}
+		initContainer.VolumeMounts = append(initContainer.VolumeMounts, volumeMount)
+	}
+
+	return &initContainer, nil
+}
+
+func GetLegendFixerInitContainer(image string) *corev1.Container {
+	return &corev1.Container{
+		Name:            "legend-fixer",
+		Image:           image,
+		ImagePullPolicy: corev1.PullIfNotPresent,
+		Command: []string{
+			"/bin/bash",
+			"/input/legend-fixer.sh",
+		},
+		VolumeMounts: []corev1.VolumeMount{
+			getDataVolumeMount(),
+			getConfigVolumeMount(),
+		},
+	}
+}
+
+func GetConfigMapData(wms *pdoknlv3.WMS) map[string]string {
+	data := map[string]string{
+		"default_mapserver.conf": defaultMapserverConf,
+	}
+
+	addLayerInput(wms, data)
+	if wms.Spec.Options.RewriteGroupToDataLayers != nil && *wms.Spec.Options.RewriteGroupToDataLayers {
+		addLegendFixerConfig(wms, data)
+	}
+	return data
+}
+
+func getDataVolumeMount() corev1.VolumeMount {
+	return corev1.VolumeMount{Name: "data", MountPath: "/var/www", ReadOnly: false}
+}
+
+func getConfigVolumeMount() corev1.VolumeMount {
+	return corev1.VolumeMount{Name: mapserver.ConfigMapLegendGeneratorVolumeName, MountPath: "/input", ReadOnly: true}
+}

…ontroller/legendgenerator/mapper_test.go …legendgenerator/legend_generator_test.gointernal/controller/legendgenerator/mapper_test.go renamed to internal/controller/legendgenerator/legend_generator_test.go internal/controller/legendgenerator/mapper_test.go renamed to internal/controller/legendgenerator/legend_generator_test.go

@@ -3,15 +3,13 @@ package legendgenerator
 import (
 	"fmt"
 	pdoknlv3 "github.com/pdok/mapserver-operator/api/v3"
-	"github.com/pdok/mapserver-operator/internal/controller"
-	smoothoperatorutils "github.com/pdok/smooth-operator/pkg/util"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"os"
+
+	_ "embed"
 	"sigs.k8s.io/yaml"
 	"strings"
 )
 
+// TODO Reuse default_mapserver.conf from static_files?
 const (
 	defaultMapserverConf = `CONFIG
   ENV
@@ -21,6 +19,9 @@ END
 `
 )
 
+//go:embed legend-fixer.sh
+var legendFixerScript string
+
 type LegendReference struct {
 	Layer string `yaml:"layer" json:"layer"`
 	Style string `yaml:"style" json:"style"`
@@ -30,33 +31,6 @@ type OgcWebserviceProxyConfig struct {
 	GroupLayers map[string][]string `yaml:"grouplayers" json:"grouplayers"`
 }
 
-func getBareConfigMapLegendGenerator(obj metav1.Object) *corev1.ConfigMap {
-	return &corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      obj.GetName() + "-legend-generator",
-			Namespace: obj.GetNamespace(),
-		},
-	}
-}
-
-func GetLegendGeneratorConfigMap(wms *pdoknlv3.WMS) *corev1.ConfigMap {
-	result := getBareConfigMapLegendGenerator(wms)
-	labels := controller.AddCommonLabels(wms, smoothoperatorutils.CloneOrEmptyMap(wms.GetLabels()))
-	result.Labels = labels
-
-	result.Immutable = smoothoperatorutils.Pointer(true)
-	result.Data = map[string]string{}
-	result.Data["default_mapserver.conf"] = defaultMapserverConf
-
-	addLayerInput(wms, result.Data)
-
-	if wms.Spec.Options.RewriteGroupToDataLayers != nil && *wms.Spec.Options.RewriteGroupToDataLayers {
-		addLegendFixerConfig(wms, result.Data)
-	}
-
-	return result
-}
-
 func addLayerInput(wms *pdoknlv3.WMS, data map[string]string) {
 	legendReferences := make([]LegendReference, 0)
 
@@ -100,10 +74,7 @@ func processLayer(layer *pdoknlv3.Layer, legendReferences *[]LegendReference) {
 }
 
 func addLegendFixerConfig(wms *pdoknlv3.WMS, data map[string]string) {
-	fileBytes, err := os.ReadFile("./legend-fixer.sh")
-	if err == nil {
-		data["legend-fixer.sh"] = string(fileBytes)
-	}
+	data["legend-fixer.sh"] = legendFixerScript
 
 	topLayer := wms.Spec.Service.Layer
 
@@ -154,7 +125,7 @@ func addLegendFixerConfig(wms *pdoknlv3.WMS, data map[string]string) {
 	}
 
 	ogcWebServiceProxyConfig := OgcWebserviceProxyConfig{GroupLayers: groupLayers}
-	proxyConfigData, err := yaml.Marshal(ogcWebServiceProxyConfig)
+	proxyConfigData, _ := yaml.Marshal(ogcWebServiceProxyConfig)
 	data["ogc-webservice-proxy-config.yaml"] = string(proxyConfigData)
 }