CLOUDP-58413: Deploy any version of MongoDB (#29)

Author: chatton

docker/Dockerfile.operator

@@ -19,16 +19,22 @@ RUN go mod vendor && /bin/operator-sdk generate k8s
 # TODO: This build takes longer than it needs to and can still be optimized
 RUN go build -o build/_output/bin/mongodb-kubernetes-operator -mod=vendor github.com/mongodb/mongodb-kubernetes-operator/cmd/manager
 
+ENV manifest_version=4.2
+RUN mkdir -p /content/ \
+        && chmod -R +r /content/ \
+        && curl --fail --retry 3 -o /content/version_manifest.json "https://opsmanager.mongodb.com/static/version_manifest/${manifest_version}.json"
+
 FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
 
 ENV OPERATOR=/usr/local/bin/mongodb-kubernetes-operator \
     USER_UID=1001 \
     USER_NAME=mongodb-kubernetes-operator
 
-# copy the operator binary
+# copy the operator binary and version manifest
 COPY --from=builder /go/build/_output/bin/mongodb-kubernetes-operator ${OPERATOR}
-
 COPY --from=builder /go/build/bin /usr/local/bin
+COPY --from=builder /content/version_manifest.json /usr/local/version_manifest.json
+
 RUN  /usr/local/bin/user_setup
 
 ENTRYPOINT ["/usr/local/bin/entrypoint"]

pkg/automationconfig/automation_config.go

@@ -162,6 +162,28 @@ type AutomationConfig struct {
 	Options  Options                `json:"options"`
 }
 
+type VersionManifest struct {
+	Updated  int                    `json:"updated"`
+	Versions []MongoDbVersionConfig `json:"versions"`
+}
+
+// BuildsForVersion returns the MongoDbVersionConfig containing all of the version informatioon
+// for the given mongodb version provided
+func (v VersionManifest) BuildsForVersion(version string) MongoDbVersionConfig {
+	var builds []BuildConfig
+	for _, versionConfig := range v.Versions {
+		if versionConfig.Name != version {
+			continue
+		}
+		builds = versionConfig.Builds
+		break
+	}
+	return MongoDbVersionConfig{
+		Name:   version,
+		Builds: builds,
+	}
+}
+
 type Options struct {
 	DownloadBase string `json:"downloadBase"`
 }

pkg/automationconfig/automation_config_test.go

@@ -146,3 +146,29 @@ func TestProcessHasPortSetToDefault(t *testing.T) {
 	assert.Equal(t, ac.Processes[1].Args26.Net.Port, 27017)
 	assert.Equal(t, ac.Processes[2].Args26.Net.Port, 27017)
 }
+
+func TestVersionManifest_BuildsForVersion(t *testing.T) {
+	vm := VersionManifest{
+		Updated: 0,
+		Versions: []MongoDbVersionConfig{
+			defaultMongoDbVersion("4.2.0"),
+			defaultMongoDbVersion("4.2.3"),
+			defaultMongoDbVersion("4.2.4"),
+		},
+	}
+
+	version := vm.BuildsForVersion("4.2.0")
+	assert.Len(t, version.Builds, 1)
+	assert.Equal(t, defaultMongoDbVersion("4.2.0"), version)
+
+	version = vm.BuildsForVersion("4.2.3")
+	assert.Len(t, version.Builds, 1)
+	assert.Equal(t, defaultMongoDbVersion("4.2.3"), version)
+
+	version = vm.BuildsForVersion("4.2.4")
+	assert.Len(t, version.Builds, 1)
+	assert.Equal(t, defaultMongoDbVersion("4.2.4"), version)
+
+	version = vm.BuildsForVersion("4.2.1")
+	assert.Empty(t, version.Builds)
+}

pkg/controller/mongodb/mongodb_controller.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 
 	mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1"
@@ -31,6 +32,7 @@ const (
 	agentName                 = "mongodb-agent"
 	mongodbName               = "mongod"
 	agentImageEnvVariable     = "AGENT_IMAGE"
+	versionManifestFilePath   = "/usr/local/version_manifest.json"
 	readinessProbePath        = "/var/lib/mongodb-mms-automation/probes/readinessprobe"
 	agentHealthStatusFilePath = "/var/log/mongodb-mms-automation/agent-health-status.json"
 	clusterFilePath           = "/var/lib/automation/config/automation-config"
@@ -39,13 +41,21 @@ const (
 // Add creates a new MongoDB Controller and adds it to the Manager. The Manager will set fields on the Controller
 // and Start it when the Manager is Started.
 func Add(mgr manager.Manager) error {
-	return add(mgr, newReconciler(mgr))
+	return add(mgr, newReconciler(mgr, readVersionManifestFromDisk))
 }
 
+// ManifestProvider is a function which returns the VersionManifest which
+// contains the list of all available MongoDB versions
+type ManifestProvider func() (automationconfig.VersionManifest, error)
+
 // newReconciler returns a new reconcile.Reconciler
-func newReconciler(mgr manager.Manager) reconcile.Reconciler {
+func newReconciler(mgr manager.Manager, manifestProvider ManifestProvider) reconcile.Reconciler {
 	mgrClient := mgr.GetClient()
-	return &ReplicaSetReconciler{client: mdbClient.NewClient(mgrClient), scheme: mgr.GetScheme()}
+	return &ReplicaSetReconciler{
+		client:           mdbClient.NewClient(mgrClient),
+		scheme:           mgr.GetScheme(),
+		manifestProvider: manifestProvider,
+	}
 }
 
 // add adds a new Controller to mgr with r as the reconcile.Reconciler
@@ -71,8 +81,9 @@ var _ reconcile.Reconciler = &ReplicaSetReconciler{}
 type ReplicaSetReconciler struct {
 	// This client, initialized using mgr.Client() above, is a split client
 	// that reads objects from the cache and writes to the apiserver
-	client mdbClient.Client
-	scheme *runtime.Scheme
+	client           mdbClient.Client
+	scheme           *runtime.Scheme
+	manifestProvider func() (automationconfig.VersionManifest, error)
 }
 
 // Reconcile reads that state of the cluster for a MongoDB object and makes changes based on the state read
@@ -102,7 +113,7 @@ func (r *ReplicaSetReconciler) Reconcile(request reconcile.Request) (reconcile.R
 
 	// TODO: Read current automation config version from config map
 	if err := r.ensureAutomationConfig(mdb); err != nil {
-		log.Warnf("failed creating config map: %s", err)
+		log.Warnf("error creating automation config config map: %s", err)
 		return reconcile.Result{}, err
 	}
 
@@ -127,7 +138,7 @@ func (r *ReplicaSetReconciler) Reconcile(request reconcile.Request) (reconcile.R
 }
 
 func (r ReplicaSetReconciler) ensureAutomationConfig(mdb mdbv1.MongoDB) error {
-	cm, err := buildAutomationConfigConfigMap(mdb)
+	cm, err := r.buildAutomationConfigConfigMap(mdb)
 	if err != nil {
 		return err
 	}
@@ -137,7 +148,7 @@ func (r ReplicaSetReconciler) ensureAutomationConfig(mdb mdbv1.MongoDB) error {
 	return nil
 }
 
-func buildAutomationConfig(mdb mdbv1.MongoDB) automationconfig.AutomationConfig {
+func buildAutomationConfig(mdb mdbv1.MongoDB, mdbVersionConfig automationconfig.MongoDbVersionConfig) automationconfig.AutomationConfig {
 	domain := getDomain(mdb.ServiceName(), mdb.Namespace, "")
 	return automationconfig.NewBuilder().
 		SetTopology(automationconfig.ReplicaSetTopology).
@@ -146,10 +157,26 @@ func buildAutomationConfig(mdb mdbv1.MongoDB) automationconfig.AutomationConfig
 		SetMembers(mdb.Spec.Members).
 		SetMongoDBVersion(mdb.Spec.Version).
 		SetAutomationConfigVersion(1). // TODO: Correctly set the version
-		AddVersion(buildVersion406()).
+		AddVersion(mdbVersionConfig).
 		Build()
 }
 
+func readVersionManifestFromDisk() (automationconfig.VersionManifest, error) {
+	bytes, err := ioutil.ReadFile(versionManifestFilePath)
+	if err != nil {
+		return automationconfig.VersionManifest{}, err
+	}
+	return versionManifestFromBytes(bytes)
+}
+
+func versionManifestFromBytes(bytes []byte) (automationconfig.VersionManifest, error) {
+	versionManifest := automationconfig.VersionManifest{}
+	if err := json.Unmarshal(bytes, &versionManifest); err != nil {
+		return automationconfig.VersionManifest{}, err
+	}
+	return versionManifest, nil
+}
+
 // buildService creates a Service that will be used for the Replica Set StatefulSet
 // that allows all the members of the STS to see each other.
 // TODO: Make sure this Service is as minimal as posible, to not interfere with
@@ -167,36 +194,12 @@ func buildService(mdb mdbv1.MongoDB) corev1.Service {
 		Build()
 }
 
-// buildVersion406 returns a compatible version.
-func buildVersion406() automationconfig.MongoDbVersionConfig {
-	// TODO: For now we only have 2 versions, that match the versions used for testing.
-	return automationconfig.MongoDbVersionConfig{
-		Builds: []automationconfig.BuildConfig{
-			{
-				Architecture: "amd64",
-				GitVersion:   "caa42a1f75a56c7643d0b68d3880444375ec42e3",
-				Platform:     "linux",
-				Url:          "/linux/mongodb-linux-x86_64-rhel62-4.0.6.tgz",
-				Flavor:       "rhel",
-				MaxOsVersion: "8.0",
-				MinOsVersion: "7.0",
-			},
-			{
-				Architecture: "amd64",
-				GitVersion:   "caa42a1f75a56c7643d0b68d3880444375ec42e3",
-				Platform:     "linux",
-				Url:          "/linux/mongodb-linux-x86_64-ubuntu1604-4.0.6.tgz",
-				Flavor:       "ubuntu",
-				MaxOsVersion: "17.04",
-				MinOsVersion: "16.04",
-			},
-		},
-		Name: "4.0.6",
+func (r ReplicaSetReconciler) buildAutomationConfigConfigMap(mdb mdbv1.MongoDB) (corev1.ConfigMap, error) {
+	manifest, err := r.manifestProvider()
+	if err != nil {
+		return corev1.ConfigMap{}, fmt.Errorf("error reading version manifest from disk: %+v", err)
 	}
-}
-
-func buildAutomationConfigConfigMap(mdb mdbv1.MongoDB) (corev1.ConfigMap, error) {
-	ac := buildAutomationConfig(mdb)
+	ac := buildAutomationConfig(mdb, manifest.BuildsForVersion(mdb.Spec.Version))
 	acBytes, err := json.Marshal(ac)
 	if err != nil {
 		return corev1.ConfigMap{}, err

pkg/controller/mongodb/replicaset_controller_test.go

@@ -6,6 +6,8 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig"
+
 	mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1"
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client"
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/resourcerequirements"
@@ -34,12 +36,34 @@ func newTestReplicaSet() mdbv1.MongoDB {
 	}
 }
 
+func mockManifestProvider(version string) func() (automationconfig.VersionManifest, error) {
+	return func() (automationconfig.VersionManifest, error) {
+		return automationconfig.VersionManifest{
+			Updated: 0,
+			Versions: []automationconfig.MongoDbVersionConfig{
+				{
+					Name: version,
+					Builds: []automationconfig.BuildConfig{{
+						Platform:     "platform",
+						Url:          "url",
+						GitVersion:   "gitVersion",
+						Architecture: "arch",
+						Flavor:       "flavor",
+						MinOsVersion: "0",
+						MaxOsVersion: "10",
+						Modules:      []string{},
+					}},
+				}},
+		}, nil
+	}
+}
+
 func TestKubernetesResources_AreCreated(t *testing.T) {
 	// TODO: Create builder/yaml fixture of some type to construct MDB objects for unit tests
 	mdb := newTestReplicaSet()
 
 	mgr := client.NewManager(&mdb)
-	r := newReconciler(mgr)
+	r := newReconciler(mgr, mockManifestProvider(mdb.Spec.Version))
 
 	res, err := r.Reconcile(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}})
 	assertReconciliationSuccessful(t, res, err)
@@ -56,7 +80,7 @@ func TestKubernetesResources_AreCreated(t *testing.T) {
 func TestStatefulSet_IsCorrectlyConfigured(t *testing.T) {
 	mdb := newTestReplicaSet()
 	mgr := client.NewManager(&mdb)
-	r := newReconciler(mgr)
+	r := newReconciler(mgr, mockManifestProvider(mdb.Spec.Version))
 	res, err := r.Reconcile(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}})
 	assertReconciliationSuccessful(t, res, err)