Feature/compress user data (#14)

* Compress user_data using gzip to avoid to exceed the hard limit (32KB)

* Fix a lint error

* Extract a function compressing bytes to the cloud package

* Pass buffer to CompressData function and add unit tests

* Change the function to compress and encode string

* Fix typos

* Remove code added mistakenly during merge

* Use Close instead of Flush because  Flush doesn't work in Docker-Linux environment

Co-authored-by: Wonkun Kim <[email protected]>

Author: wongni

controllers/cloudstackmachine_controller.go

@@ -18,7 +18,6 @@ package controllers
 
 import (
 	"context"
-	"encoding/base64"
 	"errors"
 	"fmt"
 	"reflect"
@@ -167,10 +166,9 @@ func (r *CloudStackMachineReconciler) reconcile(
 	if !ok {
 		return ctrl.Result{}, errors.New("Bootstrap secret data not ok.")
 	}
-	userData := base64.StdEncoding.EncodeToString(value)
 
 	// Create VM (or Fetch if present). Will set ready to true.
-	if err := r.CS.GetOrCreateVMInstance(csMachine, machine, csCluster, userData); err == nil {
+	if err := r.CS.GetOrCreateVMInstance(csMachine, machine, csCluster, string(value)); err == nil {
 		if !controllerutil.ContainsFinalizer(csMachine, infrav1.MachineFinalizer) { // Fetched or Created?
 			log.Info("CloudStack instance Created", "instanceStatus", csMachine.Status, "instanceSpec", csMachine.Spec)
 			controllerutil.AddFinalizer(csMachine, infrav1.MachineFinalizer)

pkg/cloud/helpers.go

@@ -16,10 +16,27 @@ limitations under the License.
 
 package cloud
 
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/base64"
+)
+
 type set func(string)
 
 func setIfNotEmpty(str string, setFn set) {
 	if str != "" {
 		setFn(str)
 	}
 }
+
+func CompressAndEncodeString(str string) (string, error) {
+	buf := &bytes.Buffer{}
+	gzipWriter := gzip.NewWriter(buf)
+	if _, err := gzipWriter.Write([]byte(str)); err != nil {
+		gzipWriter.Close()
+		return "", err
+	}
+	gzipWriter.Close()
+	return base64.StdEncoding.EncodeToString(buf.Bytes()), nil
+}

pkg/cloud/helpers_test.go

@@ -17,6 +17,10 @@ limitations under the License.
 package cloud_test
 
 import (
+	"bytes"
+	"compress/gzip"
+	"encoding/base64"
+	"io/ioutil"
 	"os"
 	"path"
 
@@ -42,6 +46,19 @@ var _ = Describe("Helpers", func() {
 			Ω(err.Error()).Should(ContainSubstring("section Global not found"))
 		})
 	})
+
+	It("should compress and encode string", func() {
+		str := "Hello World"
+
+		compressedAndEncodedData, err := cloud.CompressAndEncodeString(str)
+
+		compressedData, _ := base64.StdEncoding.DecodeString(compressedAndEncodedData)
+		reader, _ := gzip.NewReader(bytes.NewReader(compressedData))
+		result, _ := ioutil.ReadAll(reader)
+
+		Ω(err).Should(BeNil())
+		Ω(string(result)).Should(Equal(str))
+	})
 })
 
 func getConfigPath(filename string) string {

pkg/cloud/instance.go

@@ -150,10 +150,19 @@ func (c *client) GetOrCreateVMInstance(
 	setIfNotEmpty(csMachine.Name, p.SetName)
 	setIfNotEmpty(csMachine.Name, p.SetDisplayname)
 	setIfNotEmpty(csMachine.Spec.SSHKey, p.SetKeypair)
-	setIfNotEmpty(userData, p.SetUserdata)
-	p.SetAffinitygroupids(csMachine.Spec.AffinityGroupIds)
+
+	if compressedAndEncodedUserData, err := CompressAndEncodeString(userData); err != nil {
+		return err
+	} else {
+		setIfNotEmpty(compressedAndEncodedUserData, p.SetUserdata)
+	}
+
+	if len(csMachine.Spec.AffinityGroupIds) > 0 {
+		p.SetAffinitygroupids(csMachine.Spec.AffinityGroupIds)
+	}
 	setIfNotEmpty(csCluster.Spec.Account, p.SetAccount)
 	setIfNotEmpty(csCluster.Status.DomainID, p.SetDomainid)
+
 	// If this VM instance is a control plane, consider setting it's IP.
 	_, isControlPlanceMachine := machine.ObjectMeta.Labels["cluster.x-k8s.io/control-plane"]
 	if isControlPlanceMachine && csCluster.Status.NetworkType == NetworkTypeShared {