chore: add support for vcr compression and trim k8s cassettes (#3136)

* chore: add support for vcr compression and trim k8s cassettes

* Fix

* Fix linter

* Fix i18n for error message

* Apply review suggestions

* Apply review suggestions

* Fix lint

* add documentation

* Fix useless continue

Author: remyleone

TESTING.md

@@ -62,3 +62,16 @@ Running a single test:
 ```sh
 TF_UPDATE_CASSETTES=true TF_LOG=DEBUG SCW_DEBUG=1 TF_ACC=1 go test ./scaleway -v -run=TestAccScalewayDataSourceRDBInstance_Basic -timeout=120m -parallel=10
 ```
+
+## Compressing the cassettes
+
+We record interactions with the Scaleway API in cassettes, which are stored in the `testdata` directory of each service.
+Each wait function used in the resources will perform several requests to the API for pulling a resource state, which can lead to large cassettes.
+We use a compressor to reduce the size of these cassettes once they are recorded.
+By doing so, tests can run faster and the cassettes are easier to read.
+
+To use the compressor on a given cassette, run the following command:
+
+```sh
+go run -v ./cmd/vcr-compressor internal/services/rdb/testdata/acl-basic.cassette
+```

cmd/vcr-compressor/main.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"os"
+
+	"github.com/scaleway/scaleway-sdk-go/api/k8s/v1"
+	"gopkg.in/dnaeon/go-vcr.v3/cassette"
+)
+
+var transientStates = map[string]bool{
+	k8s.ClusterStatusCreating.String(): true,
+	k8s.ClusterStatusDeleting.String(): true,
+	k8s.ClusterStatusUpdating.String(): true,
+	k8s.PoolStatusDeleting.String():    true,
+	k8s.PoolStatusScaling.String():     true,
+	k8s.PoolStatusUpgrading.String():   true,
+}
+
+func main() {
+	if len(os.Args) < 2 {
+		log.Fatalf("Usage: %s <cassette_file_name_without_yaml>\n", os.Args[0])
+	}
+
+	path := os.Args[1]
+
+	inputCassette, err := cassette.Load(path)
+	if err != nil {
+		log.Fatalf("Error while reading file : %v\n", err)
+	}
+
+	outputCassette := cassette.New(path)
+	transitioning := false
+
+	for i := range len(inputCassette.Interactions) {
+		interaction := inputCassette.Interactions[i]
+		responseBody := interaction.Response.Body
+		requestMethod := interaction.Request.Method
+
+		if requestMethod != "GET" {
+			transitioning = false
+
+			log.Printf("Interaction %d is not a GET request. Recording it\n", i)
+			outputCassette.AddInteraction(interaction)
+
+			continue
+		}
+
+		if responseBody == "" {
+			log.Printf("Interaction %d got an empty response body. Recording it\n", i)
+			outputCassette.AddInteraction(interaction)
+
+			continue
+		}
+
+		var m map[string]interface{}
+
+		err := json.Unmarshal([]byte(responseBody), &m)
+		if err != nil {
+			log.Printf("Interaction %d have an error with unmarshalling response body: %v. Recording it\n", i, err)
+			outputCassette.AddInteraction(interaction)
+
+			continue
+		}
+
+		if m["status"] == nil {
+			log.Printf("Interaction %d does not contain a status field. Recording it\n", i)
+			outputCassette.AddInteraction(interaction)
+
+			continue
+		}
+
+		status := m["status"].(string)
+		// We test if the state is transient
+		if _, ok := transientStates[status]; ok {
+			if transitioning {
+				log.Printf("Interaction %d is in a transient state while we are already in transitient state. No need to record it: %s\n", i, status)
+			} else {
+				log.Printf("Interaction %d is in a transient state: %s, Recording it\n", i, status)
+
+				transitioning = true
+
+				outputCassette.AddInteraction(interaction)
+			}
+		} else {
+			if transitioning {
+				log.Printf("Interaction %d is not in a transient state anymore: %s, Recording it\n", i, status)
+
+				outputCassette.AddInteraction(interaction)
+
+				transitioning = false
+			} else {
+				log.Printf("Interaction %d is not in a transient state: %s, Recording it\n", i, status)
+				outputCassette.AddInteraction(interaction)
+			}
+		}
+	}
+
+	err = outputCassette.Save()
+	if err != nil {
+		log.Fatalf("error while saving file: %v", err)
+	}
+}

cmd/vcr-viewer/main.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"os"
+
+	"gopkg.in/dnaeon/go-vcr.v3/cassette"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		log.Fatalf("Usage: %s <cassette_file_name_without_yaml>\n", os.Args[0])
+	}
+
+	path := os.Args[1]
+
+	data, err := cassette.Load(path)
+	if err != nil {
+		log.Fatalf("Error while reading file: %v\n", err)
+	}
+
+	for i := range len(data.Interactions) {
+		interaction := data.Interactions[i]
+
+		log.Println("--------------")
+		log.Printf("Interaction %d:\n", i+1)
+		log.Printf("  Request:\n")
+		log.Printf("    Method: %s\n", interaction.Request.Method)
+		log.Printf("    URL: %s\n", interaction.Request.URL)
+
+		if interaction.Request.Body != "" {
+			log.Printf("    Body: %s\n", interaction.Request.Body)
+		}
+
+		log.Printf("  Response:\n")
+		log.Printf("    Status: %s\n", interaction.Response.Status)
+		log.Printf("    Body: %s\n", interaction.Response.Body)
+
+		var m map[string]interface{}
+
+		err := json.Unmarshal([]byte(interaction.Response.Body), &m)
+		if err != nil {
+			continue
+		}
+
+		if m["status"] != nil {
+			log.Println("++++++++++++++++")
+			log.Printf("status: %s\n", m["status"])
+			log.Println("++++++++++++++++")
+		}
+
+		log.Println("--------------")
+	}
+}