retry proto registration

Author: Tofel

.github/workflows/framework-dockercompose-tests.yml

@@ -18,7 +18,7 @@ jobs:
           - name: TestChipIngressSmoke
             config: smoke_chip.toml
             count: 1
-            timeout: 10m
+            timeout: 3m
     name: ${{ matrix.test.name }}
     steps:
       - name: Checkout repo
@@ -71,6 +71,7 @@ jobs:
         env:
           CTF_CONFIGS: ${{ matrix.test.config }}
           CHIP_INGRESS_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.us-west-2.amazonaws.com/atlas-chip-ingress:qa-latest
+          CTF_LOG_LEVEL: debug
         run: |
           go test -timeout ${{ matrix.test.timeout }} -v -count ${{ matrix.test.count }} -run ${{ matrix.test.name }}
 

framework/components/dockercompose/chip_ingress_set/chip_ingress.go

@@ -128,6 +128,7 @@ func New(in *Input) (*Output, error) {
 			wait.NewHostPortStrategy(DEFAULT_RED_PANDA_SCHEMA_REGISTRY_PORT).WithPollInterval(100*time.Millisecond),
 			wait.NewHostPortStrategy(DEFAULT_RED_PANDA_KAFKA_PORT).WithPollInterval(100*time.Millisecond),
 			wait.ForHTTP("/v1/status/ready").WithPort("9644"), // admin API port
+			wait.ForHTTP("/status/ready").WithPort(DEFAULT_RED_PANDA_SCHEMA_REGISTRY_PORT).WithPollInterval(100*time.Millisecond),
 		).WithDeadline(2*time.Minute),
 	).WaitForService(DEFAULT_RED_PANDA_CONSOLE_SERVICE_NAME,
 		wait.ForAll(

framework/components/dockercompose/chip_ingress_set/protos.go

@@ -11,7 +11,9 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"time"
 
+	"github.com/avast/retry-go/v4"
 	"github.com/google/go-github/v72/github"
 	"github.com/pkg/errors"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework"
@@ -510,23 +512,33 @@ func registerSingleProto(
 	}
 
 	url := fmt.Sprintf("%s/subjects/%s/versions", registryURL, subject)
-	// Force IPv4 to avoid Docker IPv6 port forwarding issues
-	url = strings.Replace(url, "localhost", "127.0.0.1", 1)
-	framework.L.Debug().Msgf("Registering schema to URL: %s", url)
+	maxAttempts := uint(10)
 
-	resp, respErr := http.Post(url, "application/vnd.schemaregistry.v1+json", bytes.NewReader(payload))
-	if respErr != nil {
-		return 0, errors.Wrap(respErr, "failed to post to schema registry")
-	}
-	defer resp.Body.Close()
+	var resp *http.Response
+	retry.Do(func() error {
+		var respErr error
+		resp, respErr = http.Post(url, "application/vnd.schemaregistry.v1+json", bytes.NewReader(payload))
+		if respErr != nil {
+			return errors.Wrap(respErr, "failed to post to schema registry")
+		}
 
-	if resp.StatusCode >= 300 {
-		data, dataErr := io.ReadAll(resp.Body)
-		if dataErr != nil {
-			return 0, errors.Wrap(dataErr, "failed to read response body")
+		if resp.StatusCode >= 300 {
+			data, dataErr := io.ReadAll(resp.Body)
+			if dataErr != nil {
+				return errors.Wrap(dataErr, "failed to read response body")
+			}
+			return fmt.Errorf("schema registry error (%d): %s", resp.StatusCode, data)
 		}
-		return 0, fmt.Errorf("schema registry error (%d): %s", resp.StatusCode, data)
-	}
+
+		return nil
+	}, retry.Attempts(maxAttempts), retry.Delay(100*time.Millisecond), retry.DelayType(retry.BackOffDelay), retry.OnRetry(func(n uint, err error) {
+		framework.L.Debug().Str("attempt/max", fmt.Sprintf("%d/%d", n, maxAttempts)).Msgf("Retrying to register schema %s: %v", subject, err)
+	}), retry.RetryIf(func(err error) bool {
+		// we don't want to retry all errors, because some of them are are expected (e.g. missing dependencies)
+		// and will be handled by higher-level code
+		return strings.Contains(err.Error(), "connection reset by peer")
+	}))
+	defer resp.Body.Close()
 
 	var result struct {
 		ID int `json:"id"`

framework/components/dockercompose/go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
+	github.com/avast/retry-go/v4 v4.6.1 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.31.0 // indirect
 	github.com/aws/aws-sdk-go-v2/config v1.27.39 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.37 // indirect

framework/components/dockercompose/go.sum

@@ -40,6 +40,8 @@ github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew
 github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/avast/retry-go/v4 v4.6.1 h1:VkOLRubHdisGrHnTu89g08aQEWEgRU7LVEop3GbIcMk=
+github.com/avast/retry-go/v4 v4.6.1/go.mod h1:V6oF8njAwxJ5gRo1Q7Cxab24xs5NCWZBeaHHBklR8mA=
 github.com/aws/aws-sdk-go-v2 v1.31.0 h1:3V05LbxTSItI5kUqNwhJrrrY1BAXxXt0sN0l72QmG5U=
 github.com/aws/aws-sdk-go-v2 v1.31.0/go.mod h1:ztolYtaEUtdpf9Wftr31CJfLVjOnD/CVRkKOOYgF8hA=
 github.com/aws/aws-sdk-go-v2/config v1.27.39 h1:FCylu78eTGzW1ynHcongXK9YHtoXD5AiiUqq3YfJYjU=