diff --git a/api/v1/passboltsecret_types.go b/api/v1/passboltsecret_types.go
index de63b5e5..54c26030 100644
--- a/api/v1/passboltsecret_types.go
+++ b/api/v1/passboltsecret_types.go
@@ -112,6 +112,11 @@ type PassboltSecretStatus struct {
 	LastSync metav1.Time `json:"lastSync"`
 	// SyncErrors is a list of errors that occurred during the last sync.
 	SyncErrors []SyncError `json:"syncErrors,omitempty"`
+	// FailureCount is the number of times the secret failed to sync.
+	// This is used to determine if the secret should be retried.
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default=0
+	FailureCount int `json:"failureCount,omitempty"`
 }
 
 //+kubebuilder:object:root=true
diff --git a/config/crd/bases/passbolt.tagesspiegel.de_passboltsecrets.yaml b/config/crd/bases/passbolt.tagesspiegel.de_passboltsecrets.yaml
index ee80ece8..fe5a88cc 100644
--- a/config/crd/bases/passbolt.tagesspiegel.de_passboltsecrets.yaml
+++ b/config/crd/bases/passbolt.tagesspiegel.de_passboltsecrets.yaml
@@ -96,6 +96,11 @@ spec:
           status:
             description: PassboltSecretStatus defines the observed state of PassboltSecret
             properties:
+              failureCount:
+                default: 0
+                description: FailureCount is the number of times the secret failed
+                  to sync. This is used to determine if the secret should be retried.
+                type: integer
               lastSync:
                 description: LastSync is the last time the secret was synced from
                   passbolt.
diff --git a/e2e/lib.sh b/e2e/lib.sh
index edd94651..e1a2766b 100644
--- a/e2e/lib.sh
+++ b/e2e/lib.sh
@@ -143,3 +143,25 @@ EOF
 )"
     sleep 5
 }
+
+# createPassboltSecretV1 <name>
+function createPassboltSecretV1WithSecretNotFound() {
+    createPassboltSecret "$(cat <<EOF
+apiVersion: passbolt.tagesspiegel.de/v1
+kind: PassboltSecret
+metadata:
+  name: ${1}
+spec:
+  leaveOnDelete: false
+  secretType: Opaque
+  passboltSecrets:
+    secret:
+      id: 00000000-0000-0000-0000-000000000000
+      field: username
+  plainTextFields:
+    key: value
+    foo: bar
+EOF
+)"
+    sleep 5
+}
diff --git a/e2e/run.sh b/e2e/run.sh
index fa39d9b4..77f2a09f 100755
--- a/e2e/run.sh
+++ b/e2e/run.sh
@@ -87,3 +87,24 @@ compareLength "6" ${payload_length}
 echo -e "${color_blue}Checking if Kubernetes secret ${color_yellow}${secret_name}${color_blue} exists and has the right .data length${NC}"
 payload_length=$(getKubernetesSecret ${secret_name} | jq -r ".data | length")
 compareLength "6" ${payload_length}
+
+############################
+########## Test 4 ##########
+############################
+api_version="v1"
+echo -e "${color_magenta}4: Testing API version: ${api_version}${color_reset}"
+secret_name="${api_version}-backoff-check"
+createPassboltSecretV1WithSecretNotFound ${secret_name}
+
+echo -e $(getPassboltSecret ${secret_name} ${api_version})
+
+sync_status=$(getPassboltSecret ${secret_name} ${api_version} | jq -r ".status.lastSync")
+
+# check if status is not Success
+if [ "${sync_status}" != "Success" ]; then
+    echo -e "${color_red}Expected status to be not ${color_yellow}Success${color_red} but got ${color_yellow}${sync_status}${color_reset}"
+    exit 1
+fi
+
+payload_length=$(getPassboltSecret ${secret_name} ${api_version} | jq -r "(.spec.passboltSecrets | length) + (.spec.plainTextFields | length)")
+compareLength "0" ${payload_length}
diff --git a/internal/controller/passboltsecret_controller.go b/internal/controller/passboltsecret_controller.go
index c2e7ac90..71ad6fc1 100644
--- a/internal/controller/passboltsecret_controller.go
+++ b/internal/controller/passboltsecret_controller.go
@@ -83,6 +83,10 @@ func (r *PassboltSecretReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 		}
 		return errResult, err
 	}
+
+	// limit the number of retries
+	errResult.RequeueAfter = time.Duration(2^secret.Status.FailureCount) * (5 * time.Second)
+
 	// cleanup status
 	secret.Status.SyncErrors = []passboltv1.SyncError{}
 
@@ -94,6 +98,7 @@ func (r *PassboltSecretReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 	if secret.Spec.SecretType != corev1.SecretTypeOpaque && secret.Spec.SecretType != corev1.SecretTypeDockerConfigJson {
 		logr.Info("unsupported secret type", "type", secret.Spec.SecretType)
 		secret.Status.SyncStatus = passboltv1.SyncStatusError
+		secret.Status.FailureCount++
 		secret.Status.SyncErrors = append(secret.Status.SyncErrors, passboltv1.SyncError{
 			Message: fmt.Sprintf("unsupported secret type %q", secret.Spec.SecretType),
 			Time:    metav1.Now(),
@@ -118,8 +123,14 @@ func (r *PassboltSecretReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 
 	opRslt, err := controllerutil.CreateOrUpdate(ctx, r.Client, k8sSecret, util.UpdateSecret(ctx, r.PassboltClient, r.Scheme, secret, k8sSecret))
 	if err != nil {
+		if secret.Status.FailureCount >= 3 {
+			// if the secret failed to sync more than 3 times, we stop trying
+			logr.Info("secret failed to sync more than 3 times. stopping sync", "name", secret.GetName(), "namespace", secret.GetNamespace())
+			return ctrl.Result{}, nil
+		}
 		if snErr, ok := err.(passboltv1.SyncError); ok {
 			secret.Status.SyncStatus = passboltv1.SyncStatusError
+			secret.Status.FailureCount++
 			secret.Status.SyncErrors = append(secret.Status.SyncErrors, snErr)
 			if err := r.Client.Status().Update(ctx, secret); err != nil {
 				return errResult, err
@@ -128,6 +139,8 @@ func (r *PassboltSecretReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 		}
 		return errResult, err
 	}
+	// reset failure counter to 0
+	secret.Status.FailureCount = 0
 
 	// if the secret was not changed and the status is already success, we can skip the update
 	if opRslt == controllerutil.OperationResultNone && secret.Status.SyncStatus == passboltv1.SyncStatusSuccess {