feat(bootstrapper): add idempotent operations for secrets and organizations (#338)

## 📝 Description
This change ensures idempotency when bootstrapping installations,
allowing gen-secrets and init-org to be run multiple times safely.

Root Users credentials are now keps in authentication secret, because
this one is not managed by the helm chart and is preserved between
cluster restarts.

- Use CreateSecretIfNotExists instead of UpsertSecret for all secret
generation
- Ensure secrets aren't overwritten when they already exist
- Add Describe method to OrgClient to check if an organization exists 
- Add OrganizationExists function to verify existing organizations
- Modify CreateSemaphoreOrganization to use existing orgs when available
- Update init-org command to skip creation if organization already
exists
- Add corresponding unit tests for both secret and organization
functionality

New way of extracting Root User credentials

```bash
echo "Email: $(kubectl get secret semaphore-authentication -n default -o jsonpath='{.data.ROOT_USER_EMAIL}' | base64 -d)"; echo "Password: $(kubectl get secret semaphore-authentication -n default -o jsonpath='{.data.ROOT_USER_PASSWORD}' | base64 -d)"; echo "API Token: $(kubectl get secret semaphore-authentication -n default -o jsonpath='{.data.ROOT_USER_TOKEN}' | base64 -d)"
```

## ✅ Checklist
- [x] I have tested this change
- [x] This change requires documentation update

Author: radwo

bootstrapper/cmd/gen_secrets.go

@@ -33,7 +33,7 @@ var genSecretsCmd = &cobra.Command{
 
 func generateJWTSecret(client *kubernetes.KubernetesClient) {
 	secretName := utils.AssertEnv("JWT_SECRET_NAME")
-	err := client.UpsertSecret(secretName, map[string]string{
+	err := client.CreateSecretIfNotExists(secretName, map[string]string{
 		"logs":      random.Base64String(32),
 		"artifacts": random.Base64String(32),
 	})
@@ -45,7 +45,7 @@ func generateJWTSecret(client *kubernetes.KubernetesClient) {
 
 func generateAuthenticationSecret(client *kubernetes.KubernetesClient) {
 	secretName := utils.AssertEnv("AUTHENTICATION_SECRET_NAME")
-	err := client.UpsertSecret(secretName, map[string]string{
+	err := client.CreateSecretIfNotExists(secretName, map[string]string{
 		"SESSION_SECRET_KEY_BASE":   random.Base64String(64),
 		"TOKEN_HASHING_SALT":        random.Base64String(32),
 		"OIDC_CLIENT_SECRET":        random.Base64String(32),
@@ -55,13 +55,13 @@ func generateAuthenticationSecret(client *kubernetes.KubernetesClient) {
 	})
 
 	if err != nil {
-		log.Fatalf("Failed to generate JWT secrets: %v", err)
+		log.Fatalf("Failed to generate authentication secrets: %v", err)
 	}
 }
 
 func generateEncryptionKey(client *kubernetes.KubernetesClient) {
 	secretName := utils.AssertEnv("ENCRYPTION_SECRET_NAME")
-	err := client.UpsertSecret(secretName, map[string]string{
+	err := client.CreateSecretIfNotExists(secretName, map[string]string{
 		"key": random.Base64String(32),
 	})
 
@@ -102,7 +102,7 @@ func generateOpenIDSecret(client *kubernetes.KubernetesClient) error {
 	timestamp := strconv.FormatInt(time.Now().Unix(), 10)
 	privateKeyName := fmt.Sprintf("%s.pem", timestamp)
 
-	err = client.UpsertSecret(secretName, map[string]string{
+	err = client.CreateSecretIfNotExists(secretName, map[string]string{
 		privateKeyName: string(privateKey),
 	})
 
@@ -124,7 +124,7 @@ func generateVaultSecret(client *kubernetes.KubernetesClient) error {
 	privateKeyName := fmt.Sprintf("%s.prv.pem", timestamp)
 	publicKeyName := fmt.Sprintf("%s.pub.pem", timestamp)
 
-	err = client.UpsertSecret(secretName, map[string]string{
+	err = client.CreateSecretIfNotExists(secretName, map[string]string{
 		privateKeyName: string(privateKey),
 		publicKeyName:  string(publicKey),
 	})

bootstrapper/cmd/gen_secrets_test.go

@@ -99,12 +99,14 @@ func TestGenerateOpenIDSecret(t *testing.T) {
 
 	// Verify there's exactly one key pair with correct format
 	foundPrivate := false
+	originalKeyName := ""
 
 	// Helper function to check keys in either StringData or Data
 	checkKeys := func(data map[string]string) {
 		for k, v := range data {
 			if strings.HasSuffix(k, ".pem") && strings.Contains(v, "-----BEGIN RSA PRIVATE KEY-----") {
 				foundPrivate = true
+				originalKeyName = k
 			}
 		}
 	}
@@ -122,6 +124,30 @@ func TestGenerateOpenIDSecret(t *testing.T) {
 	}
 
 	assert.True(t, foundPrivate, "No valid RSA private key found in secret")
+
+	// Now test that calling it again doesn't change the secret
+	err = generateOpenIDSecret(mockClient.KubernetesClient)
+	assert.NoError(t, err)
+
+	// Get the secret again
+	updatedSecret, err := mockClient.Clientset.CoreV1().Secrets("test").Get(context.Background(), "openid-secret", metav1.GetOptions{})
+	assert.NoError(t, err)
+
+	// Verify the original key is still there and no new key was added
+	found := false
+	keyCount := 0
+
+	// Check in StringData or Data
+	if len(updatedSecret.StringData) > 0 {
+		keyCount = len(updatedSecret.StringData)
+		_, found = updatedSecret.StringData[originalKeyName]
+	} else if len(updatedSecret.Data) > 0 {
+		keyCount = len(updatedSecret.Data)
+		_, found = updatedSecret.Data[originalKeyName]
+	}
+
+	assert.True(t, found, "Original key should still be present")
+	assert.Equal(t, 1, keyCount, "There should still be only one key")
 }
 
 func TestGenerateVaultSecret(t *testing.T) {
@@ -145,6 +171,8 @@ func TestGenerateVaultSecret(t *testing.T) {
 	foundPrivate := false
 	foundPublic := false
 	timestamp := ""
+	originalPrivateKey := ""
+	originalPublicKey := ""
 
 	// Helper function to check keys in either StringData or Data
 	checkKeys := func(data map[string]string) {
@@ -153,9 +181,11 @@ func TestGenerateVaultSecret(t *testing.T) {
 				foundPrivate = true
 				// Extract timestamp from the key name
 				timestamp = strings.TrimSuffix(k, ".prv.pem")
+				originalPrivateKey = k
 			}
 			if strings.HasSuffix(k, ".pub.pem") && strings.Contains(v, "-----BEGIN RSA PUBLIC KEY-----") {
 				foundPublic = true
+				originalPublicKey = k
 			}
 		}
 	}
@@ -179,6 +209,92 @@ func TestGenerateVaultSecret(t *testing.T) {
 	if foundPrivate && foundPublic {
 		assert.Contains(t, secret.StringData, fmt.Sprintf("%s.pub.pem", timestamp), "Public key name does not match private key timestamp")
 	}
+
+	// Test CreateSecretIfNotExists behavior by calling generateVaultSecret again
+	err = generateVaultSecret(mockClient.KubernetesClient)
+	assert.NoError(t, err)
+
+	// Get the secret again
+	updatedSecret, err := mockClient.Clientset.CoreV1().Secrets("test").Get(context.Background(), utils.AssertEnv("VAULT_SECRET_NAME"), metav1.GetOptions{})
+	assert.NoError(t, err)
+
+	// Verify the original keys are still there and no new keys were added
+	foundOrigPrivate := false
+	foundOrigPublic := false
+	keyCount := 0
+
+	// Check in StringData or Data
+	if len(updatedSecret.StringData) > 0 {
+		keyCount = len(updatedSecret.StringData)
+		_, foundOrigPrivate = updatedSecret.StringData[originalPrivateKey]
+		_, foundOrigPublic = updatedSecret.StringData[originalPublicKey]
+	} else if len(updatedSecret.Data) > 0 {
+		keyCount = len(updatedSecret.Data)
+		_, foundOrigPrivate = updatedSecret.Data[originalPrivateKey]
+		_, foundOrigPublic = updatedSecret.Data[originalPublicKey]
+	}
+
+	assert.True(t, foundOrigPrivate, "Original private key should still be present")
+	assert.True(t, foundOrigPublic, "Original public key should still be present")
+	assert.Equal(t, 2, keyCount, "There should still be exactly two keys")
+}
+
+func TestBasicSecretsGeneration(t *testing.T) {
+	// Set required env vars
+	cleanup := setTestEnv("")
+	defer cleanup()
+
+	mockClient := NewMockKubernetesClient()
+
+	// First generation of secrets
+	generateJWTSecret(mockClient.KubernetesClient)
+	generateAuthenticationSecret(mockClient.KubernetesClient)
+	generateEncryptionKey(mockClient.KubernetesClient)
+
+	// Verify the secrets were created
+	secretNames := []string{
+		utils.AssertEnv("JWT_SECRET_NAME"),
+		utils.AssertEnv("AUTHENTICATION_SECRET_NAME"),
+		utils.AssertEnv("ENCRYPTION_SECRET_NAME"),
+	}
+
+	for _, secretName := range secretNames {
+		secret, err := mockClient.Clientset.CoreV1().Secrets("test").Get(context.Background(), secretName, metav1.GetOptions{})
+		assert.NoError(t, err)
+		assert.NotNil(t, secret)
+
+		// Store original data for comparison
+		originalData := make(map[string]string)
+		if len(secret.StringData) > 0 {
+			for k, v := range secret.StringData {
+				originalData[k] = v
+			}
+		} else if len(secret.Data) > 0 {
+			for k, v := range secret.Data {
+				originalData[k] = string(v)
+			}
+		}
+
+		// Call generation functions again
+		generateJWTSecret(mockClient.KubernetesClient)
+		generateAuthenticationSecret(mockClient.KubernetesClient)
+		generateEncryptionKey(mockClient.KubernetesClient)
+
+		// Verify secrets haven't changed
+		updatedSecret, err := mockClient.Clientset.CoreV1().Secrets("test").Get(context.Background(), secretName, metav1.GetOptions{})
+		assert.NoError(t, err)
+
+		// Check that data is the same
+		if len(updatedSecret.StringData) > 0 {
+			for k, v := range updatedSecret.StringData {
+				assert.Equal(t, originalData[k], v, "Secret data should not have changed for %s", secretName)
+			}
+		} else if len(updatedSecret.Data) > 0 {
+			for k, v := range updatedSecret.Data {
+				assert.Equal(t, originalData[k], string(v), "Secret data should not have changed for %s", secretName)
+			}
+		}
+	}
 }
 
 func TestSecretGenerationForEditions(t *testing.T) {

bootstrapper/cmd/init_org.go

@@ -32,7 +32,7 @@ var initOrgCmd = &cobra.Command{
 		orgUsername := utils.AssertEnv("ORGANIZATION_USERNAME")
 		userName := utils.AssertEnv("ROOT_NAME")
 		userEmail := utils.AssertEnv("ROOT_EMAIL")
-		rootUserSecretName := utils.AssertEnv("ROOT_USER_SECRET_NAME")
+		authenticationSecretName := utils.AssertEnv("AUTHENTICATION_SECRET_NAME")
 
 		kubernetesClient := kubernetes.NewClient()
 		instanceConfigClient := clients.NewInstanceConfigClient()
@@ -44,7 +44,15 @@ var initOrgCmd = &cobra.Command{
 		//
 		waitForIngress(domain)
 
-		userId := user.CreateSemaphoreUser(kubernetesClient, userName, userEmail, rootUserSecretName)
+		// First check if the organization already exists
+		exists, existingOrgId := organization.OrganizationExists(orgUsername)
+		if exists {
+			log.Infof("Organization %s already exists with ID %s. Skipping organization creation.", orgUsername, existingOrgId)
+			// Return early since organization already exists
+			return
+		}
+
+		userId := user.CreateSemaphoreUser(kubernetesClient, userName, userEmail, authenticationSecretName)
 		orgId := organization.CreateSemaphoreOrganization(orgUsername, userId)
 
 		if os.Getenv("DEFAULT_AGENT_TYPE_ENABLED") == "true" {