feat: ability to reference existing sm (#45)

* feat: bump submodule

* feat: bump versions

* feat: update stack reference architecture

* docs: remove documentation around manual steps

* fix: fix issue causing clashes in code engine names

* feat: bump to rag app da v1.5.0

* chore: bump to Secret Manager to version v1.12.0 Security Compliance Center to version v1.6.3

* feat: allow to pass crn to existing secret manager instance

* feat: add input wrt platform metrics and logging

* chore: bump DA versions

* chore: bump DA versions

* fix: obs to use resource group

* fix: missing secret manager region

* fix: reverse sm config

* feat: bump sm to v1.12.2

* fix: some minor improvements to deploy-many scripts

* fix: missing rg config in sm stack

* docs: update doc with new options

* fix: point to correct region for existing sm

Author: vburckhardt

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "go.sum|^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2024-04-29T15:33:54Z",
+  "generated_at": "2024-04-30T13:49:16Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -82,23 +82,23 @@
         "hashed_secret": "bbc4e9d52252171a3a306be55086c65b126189e8",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 32,
+        "line_number": 34,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "d9e9019d9eb455a3d72a3bc252c26927bb148a10",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 47,
+        "line_number": 51,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "b13d7622394e85c3b2694f426bc096b093764462",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 51,
+        "line_number": 55,
         "type": "Secret Keyword",
         "verified_result": null
       }

README.md

@@ -24,6 +24,8 @@ Before deploying the stack, ensure you have:
 **Important**:
 - Ensure region is either us-south or eu-de as watsonx can only be deployed in those 2 locations for now.
 - Ensure that the prefix is globally unique. It is used for the container registry namespace (which needs to be globally unique) in this alpha version.
+- If specifying `existing_secrets_manager_crn`, the ibmcloud_api_key that is passed as an input must have the documented read and write access to the instance
+- If specifying `existing_secrets_manager_crn`, ensure that the default security group does not contain secrets named `signing-key` and `ibmcloud-api-key` . The RAG DA currently always attempt to create secret with those names (temporary issue - to be fixed).
 
 ```json
 {
@@ -34,7 +36,9 @@ Before deploying the stack, ensure you have:
         "region": "<region where all resources are deployed>",
         "sample_app_git_url": "https://github.com/IBM/gen-ai-rag-watsonx-sample-application",
         "watsonx_admin_api_key": "<optional - admin key to use for watson if different from ibmcloud_api_key>",
-        "signing_key": "signing key used to sign build artifacts"
+        "signing_key": "signing key used to sign build artifacts",
+        "existing_secrets_manager_crn": "<optional> - reuse an existing secret manager instance",
+        "enable_platform_logs_metrics": "<optional> - set to true to enable observability instance to capture regional logs"
     }
 }
 ```
@@ -49,7 +53,9 @@ Example:
         "region": "eu-de",
         "sample_app_git_url": "https://github.com/IBM/gen-ai-rag-watsonx-sample-application",
         "watsonx_admin_api_key": "<optional - admin key to use for watson if different from ibmcloud_api_key>",
-        "signing_key": "signing key used to sign build artifacts"
+        "signing_key": "signing key used to sign build artifacts",
+        "enable_platform_logs_metrics": "false",
+        "existing_secrets_manager_crn": "crn:v1:bluemix:public:secrets-manager:us-south:a/190c293e9fda4c6684b5acf4b17871b8:14580411-4fa2-42d3-af3f-ab7fc6371b6d::"
     }
 }
 ```

deploy-many.sh

@@ -63,7 +63,7 @@ function validate_config() {
 
   STATE=$(get_validation_state)
 
-  if [[ "$STATE" != "validated" && "$STATE" != "deployed" && "$STATE" != "deploying_failed" ]]; then
+  if [[ "$STATE" != "validated" && "$STATE" != "deployed" && "$STATE" != "deploying_failed" && "$STATE" != "approved" ]]; then
     $CLI_CMD project config-validate --project-id "$PROJECT_ID" --id "$CONFIG_ID" --output json > /tmp/validation.json
   fi
 }
@@ -74,7 +74,7 @@ function wait_for_validation() {
 
     STATE=$(get_validation_state)
 
-    if [[ "$STATE" == "validated" || "$STATE" == "deployed" || "$STATE" == "deploying_failed" ]]; then
+    if [[ "$STATE" == "validated" || "$STATE" == "deployed" || "$STATE" == "deploying_failed" || "$STATE" == "approved" || "$STATE" == "deploying" ]]; then
       break
     fi
 

stack_definition.json

@@ -71,7 +71,7 @@
   "members": [
     {
       "name": "1 - Account Infrastructure Base",
-      "version_locator": "7df1e4ca-d54c-4fd0-82ce-3d13247308cd.65c68525-b954-460c-96e6-ad7300c3b317",
+      "version_locator": "7df1e4ca-d54c-4fd0-82ce-3d13247308cd.c8bf5993-d693-4cac-9479-83c5fb6f96e1",
       "inputs": [
         {
           "name": "prefix",
@@ -139,7 +139,7 @@
     },
     {
       "name": "2b - Security Service - Secret Manager",
-      "version_locator": "7df1e4ca-d54c-4fd0-82ce-3d13247308cd.e9f80ad1-b9a0-4884-be72-650403bdcf76",
+      "version_locator": "7df1e4ca-d54c-4fd0-82ce-3d13247308cd.7230f742-7f93-4647-b53d-89b7905fb5a6",
       "inputs": [
         {
           "name": "prefix",
@@ -162,8 +162,12 @@
           "value": "ref:../2a - Security Service - Key Management/outputs/key_protect_crn"
         },
         {
-          "name": "existing_resource_group",
+          "name": "use_existing_resource_group",
           "value": true
+        },
+        {
+          "name": "existing_secrets_manager_crn",
+          "value": "ref:../../inputs/existing_secrets_manager_crn"
         }
       ]
     },
@@ -225,10 +229,10 @@
           "name": "prefix",
           "value": "ref:../../inputs/prefix"
         },
-				{
-					"name": "use_existing_resource_group",
-					"value": true
-				},
+        {
+          "name": "use_existing_resource_group",
+          "value": true
+        },
         {
           "name": "enable_platform_logs",
           "value": "ref:../../inputs/enable_platform_logs_metrics"
@@ -241,7 +245,7 @@
     },
     {
       "name": "4 - WatsonX SaaS services",
-      "version_locator": "8bfb1293-8b85-4d3f-a89f-015d0a0719df.76f66c57-96b7-4ed9-a669-2ddbab72a3db",
+      "version_locator": "8bfb1293-8b85-4d3f-a89f-015d0a0719df.72efd2d9-801d-49b1-bfe3-614de8f0a3d7",
       "inputs": [
         {
           "name": "ibmcloud_api_key",
@@ -314,21 +318,21 @@
           "value": "ref:../1 - Account Infrastructure Base/outputs/devops_resource_group_name"
         },
         {
-					"name": "sm_secret_group",
-					"value": "Default"
-				},
-				{
-					"name": "sm_resource_group",
-					"value": "ref:../2b - Security Service - Secret Manager/outputs/resource_group_name"
-				},
-				{
-					"name": "sm_name",
-					"value": "ref:../2b - Security Service - Secret Manager/outputs/secrets_manager_name"
-				},
-				{
-					"name": "sm_location",
-					"value": "ref:../2b - Security Service - Secret Manager/inputs/region"
-				},
+          "name": "sm_secret_group",
+          "value": "Default"
+        },
+        {
+          "name": "sm_resource_group",
+          "value": "ref:../2b - Security Service - Secret Manager/outputs/resource_group_name"
+        },
+        {
+          "name": "sm_name",
+          "value": "ref:../2b - Security Service - Secret Manager/outputs/secrets_manager_name"
+        },
+        {
+          "name": "sm_location",
+          "value": "ref:../2b - Security Service - Secret Manager/outputs/secrets_manager_region"
+        },
         {
           "name": "ci_code_engine_project",
           "value": "Generative_AI_Sample_App_CI_Project"