From 844231e0c741e5769225cb2c819027f120160558 Mon Sep 17 00:00:00 2001 From: Nupur Goyal Date: Wed, 17 Sep 2025 15:45:18 +0530 Subject: [PATCH 01/10] fixing detect-secrets --- .secrets.baseline | 104 +++++++- tools/access-management/README.md | 11 + tools/access-management/permissions.sh | 222 ++++++++++++++++++ tools/minimal-demo-prod-scripts/README.md | 96 ++++++++ .../catalog_values_demo_deployment.json | 69 ++++++ .../catalog_values_minimal_deployment.json | 69 ++++++ .../catalog_values_production_deployment.json | 69 ++++++ tools/minimal-demo-prod-scripts/cp.sh | 48 ++++ .../create_lsf_environment.sh | 102 ++++++++ tools/minimal-demo-prod-scripts/destroy.sh | 48 ++++ tools/minimal-demo-prod-scripts/jump.sh | 39 +++ tools/minimal-demo-prod-scripts/show.sh | 32 +++ tools/minimal-demo-prod-scripts/submit.sh | 14 ++ 13 files changed, 918 insertions(+), 5 deletions(-) create mode 100644 tools/access-management/README.md create mode 100755 tools/access-management/permissions.sh create mode 100644 tools/minimal-demo-prod-scripts/README.md create mode 100644 tools/minimal-demo-prod-scripts/catalog_values_demo_deployment.json create mode 100644 tools/minimal-demo-prod-scripts/catalog_values_minimal_deployment.json create mode 100644 tools/minimal-demo-prod-scripts/catalog_values_production_deployment.json create mode 100755 tools/minimal-demo-prod-scripts/cp.sh create mode 100755 tools/minimal-demo-prod-scripts/create_lsf_environment.sh create mode 100755 tools/minimal-demo-prod-scripts/destroy.sh create mode 100755 tools/minimal-demo-prod-scripts/jump.sh create mode 100755 tools/minimal-demo-prod-scripts/show.sh create mode 100755 tools/minimal-demo-prod-scripts/submit.sh diff --git a/.secrets.baseline b/.secrets.baseline index e72e0cdb..6f286eb8 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2025-06-19T07:38:57Z", + "generated_at": "2025-09-17T10:13:45Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -77,9 +77,103 @@ } ], "results": { - "samples/configs/hpc_catalog_values.json": [ + "tools/minimal-demo-prod-scripts/README.md": [ { - "hashed_secret": "5073c7ac17500ef0678aebc7138a996b4f75d623", + "hashed_secret": "89a6cfe2a229151e8055abee107d45ed087bbb4f", + "is_secret": true, + "is_verified": false, + "line_number": 36, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "365b78d42089abe4583503eded60fa7c1b3e1cd0", + "is_secret": true, + "is_verified": false, + "line_number": 56, + "type": "Secret Keyword", + "verified_result": null + } + ], + "tools/minimal-demo-prod-scripts/catalog_values_demo_deployment.json": [ + { + "hashed_secret": "216168000275f83a7bc3599e708c5bafab959783", + "is_secret": true, + "is_verified": false, + "line_number": 2, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "55aac310878e11ce14b29027f77e58cb7c4fe7a4", + "is_secret": true, + "is_verified": false, + "line_number": 8, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", + "is_secret": true, + "is_verified": false, + "line_number": 56, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "b295b04949a98dc50ba65adcddd588077b93ab3c", + "is_secret": true, + "is_verified": false, + "line_number": 65, + "type": "Secret Keyword", + "verified_result": null + } + ], + "tools/minimal-demo-prod-scripts/catalog_values_minimal_deployment.json": [ + { + "hashed_secret": "216168000275f83a7bc3599e708c5bafab959783", + "is_secret": true, + "is_verified": false, + "line_number": 2, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "55aac310878e11ce14b29027f77e58cb7c4fe7a4", + "is_secret": true, + "is_verified": false, + "line_number": 8, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", + "is_secret": true, + "is_verified": false, + "line_number": 56, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "b295b04949a98dc50ba65adcddd588077b93ab3c", + "is_secret": true, + "is_verified": false, + "line_number": 65, + "type": "Secret Keyword", + "verified_result": null + } + ], + "tools/minimal-demo-prod-scripts/catalog_values_production_deployment.json": [ + { + "hashed_secret": "216168000275f83a7bc3599e708c5bafab959783", + "is_secret": true, + "is_verified": false, + "line_number": 2, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "55aac310878e11ce14b29027f77e58cb7c4fe7a4", "is_secret": true, "is_verified": false, "line_number": 8, @@ -90,7 +184,7 @@ "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", "is_secret": true, "is_verified": false, - "line_number": 37, + "line_number": 56, "type": "Secret Keyword", "verified_result": null }, @@ -98,7 +192,7 @@ "hashed_secret": "b295b04949a98dc50ba65adcddd588077b93ab3c", "is_secret": true, "is_verified": false, - "line_number": 60, + "line_number": 65, "type": "Secret Keyword", "verified_result": null } diff --git a/tools/access-management/README.md b/tools/access-management/README.md new file mode 100644 index 00000000..47fde9bd --- /dev/null +++ b/tools/access-management/README.md @@ -0,0 +1,11 @@ +# IAM Permissions Assignment for LSF Deployment + +### Before deploying an IBM Cloud LSF cluster, specific IAM permissions must be assigned to either a user or an access group. This script automates that process. + +How to run: + +``` +chmod +x permissions.sh + +./permissions.sh +``` diff --git a/tools/access-management/permissions.sh b/tools/access-management/permissions.sh new file mode 100755 index 00000000..35af8fef --- /dev/null +++ b/tools/access-management/permissions.sh @@ -0,0 +1,222 @@ +#!/usr/bin/env sh +set -e + +##################################### +# 1. Prompt for required inputs +##################################### +echo "πŸ”§ IBM Cloud Permissions Assignment Script (Interactive Mode)" + +read -rp "Enter admin email (your IBMid): " ADMIN_EMAIL +if [ -z "$ADMIN_EMAIL" ]; then + echo "❌ ADMIN_EMAIL is required." + exit 1 +fi + +read -rp "Enter Resource Group ID: " RESOURCE_GROUP_ID +if [ -z "$RESOURCE_GROUP_ID" ]; then + echo "❌ RESOURCE_GROUP_ID is required." + exit 1 +fi + +read -rp "Enter Account ID: " ACCOUNT_ID +if [ -z "$ACCOUNT_ID" ]; then + echo "❌ ACCOUNT_ID is required." + exit 1 +fi + +echo "Do you want to assign roles to an Access Group or a User?" +select target_type in "Access Group" "User"; do + case $target_type in + "Access Group") + read -rp "Enter Access Group Name: " ACCESS_GROUP + break + ;; + "User") + read -rp "Enter target User Email: " USER_EMAIL + break + ;; + *) + echo "❗ Invalid selection. Choose 1 or 2." + ;; + esac +done + +##################################### +# 2. Check IAM Administrator rights +##################################### +echo "πŸ” Checking if $ADMIN_EMAIL can assign IAM permissions..." +has_permission=false + +check_policies() { + echo "$1" | jq -e ' + .[]? | + select(.roles[]?.display_name == "Administrator") | + select(.resources[]?.attributes[]? | select(.name == "serviceName" and .value == "iam-identity")) | + select(.resources[]?.attributes[]?.name == "accountId") | + select(all(.resources[]?.attributes[]?.name; . != "resourceGroupId")) + ' >/dev/null +} + +USER_POLICIES=$(ibmcloud iam user-policies "$ADMIN_EMAIL" --output json 2>/dev/null || echo "[]") +if echo "$USER_POLICIES" | jq empty 2>/dev/null; then + if check_policies "$USER_POLICIES"; then + has_permission=true + fi +fi + +ACCESS_GROUPS_FOR_ADMIN=$(ibmcloud iam access-groups --ibm-id "$ADMIN_EMAIL" --output json 2>/dev/null || echo "[]") +if echo "$ACCESS_GROUPS_FOR_ADMIN" | jq empty 2>/dev/null; then + for GROUP_ID in $(echo "$ACCESS_GROUPS_FOR_ADMIN" | jq -r '.[].id // empty'); do + GROUP_POLICIES=$(ibmcloud iam access-group-policies "$GROUP_ID" --output json 2>/dev/null || echo "[]") + if echo "$GROUP_POLICIES" | jq empty 2>/dev/null; then + if check_policies "$GROUP_POLICIES"; then + has_permission=true + break + fi + fi + done +fi + +if [ "$has_permission" != true ]; then + echo "❌ $ADMIN_EMAIL does NOT have account-level iam-identity Administrator rights β€” cannot assign permissions." + exit 1 +fi + +echo "βœ… $ADMIN_EMAIL has account-level iam-identity Administrator rights β€” proceeding." + +##################################### +# 3. Role assignment definitions +##################################### +PERMISSIONS_LIST="apprapp|Administrator|Manager +cloud-object-storage|Service Configuration Reader|Writer +dns-svcs|Editor|Manager +sysdig-monitor|Administrator|Manager +kms|Service Configuration Reader|Manager +secrets-manager|Administrator|Manager +sysdig-secure|Administrator| +iam-identity|Administrator| +is|Editor|" + +# New friendly names list (service|friendly name) +FRIENDLY_NAMES="apprapp|App Configuration +cloud-object-storage|Cloud Object Storage +dns-svcs|DNS Services +sysdig-monitor|Cloud Monitoring +kms|Key Protect +secrets-manager|Secrets Manager +sysdig-secure|Security and Compliance Center Workload Protection +iam-identity|IAM Identity Service +is|VPC Infrastructure Services" + +get_friendly_name() { + local service="$1" + echo "$FRIENDLY_NAMES" | while IFS='|' read -r svc fname; do + if [ "$svc" = "$service" ]; then + echo "$fname" + return + fi + done +} + +##################################### +# 4. Helper to check if policy exists +##################################### +policy_exists() { + local SERVICE="$1" + local ROLES="$2" + local RG_ID="$3" + local ACCOUNT_ID="$4" + + local existing_policies + if [ -n "$ACCESS_GROUP" ]; then + existing_policies=$(ibmcloud iam access-group-policies "$ACCESS_GROUP" --output json 2>/dev/null || echo "[]") + elif [ -n "$USER_EMAIL" ]; then + existing_policies=$(ibmcloud iam user-policies "$USER_EMAIL" --output json 2>/dev/null || echo "[]") + else + echo "❗ ERROR: Neither ACCESS_GROUP nor USER_EMAIL is set in policy_exists" + return 1 + fi + + echo "$existing_policies" | jq -e \ + --arg service "$SERVICE" \ + --arg roles "$ROLES" \ + --arg rg_id "$RG_ID" \ + --arg account_id "$ACCOUNT_ID" ' + .[] | + select(([.roles[].display_name] | sort) == ($roles | split(",") | sort)) | + if $service == "" then + select(any(.resources[].attributes[]?; + .name == "resourceGroupId" and .value == $rg_id)) | + select(all(.resources[].attributes[]?.name; . != "serviceName")) + else + select(any(.resources[].attributes[]?; + .name == "resourceGroupId" and .value == $rg_id)) | + select(any(.resources[].attributes[]?; + .name == "serviceName" and .value == $service)) | + select([.resources[].attributes[]?.name] | unique | sort + == ["accountId","resourceGroupId","serviceName"]) + end + ' >/dev/null +} + +##################################### +# 5. Main logic: Assign roles +##################################### +if [ -n "$ACCESS_GROUP" ] && [ -z "$USER_EMAIL" ]; then + echo "πŸ” Assigning roles to access group: $ACCESS_GROUP" + echo "$PERMISSIONS_LIST" | while IFS='|' read -r SERVICE_NAME PLATFORM_ROLE SERVICE_ROLE; do + [ -n "$SERVICE_ROLE" ] && ROLES="$PLATFORM_ROLE,$SERVICE_ROLE" || ROLES="$PLATFORM_ROLE" + fname=$(get_friendly_name "$SERVICE_NAME") + [ -n "$fname" ] && DISPLAY_NAME="$SERVICE_NAME ($fname)" || DISPLAY_NAME="$SERVICE_NAME" + + if ! policy_exists "$SERVICE_NAME" "$ROLES" "$RESOURCE_GROUP_ID" "$ACCOUNT_ID"; then + echo "Assigning roles '$ROLES' for service $DISPLAY_NAME" + ibmcloud iam access-group-policy-create "$ACCESS_GROUP" \ + --roles "$ROLES" \ + --service-name "$SERVICE_NAME" \ + --resource-group-id "$RESOURCE_GROUP_ID" || echo "⚠️ Failed to assign $ROLES for $DISPLAY_NAME" + else + echo "βœ… Policy already exists for $DISPLAY_NAME" + fi + done + + if ! policy_exists "" "Administrator,Manager" "$RESOURCE_GROUP_ID" "$ACCOUNT_ID"; then + echo "Assigning global Administrator,Manager roles to access group: $ACCESS_GROUP" + ibmcloud iam access-group-policy-create "$ACCESS_GROUP" \ + --roles "Administrator,Manager" \ + --resource-group-id "$RESOURCE_GROUP_ID" || echo "⚠️ Failed for all-service Admin/Manager (access group)" + else + echo "βœ… All Identity and Access enabled services Administrator/Manager policy already exists for access group" + fi + +elif [ -z "$ACCESS_GROUP" ] && [ -n "$USER_EMAIL" ]; then + echo "πŸ‘€ Assigning roles to user: $USER_EMAIL" + echo "$PERMISSIONS_LIST" | while IFS='|' read -r SERVICE_NAME PLATFORM_ROLE SERVICE_ROLE; do + [ -n "$SERVICE_ROLE" ] && ROLES="$PLATFORM_ROLE,$SERVICE_ROLE" || ROLES="$PLATFORM_ROLE" + fname=$(get_friendly_name "$SERVICE_NAME") + [ -n "$fname" ] && DISPLAY_NAME="$SERVICE_NAME ($fname)" || DISPLAY_NAME="$SERVICE_NAME" + + if ! policy_exists "$SERVICE_NAME" "$ROLES" "$RESOURCE_GROUP_ID" "$ACCOUNT_ID"; then + echo "Assigning roles '$ROLES' for service $DISPLAY_NAME" + ibmcloud iam user-policy-create "$USER_EMAIL" \ + --roles "$ROLES" \ + --service-name "$SERVICE_NAME" \ + --resource-group-id "$RESOURCE_GROUP_ID" || echo "⚠️ Failed to assign $ROLES for $DISPLAY_NAME" + else + echo "βœ… Policy already exists for $DISPLAY_NAME" + fi + done + + if ! policy_exists "" "Administrator,Manager" "$RESOURCE_GROUP_ID" "$ACCOUNT_ID"; then + echo "Assigning global Administrator,Manager roles to $USER_EMAIL" + ibmcloud iam user-policy-create "$USER_EMAIL" \ + --roles "Administrator,Manager" \ + --resource-group-id "$RESOURCE_GROUP_ID" || echo "⚠️ Failed for all-service Admin/Manager" + else + echo "βœ… All Identity and Access enabled services Administrator/Manager policy already exists" + fi + +else + echo "❗ Please choose either Access Group or User." + exit 1 +fi diff --git a/tools/minimal-demo-prod-scripts/README.md b/tools/minimal-demo-prod-scripts/README.md new file mode 100644 index 00000000..dd8ba4bc --- /dev/null +++ b/tools/minimal-demo-prod-scripts/README.md @@ -0,0 +1,96 @@ +# Deploying and Connecting to LSF Environment via CLI + +### Notes: + +The must be 16 characters or fewer, i.e. abc-lsf + +The catalog_values__deployment.json specifies the configuation of the LSF environment. Please review to avoid unexpected costs. + +### Deployment Types: + +#### Minimal: +Deploys the smallest possible environment (a single management instance) for the fastest setup. All optional services (observability, logging, SCC, Atracker, Ldap etc.) are disabled. + +#### Demo: +Showcases the full set of capabilities. All optional services (observability, logging, SCC, etc.) are enabled. Deployment takes longer compared to minimal. + +#### Production: +Allows customization for production-grade deployments. Optional services like observability, logging, and SCC are enabled by default but can be tailored as required. + +All JSON files are customizable (users can tweak configs as needed). +But the .env file is mandatory because that’s where the required variables must always be filled. + +## Step 1. Fill the .env file + +``` +############################################################################## +# Environment Configuration + +# Step 1: Update the variables below as needed. +# Step 2: If you require additional optional variables, update them directly +# in the JSON file(s) for your deployment type. +# Step 3: Always validate the JSON file before running the script. +############################################################################## + +# IBM Cloud API key +API_KEY="YOUR_API_KEY" + +# Account and resource details +ACCOUNT_GUID="ACCOUNT_GUID" +ZONES="ZONES" +RESOURCE_GROUP="RESOURCE_GROUP" + +# SSH key name (must exist in your account) +SSH_KEY="SSH_KEY" + +# Template JSON file (choose as per your deployment type) +TEMPLATE_FILE="catalog_values_minimal_deployment.json" + +# LSF tile version locator +# Example below is for 3.0.0 version +LSF_TILE_VERSION="1082e7d2-5e2f-0a11-a3bc-f88a8e1931fc.6c26cd4c-4f72-45e5-8bde-77387aa05138-global" + +# App Center GUI password +# Rules: Minimum 8 characters, at least 1 uppercase, 1 lowercase, 1 number, +# and 1 special character (!@#$%^&*()_+=-). No spaces allowed. +APP_CENTER_GUI_PASSWORD="APP_CENTER_GUI_PASSWORD" +``` + +## Step 2. Deploy the LSF Environment: +``` +1. chmod +x create_lsf_environment.sh +2. ./create_lsf_environment.sh +``` + +## Step 3. Connect to the LSF Cluster and Run Jobs + +Now that your environment is set up, you can connect to the LSF cluster and perform operations such as submitting jobs, monitoring workloads, viewing infrastructure details. etc. + +#### 1. To view the infra details + +``` +chmod +x show.sh + ./show.sh +``` + +#### 2. Copy the job submission script to the cluster + +``` +chmod +x cp.sh + ./cp.sh submit.sh +``` + +#### 3. Jump to the LSF Environment + +``` +chmod +x jump.sh + ./jump.sh +``` + +#### 4. Submit jobs + +``` +sh submit.sh +bjobs +lshosts -w +``` diff --git a/tools/minimal-demo-prod-scripts/catalog_values_demo_deployment.json b/tools/minimal-demo-prod-scripts/catalog_values_demo_deployment.json new file mode 100644 index 00000000..3e4b629e --- /dev/null +++ b/tools/minimal-demo-prod-scripts/catalog_values_demo_deployment.json @@ -0,0 +1,69 @@ +{ + "ibmcloud_api_key": "XX_API_KEY_XX", + "existing_resource_group": "XX_RESOURCE_GROUP_XX", + "zones": "[\"XX_ZONES_XX\"]", + "ssh_keys": "[\"XX_SSH_KEY_XX\"]", + "cluster_prefix": "XX_PREFIX_XX", + "remote_allowed_ips": "[\"XX_REMOTE_IP_XX\"]", + "app_center_gui_password": "XX_APP_CENTER_GUI_PASSWORD_XX", + + "enable_hyperthreading": true, + "vpn_enabled": false, + "TF_VERSION": "1.9", + "TF_PARALLELISM": "250", + "key_management": "key_protect", + "kms_instance_name": "__NULL__", + "kms_key_name": "__NULL__", + "enable_vpc_flow_logs": true, + "sccwp_enable": false, + "app_config_plan": "basic", + "sccwp_service_plan": "graduated-tier", + "cspm_enabled": "true", + "observability_atracker_enable": "true", + "observability_atracker_target_type": "cloudlogs", + "observability_monitoring_enable": "true", + "observability_logs_enable_for_management": "true", + "observability_logs_enable_for_compute": "true", + "observability_enable_platform_logs": "true", + "observability_enable_metrics_routing": "true", + "observability_logs_retention_period": "7", + "observability_monitoring_on_compute_nodes_enable": "true", + "observability_monitoring_plan": "graduated-tier", + "lsf_version": "fixpack_15", + "vpc_name": "__NULL__", + "cluster_subnet_id": "__NULL__", + "login_subnet_id": "__NULL__", + "vpc_cidr": "10.241.0.0/18", + "vpc_cluster_private_subnets_cidr_blocks": "10.241.0.0/20", + "vpc_cluster_login_private_subnets_cidr_blocks": "10.241.16.0/28", + "dns_domain_name": "{compute = \"lsf.com\"}", + "dns_instance_id": "__NULL__", + "dns_custom_resolver_id": "__NULL__", + "bastion_instance": "{ image = \"ibm-ubuntu-22-04-5-minimal-amd64-3\", profile = \"cx2-4x8\" }", + "deployer_instance": "{ image = \"hpc-lsf-fp15-deployer-rhel810-v1\", profile = \"bx2-8x32\" }", + "login_instance": "[{ profile = \"bx2-2x8\", image = \"hpc-lsf-fp15-compute-rhel810-v1\" }]", + "management_instances": "[{ profile = \"bx2-16x64\", count = 2, image = \"hpc-lsf-fp15-rhel810-v1\" }]", + "static_compute_instances": "[{ profile = \"bx2-4x16\", count = 0, image = \"hpc-lsf-fp15-compute-rhel810-v1\" }]", + "dynamic_compute_instances": "[{ profile = \"bx2-4x16\", count = 500, image = \"hpc-lsf-fp15-compute-rhel810-v1\" }]", + "custom_file_shares": "[{mount_path = \"/mnt/vpcstorage/tools\", size = 100, iops = 2000 }, { mount_path = \"/mnt/vpcstorage/data\", size = 100, iops = 6000 }, { mount_path = \"/mnt/scale/tools\", nfs_share = \"\" }]", + "storage_security_group_id": "__NULL__", + "enable_ldap": "false", + "ldap_basedns": "lsf.com", + "ldap_server": "__NULL__", + "ldap_server_cert": "__NULL__", + "ldap_admin_password": "", + "ldap_user_name": "", + "ldap_user_password": "", + "ldap_instance": "[{ profile = \"cx2-2x4\", image = \"ibm-ubuntu-22-04-5-minimal-amd64-1\" }]", + "skip_iam_share_authorization_policy": "false", + "skip_flowlogs_s2s_auth_policy": "false", + "skip_kms_s2s_auth_policy": "false", + "skip_iam_block_storage_authorization_policy": "false", + "existing_bastion_instance_name": "__NULL__", + "existing_bastion_instance_public_ip": "__NULL__", + "existing_bastion_security_group_id": "__NULL__", + "existing_bastion_ssh_private_key": "__NULL__", + "enable_dedicated_host": "false", + "enable_cos_integration": "false", + "cos_instance_name": "__NULL__" +} diff --git a/tools/minimal-demo-prod-scripts/catalog_values_minimal_deployment.json b/tools/minimal-demo-prod-scripts/catalog_values_minimal_deployment.json new file mode 100644 index 00000000..e554b50c --- /dev/null +++ b/tools/minimal-demo-prod-scripts/catalog_values_minimal_deployment.json @@ -0,0 +1,69 @@ +{ + "ibmcloud_api_key": "XX_API_KEY_XX", + "existing_resource_group": "XX_RESOURCE_GROUP_XX", + "zones": "[\"XX_ZONES_XX\"]", + "ssh_keys": "[\"XX_SSH_KEY_XX\"]", + "cluster_prefix": "XX_PREFIX_XX", + "remote_allowed_ips": "[\"XX_REMOTE_IP_XX\"]", + "app_center_gui_password": "XX_APP_CENTER_GUI_PASSWORD_XX", + + "enable_hyperthreading": true, + "vpn_enabled": false, + "TF_VERSION": "1.9", + "TF_PARALLELISM": "250", + "key_management": "key_protect", + "kms_instance_name": "__NULL__", + "kms_key_name": "__NULL__", + "enable_vpc_flow_logs": false, + "sccwp_enable": false, + "app_config_plan": "basic", + "sccwp_service_plan": "graduated-tier", + "cspm_enabled": "true", + "observability_atracker_enable": "false", + "observability_atracker_target_type": "cloudlogs", + "observability_monitoring_enable": "false", + "observability_logs_enable_for_management": "false", + "observability_logs_enable_for_compute": "false", + "observability_enable_platform_logs": "false", + "observability_enable_metrics_routing": "false", + "observability_logs_retention_period": "7", + "observability_monitoring_on_compute_nodes_enable": "false", + "observability_monitoring_plan": "graduated-tier", + "lsf_version": "fixpack_15", + "vpc_name": "__NULL__", + "cluster_subnet_id": "__NULL__", + "login_subnet_id": "__NULL__", + "vpc_cidr": "10.241.0.0/18", + "vpc_cluster_private_subnets_cidr_blocks": "10.241.0.0/20", + "vpc_cluster_login_private_subnets_cidr_blocks": "10.241.16.0/28", + "dns_domain_name": "{compute = \"lsf.com\"}", + "dns_instance_id": "__NULL__", + "dns_custom_resolver_id": "__NULL__", + "bastion_instance": "{ image = \"ibm-ubuntu-22-04-5-minimal-amd64-3\", profile = \"cx2-4x8\" }", + "deployer_instance": "{ image = \"hpc-lsf-fp15-deployer-rhel810-v1\", profile = \"bx2-8x32\" }", + "login_instance": "[{ profile = \"bx2-2x8\", image = \"hpc-lsf-fp15-compute-rhel810-v1\" }]", + "management_instances": "[{ profile = \"bx2-4x16\", count = 1, image = \"hpc-lsf-fp15-rhel810-v1\" }]", + "static_compute_instances": "[{ profile = \"bx2-4x16\", count = 0, image = \"hpc-lsf-fp15-compute-rhel810-v1\" }]", + "dynamic_compute_instances": "[{ profile = \"bx2-4x16\", count = 5, image = \"hpc-lsf-fp15-compute-rhel810-v1\" }]", + "custom_file_shares": "[{mount_path = \"/mnt/vpcstorage/tools\", size = 10, iops = 1000 }, { mount_path = \"/mnt/vpcstorage/data\", size = 10, iops = 1000 }, { mount_path = \"/mnt/scale/tools\", nfs_share = \"\" }]", + "storage_security_group_id": "__NULL__", + "enable_ldap": "false", + "ldap_basedns": "lsf.com", + "ldap_server": "__NULL__", + "ldap_server_cert": "__NULL__", + "ldap_admin_password": "", + "ldap_user_name": "", + "ldap_user_password": "", + "ldap_instance": "[{ profile = \"cx2-2x4\", image = \"ibm-ubuntu-22-04-5-minimal-amd64-1\" }]", + "skip_iam_share_authorization_policy": "false", + "skip_flowlogs_s2s_auth_policy": "false", + "skip_kms_s2s_auth_policy": "false", + "skip_iam_block_storage_authorization_policy": "false", + "existing_bastion_instance_name": "__NULL__", + "existing_bastion_instance_public_ip": "__NULL__", + "existing_bastion_security_group_id": "__NULL__", + "existing_bastion_ssh_private_key": "__NULL__", + "enable_dedicated_host": "false", + "enable_cos_integration": "false", + "cos_instance_name": "__NULL__" +} diff --git a/tools/minimal-demo-prod-scripts/catalog_values_production_deployment.json b/tools/minimal-demo-prod-scripts/catalog_values_production_deployment.json new file mode 100644 index 00000000..7e74fb23 --- /dev/null +++ b/tools/minimal-demo-prod-scripts/catalog_values_production_deployment.json @@ -0,0 +1,69 @@ +{ + "ibmcloud_api_key": "XX_API_KEY_XX", + "existing_resource_group": "XX_RESOURCE_GROUP_XX", + "zones": "[\"XX_ZONES_XX\"]", + "ssh_keys": "[\"XX_SSH_KEY_XX\"]", + "cluster_prefix": "XX_PREFIX_XX", + "remote_allowed_ips": "[\"XX_REMOTE_IP_XX\"]", + "app_center_gui_password": "XX_APP_CENTER_GUI_PASSWORD_XX", + + "enable_hyperthreading": true, + "vpn_enabled": false, + "TF_VERSION": "1.9", + "TF_PARALLELISM": "250", + "key_management": "key_protect", + "kms_instance_name": "__NULL__", + "kms_key_name": "__NULL__", + "enable_vpc_flow_logs": true, + "sccwp_enable": true, + "app_config_plan": "standardv2", + "sccwp_service_plan": "graduated-tier", + "cspm_enabled": "true", + "observability_atracker_enable": "true", + "observability_atracker_target_type": "cloudlogs", + "observability_monitoring_enable": "true", + "observability_logs_enable_for_management": "true", + "observability_logs_enable_for_compute": "true", + "observability_enable_platform_logs": "true", + "observability_enable_metrics_routing": "true", + "observability_logs_retention_period": "7", + "observability_monitoring_on_compute_nodes_enable": "true", + "observability_monitoring_plan": "graduated-tier", + "lsf_version": "fixpack_15", + "vpc_name": "__NULL__", + "cluster_subnet_id": "__NULL__", + "login_subnet_id": "__NULL__", + "vpc_cidr": "10.241.0.0/18", + "vpc_cluster_private_subnets_cidr_blocks": "10.241.0.0/20", + "vpc_cluster_login_private_subnets_cidr_blocks": "10.241.16.0/28", + "dns_domain_name": "{compute = \"lsf.com\"}", + "dns_instance_id": "__NULL__", + "dns_custom_resolver_id": "__NULL__", + "bastion_instance": "{ image = \"ibm-ubuntu-22-04-5-minimal-amd64-3\", profile = \"cx2-4x8\" }", + "deployer_instance": "{ image = \"hpc-lsf-fp15-deployer-rhel810-v1\", profile = \"bx2-8x32\" }", + "login_instance": "[{ profile = \"bx2-2x8\", image = \"hpc-lsf-fp15-compute-rhel810-v1\" }]", + "management_instances": "[{ profile = \"bx2-16x64\", count = 2, image = \"hpc-lsf-fp15-rhel810-v1\" }]", + "static_compute_instances": "[{ profile = \"bx2-4x16\", count = 5, image = \"hpc-lsf-fp15-compute-rhel810-v1\" }]", + "dynamic_compute_instances": "[{ profile = \"bx2-4x16\", count = 500, image = \"hpc-lsf-fp15-compute-rhel810-v1\" }]", + "custom_file_shares": "[{mount_path = \"/mnt/vpcstorage/tools\", size = 100, iops = 2000 }, { mount_path = \"/mnt/vpcstorage/data\", size = 100, iops = 6000 }, { mount_path = \"/mnt/scale/tools\", nfs_share = \"\" }]", + "storage_security_group_id": "__NULL__", + "enable_ldap": "false", + "ldap_basedns": "lsf.com", + "ldap_server": "__NULL__", + "ldap_server_cert": "__NULL__", + "ldap_admin_password": "", + "ldap_user_name": "", + "ldap_user_password": "", + "ldap_instance": "[{ profile = \"cx2-2x4\", image = \"ibm-ubuntu-22-04-5-minimal-amd64-1\" }]", + "skip_iam_share_authorization_policy": "false", + "skip_flowlogs_s2s_auth_policy": "false", + "skip_kms_s2s_auth_policy": "false", + "skip_iam_block_storage_authorization_policy": "false", + "existing_bastion_instance_name": "__NULL__", + "existing_bastion_instance_public_ip": "__NULL__", + "existing_bastion_security_group_id": "__NULL__", + "existing_bastion_ssh_private_key": "__NULL__", + "enable_dedicated_host": "false", + "enable_cos_integration": "true", + "cos_instance_name": "__NULL__" +} diff --git a/tools/minimal-demo-prod-scripts/cp.sh b/tools/minimal-demo-prod-scripts/cp.sh new file mode 100755 index 00000000..014ef4b1 --- /dev/null +++ b/tools/minimal-demo-prod-scripts/cp.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +source .env + +REGION=$(echo "$ZONES" | cut -d'-' -f1-2) + +if [ $# -eq 0 ] + then + echo "Pls provide cluster_prefix, i.e. cp.sh " + exit 1 +fi + +CURRENT_ACCOUNT_GUID=$(ibmcloud target --output json | jq -r '.account.guid') +if [ "$CURRENT_ACCOUNT_GUID" != "$ACCOUNT_GUID" ] + then + ibmcloud login -a cloud.ibm.com --apikey $API_KEY -r $REGION -g $RESOURCE_GROUP +fi + +CURRENT_ACCOUNT_NAME=$(ibmcloud target --output json | jq -r '.account.name') +echo "target account $CURRENT_ACCOUNT_GUID" + +CURRENT_REGION=$(ibmcloud target --output json | jq -r '.region.name') +if [ "$CURRENT_REGION" != "$REGION" ] + then + ibmcloud target -r $REGION +fi +echo "target region $REGION" + +# ibmcloud is ip $1-bastion-001-fip --output json | jq -r '. | .address' +BASTION_IP=$(ibmcloud is ips | grep $1-bastion | grep 001-fip | awk '{print $2}') +echo "Bastion IP: $BASTION_IP" + +# ibmcloud is instance $1-login-001 --output json | jq -r '. | .network_interfaces[0].primary_ip.address' +LOGIN_IP=$(ibmcloud is instances | grep $1-login | grep 001 | awk '{print $4}') +echo "Login IP: $LOGIN_IP" + +# ibmcloud is instance $1-mgmt-1-001 --output json | jq -r '. | .network_interfaces[0].primary_ip.address' +LSF_IP=$(ibmcloud is instances | grep $1-mgmt-1 | grep 001 | awk '{print $4}') +echo "LSF IP: $LSF_IP" + +if [ "$3" == "mgmt" ] + then + echo "copying $2 to LSF /mnt/vpcstorage/tools..." + scp -o UserKnownHostsFile=/dev/null -o StrictHostKeychecking=no -o IdentitiesOnly=yes -J ubuntu@$BASTION_IP $2 lsfadmin@$LSF_IP:/mnt/vpcstorage/tools/$(basename $2) +else + echo "copying $2 to LOGIN /home/lsfadmin..." + scp -o UserKnownHostsFile=/dev/null -o StrictHostKeychecking=no -o IdentitiesOnly=yes -J ubuntu@$BASTION_IP $2 lsfadmin@$LOGIN_IP:/home/lsfadmin/$(basename $2) +fi diff --git a/tools/minimal-demo-prod-scripts/create_lsf_environment.sh b/tools/minimal-demo-prod-scripts/create_lsf_environment.sh new file mode 100755 index 00000000..3af0fbc4 --- /dev/null +++ b/tools/minimal-demo-prod-scripts/create_lsf_environment.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash + +# Get installed plugins once +INSTALLED_PLUGINS=$(ibmcloud plugin list | awk 'NR>3 {print $1}') + +ensure_plugin() { + local plugin="$1" + if echo "$INSTALLED_PLUGINS" | grep -qw "$plugin"; then + echo "IBM Cloud $plugin plugin already installed." + else + echo "Installing IBM Cloud $plugin plugin..." + ibmcloud plugin install "$plugin" -f + fi +} + +# Ensure required plugins +ensure_plugin "catalogs-management" +ensure_plugin "schematics" +ensure_plugin "vpc-infrastructure" + +source .env + +REGION=$(echo "$ZONES" | cut -d'-' -f1-2) + +if [ $# -eq 0 ]; then + echo "Pls provide cluster_prefix, i.e. create_lsf_environment.sh " + exit 1 +fi + +REMOTE_IP=$(curl -s https://ipv4.icanhazip.com/) + +sed s/XX_PREFIX_XX/$1/g $TEMPLATE_FILE | \ +sed s/XX_API_KEY_XX/$API_KEY/g | \ +sed s/XX_RESOURCE_GROUP_XX/$RESOURCE_GROUP/g | \ +sed s/XX_SSH_KEY_XX/$SSH_KEY/g | \ +sed s/XX_ZONES_XX/$ZONES/g | \ +sed s/XX_REMOTE_IP_XX/$REMOTE_IP/g | \ +sed s/XX_APP_CENTER_GUI_PASSWORD_XX/$APP_CENTER_GUI_PASSWORD/g > environment_values_$1.json + +ibmcloud login -a cloud.ibm.com --apikey $API_KEY -r $REGION -g $RESOURCE_GROUP +ibmcloud target -r $REGION + +# Run install and capture output +INSTALL_LOG=$(mktemp) +if ! ibmcloud catalog install --timeout 3600 \ + --vl $LSF_TILE_VERSION \ + --override-values environment_values_$1.json \ + --workspace-region $REGION \ + --workspace-rg-id $RESOURCE_GROUP \ + --workspace-name $1 \ + --workspace-tf-version 1.9 | tee "$INSTALL_LOG"; then + echo "Install command failed to start" + exit 1 +fi + +# Extract WORKSPACE_ID +WORKSPACE_ID=$(ibmcloud schematics workspace list --output json \ + | jq -r '.workspaces[] | select(.name=="'"$1"'") | .id') + +# Check if FAILED appears in install output +if grep -q "FAILED" "$INSTALL_LOG"; then + echo "❌ Installation failed for workspace: $WORKSPACE_ID" + echo "Last 100 lines of logs for quick reference:" + ibmcloud schematics logs --id "$WORKSPACE_ID" | tail -n 100 + echo "For full logs, check schematics workspace at: https://cloud.ibm.com/schematics/workspaces/$WORKSPACE_ID/jobs?region=$REGION" + exit 1 +fi + +# If success, print the Output IPs +BASTION_IP=$(ibmcloud is ips | grep $1-bastion | grep 001-fip | awk '{print $2}') +echo "Bastion IP: $BASTION_IP" + +LOGIN_IP=$(ibmcloud is instances | grep $1-login | grep 001 | awk '{print $4}') +echo "Login IP: $LOGIN_IP" + +LSF_IP=$(ibmcloud is instances | grep $1-mgmt-1 | grep 001 | awk '{print $4}') +echo "LSF IP: $LSF_IP" + +echo "" +echo "βœ… LSF environment setup completed!" +echo "" +echo "Next Steps:" +echo "----------------------------------------" +echo "Copy the job submission script to the cluster:" +echo " ./cp.sh $1 submit.sh" +echo "" +echo "To get the details of workspace Environment:" +echo " ./show.sh $1" +echo "" +echo "Jump to the LSF Environment:" +echo " ./jump.sh $1" +echo "" +echo "Submit jobs:" +echo " ./submit.sh" +echo " bjobs ..." +echo " bhosts ..." +echo "----------------------------------------" +echo "" +echo "Alternatively, you can run web_service.sh to configure LSF Web Services for local client access (Standalone LSF Client):" +echo " ./web_service.sh $1" +echo "" +echo "⚑ This will allow users to connect to an LSF cluster from their local environment." diff --git a/tools/minimal-demo-prod-scripts/destroy.sh b/tools/minimal-demo-prod-scripts/destroy.sh new file mode 100755 index 00000000..a6634a55 --- /dev/null +++ b/tools/minimal-demo-prod-scripts/destroy.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +source .env + +REGION=$(echo "$ZONES" | cut -d'-' -f1-2) + +if [ $# -eq 0 ] + then + echo "Pls provide cluster_prefix, i.e. destroy.sh " + exit 1 +fi + +CURRENT_ACCOUNT_GUID=$(ibmcloud target --output json | jq -r '.account.guid') +if [ "$CURRENT_ACCOUNT_GUID" != "$ACCOUNT_GUID" ] + then + ibmcloud login -a cloud.ibm.com --apikey $API_KEY -r $REGION -g $RESOURCE_GROUP +fi + +CURRENT_ACCOUNT_NAME=$(ibmcloud target --output json | jq -r '.account.name') +echo "target account $CURRENT_ACCOUNT_GUID" + +CURRENT_REGION=$(ibmcloud target --output json | jq -r '.region.name') +if [ "$CURRENT_REGION" != "$REGION" ] + then + ibmcloud target -r $REGION +fi +echo "target region $REGION" + +echo "finding schematics workspace..." +# Extract WORKSPACE_ID +WORKSPACE_ID=$(ibmcloud schematics workspace list --output json \ + | jq -r '.workspaces[] | select(.name=="'"$1"'") | .id') + +ibmcloud schematics workspace get --id "$WORKSPACE_ID" + +read -p "Do you want to destroy? (yes/no) " yn + +case $yn in + yes ) echo ok, we will proceed;; + no ) echo exiting...; + exit;; + * ) echo invalid response; + exit 1;; +esac + +rm environment_values_$1.json + +ibmcloud schematics destroy --id $WORKSPACE_ID -f diff --git a/tools/minimal-demo-prod-scripts/jump.sh b/tools/minimal-demo-prod-scripts/jump.sh new file mode 100755 index 00000000..3ca27015 --- /dev/null +++ b/tools/minimal-demo-prod-scripts/jump.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +source .env + +REGION=$(echo "$ZONES" | cut -d'-' -f1-2) + +if [ $# -eq 0 ] + then + echo "Pls provide cluster_prefix, i.e. jump.sh " + exit 1 +fi + +CURRENT_ACCOUNT_GUID=$(ibmcloud target --output json | jq -r '.account.guid') +if [ "$CURRENT_ACCOUNT_GUID" != "$ACCOUNT_GUID" ] + then + ibmcloud login -a cloud.ibm.com --apikey $API_KEY -r $REGION -g $RESOURCE_GROUP +fi + +CURRENT_ACCOUNT_NAME=$(ibmcloud target --output json | jq -r '.account.name') +echo "target account $CURRENT_ACCOUNT_GUID" + +CURRENT_REGION=$(ibmcloud target --output json | jq -r '.region.name') +if [ "$CURRENT_REGION" != "$REGION" ] + then + ibmcloud target -r $REGION +fi +echo "target region $REGION" + +BASTION_IP=$(ibmcloud is ips | grep $1-bastion | grep 001-fip | awk '{print $2}') +echo "Bastion IP: $BASTION_IP" + +LOGIN_IP=$(ibmcloud is instances | grep $1-login | grep 001 | awk '{print $4}') +echo "Login IP: $LOGIN_IP" + +LSF_IP=$(ibmcloud is instances | grep $1-mgmt-1 | grep 001 | awk '{print $4}') +echo "LSF IP: $LSF_IP" + +echo "Jumping to Login Node..." +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -J ubuntu@$BASTION_IP lsfadmin@$LOGIN_IP diff --git a/tools/minimal-demo-prod-scripts/show.sh b/tools/minimal-demo-prod-scripts/show.sh new file mode 100755 index 00000000..198d4237 --- /dev/null +++ b/tools/minimal-demo-prod-scripts/show.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +source .env + +REGION=$(echo "$ZONES" | cut -d'-' -f1-2) + +if [ $# -eq 0 ] + then + echo "Pls provide cluster_prefix, i.e. show.sh " + exit 1 +fi + +CURRENT_ACCOUNT_GUID=$(ibmcloud target --output json | jq -r '.account.guid') +if [ "$CURRENT_ACCOUNT_GUID" != "$ACCOUNT_GUID" ] + then + ibmcloud login -a cloud.ibm.com --apikey $API_KEY -r $REGION -g $RESOURCE_GROUP +fi + +CURRENT_ACCOUNT_NAME=$(ibmcloud target --output json | jq -r '.account.name') +echo "target account $CURRENT_ACCOUNT_GUID" + +CURRENT_REGION=$(ibmcloud target --output json | jq -r '.region.name') +if [ "$CURRENT_REGION" != "$REGION" ] + then + ibmcloud target -r $REGION +fi +echo "target region $REGION" + +echo "finding schematics workspace..." +WORKSPACE_ID=$(ibmcloud schematics workspace list | grep $1 | awk '{ print $2 }') + +ibmcloud schematics workspace get --id $WORKSPACE_ID diff --git a/tools/minimal-demo-prod-scripts/submit.sh b/tools/minimal-demo-prod-scripts/submit.sh new file mode 100755 index 00000000..e79a5a70 --- /dev/null +++ b/tools/minimal-demo-prod-scripts/submit.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Sample Job Submission +# --------------------- +# This example submits a simple LSF job that: +# - Requests 8 cores (-n 8) +# - Runs the command: sleep 30 +# +# You can modify the command (e.g., sleep 30) or job options (-n 8) +# to suit your own workload requirements. + +CMD="bsub -n 8 sleep 30" +echo "Running: $CMD" +$CMD From 140b1a3a045df55a9376dad605bd8fd7703f73a0 Mon Sep 17 00:00:00 2001 From: Nupur Goyal Date: Wed, 17 Sep 2025 15:51:34 +0530 Subject: [PATCH 02/10] fixing detect-secrets --- .secrets.baseline | 104 ++---------------------- samples/configs/hpc_catalog_values.json | 10 +-- tools/image-builder/variables.tf | 2 +- 3 files changed, 11 insertions(+), 105 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 6f286eb8..2d87d2c0 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2025-09-17T10:13:45Z", + "generated_at": "2025-09-17T10:20:38Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -77,103 +77,9 @@ } ], "results": { - "tools/minimal-demo-prod-scripts/README.md": [ + "samples/configs/hpc_catalog_values.json": [ { - "hashed_secret": "89a6cfe2a229151e8055abee107d45ed087bbb4f", - "is_secret": true, - "is_verified": false, - "line_number": 36, - "type": "Secret Keyword", - "verified_result": null - }, - { - "hashed_secret": "365b78d42089abe4583503eded60fa7c1b3e1cd0", - "is_secret": true, - "is_verified": false, - "line_number": 56, - "type": "Secret Keyword", - "verified_result": null - } - ], - "tools/minimal-demo-prod-scripts/catalog_values_demo_deployment.json": [ - { - "hashed_secret": "216168000275f83a7bc3599e708c5bafab959783", - "is_secret": true, - "is_verified": false, - "line_number": 2, - "type": "Secret Keyword", - "verified_result": null - }, - { - "hashed_secret": "55aac310878e11ce14b29027f77e58cb7c4fe7a4", - "is_secret": true, - "is_verified": false, - "line_number": 8, - "type": "Secret Keyword", - "verified_result": null - }, - { - "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", - "is_secret": true, - "is_verified": false, - "line_number": 56, - "type": "Secret Keyword", - "verified_result": null - }, - { - "hashed_secret": "b295b04949a98dc50ba65adcddd588077b93ab3c", - "is_secret": true, - "is_verified": false, - "line_number": 65, - "type": "Secret Keyword", - "verified_result": null - } - ], - "tools/minimal-demo-prod-scripts/catalog_values_minimal_deployment.json": [ - { - "hashed_secret": "216168000275f83a7bc3599e708c5bafab959783", - "is_secret": true, - "is_verified": false, - "line_number": 2, - "type": "Secret Keyword", - "verified_result": null - }, - { - "hashed_secret": "55aac310878e11ce14b29027f77e58cb7c4fe7a4", - "is_secret": true, - "is_verified": false, - "line_number": 8, - "type": "Secret Keyword", - "verified_result": null - }, - { - "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", - "is_secret": true, - "is_verified": false, - "line_number": 56, - "type": "Secret Keyword", - "verified_result": null - }, - { - "hashed_secret": "b295b04949a98dc50ba65adcddd588077b93ab3c", - "is_secret": true, - "is_verified": false, - "line_number": 65, - "type": "Secret Keyword", - "verified_result": null - } - ], - "tools/minimal-demo-prod-scripts/catalog_values_production_deployment.json": [ - { - "hashed_secret": "216168000275f83a7bc3599e708c5bafab959783", - "is_secret": true, - "is_verified": false, - "line_number": 2, - "type": "Secret Keyword", - "verified_result": null - }, - { - "hashed_secret": "55aac310878e11ce14b29027f77e58cb7c4fe7a4", + "hashed_secret": "5073c7ac17500ef0678aebc7138a996b4f75d623", "is_secret": true, "is_verified": false, "line_number": 8, @@ -184,7 +90,7 @@ "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", "is_secret": true, "is_verified": false, - "line_number": 56, + "line_number": 37, "type": "Secret Keyword", "verified_result": null }, @@ -192,7 +98,7 @@ "hashed_secret": "b295b04949a98dc50ba65adcddd588077b93ab3c", "is_secret": true, "is_verified": false, - "line_number": 65, + "line_number": 60, "type": "Secret Keyword", "verified_result": null } diff --git a/samples/configs/hpc_catalog_values.json b/samples/configs/hpc_catalog_values.json index f9ad780b..18346e99 100644 --- a/samples/configs/hpc_catalog_values.json +++ b/samples/configs/hpc_catalog_values.json @@ -6,9 +6,9 @@ "ssh_keys": "[\"Please fill here\"]", "remote_allowed_ips": "[\"Please fill here\"]", "app_center_gui_password": "Please fill here", - "lsf_version": "Fixpack_15", + "lsf_version": "fixpack_15", "vpc_name": "__NULL__", - "cluster_subnet_id": "[]", + "cluster_subnet_id": "__NULL__", "login_subnet_id": "__NULL__", "vpc_cidr": "10.241.0.0/18", "vpc_cluster_private_subnets_cidr_blocks": "10.241.0.0/20", @@ -30,8 +30,8 @@ "enable_vpc_flow_logs": "false", "enable_ldap": "false", "ldap_basedns": "lsf.com", - "ldap_server": "null", - "ldap_server_cert": "null", + "ldap_server": "__NULL__", + "ldap_server_cert": "__NULL__", "ldap_admin_password": "", "ldap_user_name": "", "ldap_user_password": "", @@ -51,7 +51,7 @@ "observability_logs_enable_for_compute": "false", "observability_enable_platform_logs": "false", "observability_enable_metrics_routing": "false", - "observability_logs_retention_period": "false", + "observability_logs_retention_period": "7", "observability_monitoring_on_compute_nodes_enable": "false", "observability_monitoring_plan": "graduated-tier", "existing_bastion_instance_name": "__NULL__", diff --git a/tools/image-builder/variables.tf b/tools/image-builder/variables.tf index e044ede0..ddf306fc 100644 --- a/tools/image-builder/variables.tf +++ b/tools/image-builder/variables.tf @@ -190,7 +190,7 @@ variable "enable_fip" { # tflint-ignore: terraform_unused_declarations variable "cluster_name" { type = string - description = "Ensure that you have received the cluster ID from IBM technical sales. A unique identifer for HPC cluster used by IBM Cloud HPC to differentiate different HPC clusters within the same reservations. This can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. You cannot change the cluster ID after deployment." + description = "Ensure that you have received the cluster ID from IBM technical sales. A unique identifier for HPC cluster used by IBM Cloud HPC to differentiate different HPC clusters within the same reservations. This can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. You cannot change the cluster ID after deployment." validation { condition = 0 < length(var.cluster_name) && length(var.cluster_name) < 40 && can(regex("^[a-zA-Z0-9_.-]+$", var.cluster_name)) error_message = "The Cluster ID can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. Other special characters and spaces are not allowed." From 23612caaa71dc201311bb4d867d9c010edbbf956 Mon Sep 17 00:00:00 2001 From: Nupur Goyal Date: Wed, 17 Sep 2025 15:53:38 +0530 Subject: [PATCH 03/10] fixing detect-secrets --- .secrets.baseline | 122 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 121 insertions(+), 1 deletion(-) diff --git a/.secrets.baseline b/.secrets.baseline index 2d87d2c0..af25d937 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2025-09-17T10:20:38Z", + "generated_at": "2025-09-17T10:23:03Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -102,6 +102,126 @@ "type": "Secret Keyword", "verified_result": null } + ], + "tools/minimal-demo-prod-scripts/README.md": [ + { + "hashed_secret": "89a6cfe2a229151e8055abee107d45ed087bbb4f", + "is_secret": true, + "is_verified": false, + "line_number": 36, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "365b78d42089abe4583503eded60fa7c1b3e1cd0", + "is_secret": true, + "is_verified": false, + "line_number": 56, + "type": "Secret Keyword", + "verified_result": null + } + ], + "tools/minimal-demo-prod-scripts/catalog_values_demo_deployment.json": [ + { + "hashed_secret": "216168000275f83a7bc3599e708c5bafab959783", + "is_secret": true, + "is_verified": false, + "line_number": 2, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "55aac310878e11ce14b29027f77e58cb7c4fe7a4", + "is_secret": true, + "is_verified": false, + "line_number": 8, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", + "is_secret": true, + "is_verified": false, + "line_number": 56, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "b295b04949a98dc50ba65adcddd588077b93ab3c", + "is_secret": true, + "is_verified": false, + "line_number": 65, + "type": "Secret Keyword", + "verified_result": null + } + ], + "tools/minimal-demo-prod-scripts/catalog_values_minimal_deployment.json": [ + { + "hashed_secret": "216168000275f83a7bc3599e708c5bafab959783", + "is_secret": true, + "is_verified": false, + "line_number": 2, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "55aac310878e11ce14b29027f77e58cb7c4fe7a4", + "is_secret": true, + "is_verified": false, + "line_number": 8, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", + "is_secret": true, + "is_verified": false, + "line_number": 56, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "b295b04949a98dc50ba65adcddd588077b93ab3c", + "is_secret": true, + "is_verified": false, + "line_number": 65, + "type": "Secret Keyword", + "verified_result": null + } + ], + "tools/minimal-demo-prod-scripts/catalog_values_production_deployment.json": [ + { + "hashed_secret": "216168000275f83a7bc3599e708c5bafab959783", + "is_secret": true, + "is_verified": false, + "line_number": 2, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "55aac310878e11ce14b29027f77e58cb7c4fe7a4", + "is_secret": true, + "is_verified": false, + "line_number": 8, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", + "is_secret": true, + "is_verified": false, + "line_number": 56, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "b295b04949a98dc50ba65adcddd588077b93ab3c", + "is_secret": true, + "is_verified": false, + "line_number": 65, + "type": "Secret Keyword", + "verified_result": null + } ] }, "version": "0.13.1+ibm.62.dss", From 5c4047523c40ca8149810cf1380d1c493bf5538d Mon Sep 17 00:00:00 2001 From: Nupur Goyal Date: Wed, 17 Sep 2025 16:37:45 +0530 Subject: [PATCH 04/10] fixing pre-commit --- tools/access-management/permissions.sh | 2 +- tools/image-builder/variables.tf | 2 +- tools/minimal-demo-prod-scripts/cp.sh | 15 ++++---- .../create_lsf_environment.sh | 34 +++++++++---------- tools/minimal-demo-prod-scripts/destroy.sh | 11 +++--- tools/minimal-demo-prod-scripts/jump.sh | 13 ++++--- tools/minimal-demo-prod-scripts/show.sh | 9 +++-- 7 files changed, 41 insertions(+), 45 deletions(-) diff --git a/tools/access-management/permissions.sh b/tools/access-management/permissions.sh index 35af8fef..d8576ba7 100755 --- a/tools/access-management/permissions.sh +++ b/tools/access-management/permissions.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash set -e ##################################### diff --git a/tools/image-builder/variables.tf b/tools/image-builder/variables.tf index ddf306fc..e044ede0 100644 --- a/tools/image-builder/variables.tf +++ b/tools/image-builder/variables.tf @@ -190,7 +190,7 @@ variable "enable_fip" { # tflint-ignore: terraform_unused_declarations variable "cluster_name" { type = string - description = "Ensure that you have received the cluster ID from IBM technical sales. A unique identifier for HPC cluster used by IBM Cloud HPC to differentiate different HPC clusters within the same reservations. This can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. You cannot change the cluster ID after deployment." + description = "Ensure that you have received the cluster ID from IBM technical sales. A unique identifer for HPC cluster used by IBM Cloud HPC to differentiate different HPC clusters within the same reservations. This can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. You cannot change the cluster ID after deployment." validation { condition = 0 < length(var.cluster_name) && length(var.cluster_name) < 40 && can(regex("^[a-zA-Z0-9_.-]+$", var.cluster_name)) error_message = "The Cluster ID can be up to 39 alphanumeric characters including the underscore (_), the hyphen (-), and the period (.) characters. Other special characters and spaces are not allowed." diff --git a/tools/minimal-demo-prod-scripts/cp.sh b/tools/minimal-demo-prod-scripts/cp.sh index 014ef4b1..e71b6f08 100755 --- a/tools/minimal-demo-prod-scripts/cp.sh +++ b/tools/minimal-demo-prod-scripts/cp.sh @@ -13,36 +13,35 @@ fi CURRENT_ACCOUNT_GUID=$(ibmcloud target --output json | jq -r '.account.guid') if [ "$CURRENT_ACCOUNT_GUID" != "$ACCOUNT_GUID" ] then - ibmcloud login -a cloud.ibm.com --apikey $API_KEY -r $REGION -g $RESOURCE_GROUP + ibmcloud login -a cloud.ibm.com --apikey "$API_KEY" -r "$REGION" -g "$RESOURCE_GROUP" fi -CURRENT_ACCOUNT_NAME=$(ibmcloud target --output json | jq -r '.account.name') echo "target account $CURRENT_ACCOUNT_GUID" CURRENT_REGION=$(ibmcloud target --output json | jq -r '.region.name') if [ "$CURRENT_REGION" != "$REGION" ] then - ibmcloud target -r $REGION + ibmcloud target -r "$REGION" fi echo "target region $REGION" # ibmcloud is ip $1-bastion-001-fip --output json | jq -r '. | .address' -BASTION_IP=$(ibmcloud is ips | grep $1-bastion | grep 001-fip | awk '{print $2}') +BASTION_IP=$(ibmcloud is ips | grep "$1"-bastion | grep 001-fip | awk '{print $2}') echo "Bastion IP: $BASTION_IP" # ibmcloud is instance $1-login-001 --output json | jq -r '. | .network_interfaces[0].primary_ip.address' -LOGIN_IP=$(ibmcloud is instances | grep $1-login | grep 001 | awk '{print $4}') +LOGIN_IP=$(ibmcloud is instances | grep "$1"-login | grep 001 | awk '{print $4}') echo "Login IP: $LOGIN_IP" # ibmcloud is instance $1-mgmt-1-001 --output json | jq -r '. | .network_interfaces[0].primary_ip.address' -LSF_IP=$(ibmcloud is instances | grep $1-mgmt-1 | grep 001 | awk '{print $4}') +LSF_IP=$(ibmcloud is instances | grep "$1"-mgmt-1 | grep 001 | awk '{print $4}') echo "LSF IP: $LSF_IP" if [ "$3" == "mgmt" ] then echo "copying $2 to LSF /mnt/vpcstorage/tools..." - scp -o UserKnownHostsFile=/dev/null -o StrictHostKeychecking=no -o IdentitiesOnly=yes -J ubuntu@$BASTION_IP $2 lsfadmin@$LSF_IP:/mnt/vpcstorage/tools/$(basename $2) + scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -J ubuntu@"$BASTION_IP" "$2" "lsfadmin@$LSF_IP:/mnt/vpcstorage/tools/$(basename "$2")" else echo "copying $2 to LOGIN /home/lsfadmin..." - scp -o UserKnownHostsFile=/dev/null -o StrictHostKeychecking=no -o IdentitiesOnly=yes -J ubuntu@$BASTION_IP $2 lsfadmin@$LOGIN_IP:/home/lsfadmin/$(basename $2) + scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -J ubuntu@"$BASTION_IP" "$2" "lsfadmin@$LOGIN_IP:/home/lsfadmin/$(basename "$2")" fi diff --git a/tools/minimal-demo-prod-scripts/create_lsf_environment.sh b/tools/minimal-demo-prod-scripts/create_lsf_environment.sh index 3af0fbc4..d1054011 100755 --- a/tools/minimal-demo-prod-scripts/create_lsf_environment.sh +++ b/tools/minimal-demo-prod-scripts/create_lsf_environment.sh @@ -29,25 +29,25 @@ fi REMOTE_IP=$(curl -s https://ipv4.icanhazip.com/) -sed s/XX_PREFIX_XX/$1/g $TEMPLATE_FILE | \ -sed s/XX_API_KEY_XX/$API_KEY/g | \ -sed s/XX_RESOURCE_GROUP_XX/$RESOURCE_GROUP/g | \ -sed s/XX_SSH_KEY_XX/$SSH_KEY/g | \ -sed s/XX_ZONES_XX/$ZONES/g | \ -sed s/XX_REMOTE_IP_XX/$REMOTE_IP/g | \ -sed s/XX_APP_CENTER_GUI_PASSWORD_XX/$APP_CENTER_GUI_PASSWORD/g > environment_values_$1.json +sed s/XX_PREFIX_XX/"$1"/g "$TEMPLATE_FILE" | \ +sed s/XX_API_KEY_XX/"$API_KEY"/g | \ +sed s/XX_RESOURCE_GROUP_XX/"$RESOURCE_GROUP"/g | \ +sed s/XX_SSH_KEY_XX/"$SSH_KEY"/g | \ +sed s/XX_ZONES_XX/"$ZONES"/g | \ +sed s/XX_REMOTE_IP_XX/"$REMOTE_IP"/g | \ +sed s/XX_APP_CENTER_GUI_PASSWORD_XX/"$APP_CENTER_GUI_PASSWORD"/g > environment_values_"$1".json -ibmcloud login -a cloud.ibm.com --apikey $API_KEY -r $REGION -g $RESOURCE_GROUP -ibmcloud target -r $REGION +ibmcloud login -a cloud.ibm.com --apikey "$API_KEY" -r "$REGION" -g "$RESOURCE_GROUP" +ibmcloud target -r "$REGION" # Run install and capture output INSTALL_LOG=$(mktemp) if ! ibmcloud catalog install --timeout 3600 \ - --vl $LSF_TILE_VERSION \ - --override-values environment_values_$1.json \ - --workspace-region $REGION \ - --workspace-rg-id $RESOURCE_GROUP \ - --workspace-name $1 \ + --vl "$LSF_TILE_VERSION" \ + --override-values environment_values_"$1".json \ + --workspace-region "$REGION" \ + --workspace-rg-id "$RESOURCE_GROUP" \ + --workspace-name "$1" \ --workspace-tf-version 1.9 | tee "$INSTALL_LOG"; then echo "Install command failed to start" exit 1 @@ -67,13 +67,13 @@ if grep -q "FAILED" "$INSTALL_LOG"; then fi # If success, print the Output IPs -BASTION_IP=$(ibmcloud is ips | grep $1-bastion | grep 001-fip | awk '{print $2}') +BASTION_IP=$(ibmcloud is ips | grep "$1"-bastion | grep 001-fip | awk '{print $2}') echo "Bastion IP: $BASTION_IP" -LOGIN_IP=$(ibmcloud is instances | grep $1-login | grep 001 | awk '{print $4}') +LOGIN_IP=$(ibmcloud is instances | grep "$1"-login | grep 001 | awk '{print $4}') echo "Login IP: $LOGIN_IP" -LSF_IP=$(ibmcloud is instances | grep $1-mgmt-1 | grep 001 | awk '{print $4}') +LSF_IP=$(ibmcloud is instances | grep "$1"-mgmt-1 | grep 001 | awk '{print $4}') echo "LSF IP: $LSF_IP" echo "" diff --git a/tools/minimal-demo-prod-scripts/destroy.sh b/tools/minimal-demo-prod-scripts/destroy.sh index a6634a55..9d69d01e 100755 --- a/tools/minimal-demo-prod-scripts/destroy.sh +++ b/tools/minimal-demo-prod-scripts/destroy.sh @@ -13,16 +13,15 @@ fi CURRENT_ACCOUNT_GUID=$(ibmcloud target --output json | jq -r '.account.guid') if [ "$CURRENT_ACCOUNT_GUID" != "$ACCOUNT_GUID" ] then - ibmcloud login -a cloud.ibm.com --apikey $API_KEY -r $REGION -g $RESOURCE_GROUP + ibmcloud login -a cloud.ibm.com --apikey "$API_KEY" -r "$REGION" -g "$RESOURCE_GROUP" fi -CURRENT_ACCOUNT_NAME=$(ibmcloud target --output json | jq -r '.account.name') echo "target account $CURRENT_ACCOUNT_GUID" CURRENT_REGION=$(ibmcloud target --output json | jq -r '.region.name') if [ "$CURRENT_REGION" != "$REGION" ] then - ibmcloud target -r $REGION + ibmcloud target -r "$REGION" fi echo "target region $REGION" @@ -33,7 +32,7 @@ WORKSPACE_ID=$(ibmcloud schematics workspace list --output json \ ibmcloud schematics workspace get --id "$WORKSPACE_ID" -read -p "Do you want to destroy? (yes/no) " yn +read -r -p "Do you want to destroy? (yes/no) " yn case $yn in yes ) echo ok, we will proceed;; @@ -43,6 +42,6 @@ case $yn in exit 1;; esac -rm environment_values_$1.json +rm environment_values_"$1".json -ibmcloud schematics destroy --id $WORKSPACE_ID -f +ibmcloud schematics destroy --id "$WORKSPACE_ID" -f diff --git a/tools/minimal-demo-prod-scripts/jump.sh b/tools/minimal-demo-prod-scripts/jump.sh index 3ca27015..335878fe 100755 --- a/tools/minimal-demo-prod-scripts/jump.sh +++ b/tools/minimal-demo-prod-scripts/jump.sh @@ -13,27 +13,26 @@ fi CURRENT_ACCOUNT_GUID=$(ibmcloud target --output json | jq -r '.account.guid') if [ "$CURRENT_ACCOUNT_GUID" != "$ACCOUNT_GUID" ] then - ibmcloud login -a cloud.ibm.com --apikey $API_KEY -r $REGION -g $RESOURCE_GROUP + ibmcloud login -a cloud.ibm.com --apikey "$API_KEY" -r "$REGION" -g "$RESOURCE_GROUP" fi -CURRENT_ACCOUNT_NAME=$(ibmcloud target --output json | jq -r '.account.name') echo "target account $CURRENT_ACCOUNT_GUID" CURRENT_REGION=$(ibmcloud target --output json | jq -r '.region.name') if [ "$CURRENT_REGION" != "$REGION" ] then - ibmcloud target -r $REGION + ibmcloud target -r "$REGION" fi echo "target region $REGION" -BASTION_IP=$(ibmcloud is ips | grep $1-bastion | grep 001-fip | awk '{print $2}') +BASTION_IP=$(ibmcloud is ips | grep "$1"-bastion | grep 001-fip | awk '{print $2}') echo "Bastion IP: $BASTION_IP" -LOGIN_IP=$(ibmcloud is instances | grep $1-login | grep 001 | awk '{print $4}') +LOGIN_IP=$(ibmcloud is instances | grep "$1"-login | grep 001 | awk '{print $4}') echo "Login IP: $LOGIN_IP" -LSF_IP=$(ibmcloud is instances | grep $1-mgmt-1 | grep 001 | awk '{print $4}') +LSF_IP=$(ibmcloud is instances | grep "$1"-mgmt-1 | grep 001 | awk '{print $4}') echo "LSF IP: $LSF_IP" echo "Jumping to Login Node..." -ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -J ubuntu@$BASTION_IP lsfadmin@$LOGIN_IP +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -J ubuntu@"$BASTION_IP" lsfadmin@"$LOGIN_IP" diff --git a/tools/minimal-demo-prod-scripts/show.sh b/tools/minimal-demo-prod-scripts/show.sh index 198d4237..69c5e3b7 100755 --- a/tools/minimal-demo-prod-scripts/show.sh +++ b/tools/minimal-demo-prod-scripts/show.sh @@ -13,20 +13,19 @@ fi CURRENT_ACCOUNT_GUID=$(ibmcloud target --output json | jq -r '.account.guid') if [ "$CURRENT_ACCOUNT_GUID" != "$ACCOUNT_GUID" ] then - ibmcloud login -a cloud.ibm.com --apikey $API_KEY -r $REGION -g $RESOURCE_GROUP + ibmcloud login -a cloud.ibm.com --apikey "$API_KEY" -r "$REGION" -g "$RESOURCE_GROUP" fi -CURRENT_ACCOUNT_NAME=$(ibmcloud target --output json | jq -r '.account.name') echo "target account $CURRENT_ACCOUNT_GUID" CURRENT_REGION=$(ibmcloud target --output json | jq -r '.region.name') if [ "$CURRENT_REGION" != "$REGION" ] then - ibmcloud target -r $REGION + ibmcloud target -r "$REGION" fi echo "target region $REGION" echo "finding schematics workspace..." -WORKSPACE_ID=$(ibmcloud schematics workspace list | grep $1 | awk '{ print $2 }') +WORKSPACE_ID=$(ibmcloud schematics workspace list | grep "$1" | awk '{ print $2 }') -ibmcloud schematics workspace get --id $WORKSPACE_ID +ibmcloud schematics workspace get --id "$WORKSPACE_ID" From db3b1036d90a19c1d42fdef56db94e9eb55f03d9 Mon Sep 17 00:00:00 2001 From: Nupur Goyal Date: Wed, 17 Sep 2025 16:40:52 +0530 Subject: [PATCH 05/10] fixing pre-commit --- tools/minimal-demo-prod-scripts/create_lsf_environment.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tools/minimal-demo-prod-scripts/create_lsf_environment.sh b/tools/minimal-demo-prod-scripts/create_lsf_environment.sh index d1054011..6b7af2c4 100755 --- a/tools/minimal-demo-prod-scripts/create_lsf_environment.sh +++ b/tools/minimal-demo-prod-scripts/create_lsf_environment.sh @@ -95,8 +95,3 @@ echo " ./submit.sh" echo " bjobs ..." echo " bhosts ..." echo "----------------------------------------" -echo "" -echo "Alternatively, you can run web_service.sh to configure LSF Web Services for local client access (Standalone LSF Client):" -echo " ./web_service.sh $1" -echo "" -echo "⚑ This will allow users to connect to an LSF cluster from their local environment." From 12b843784ccd8a0ac05fa2f9485dc659ee8b9620 Mon Sep 17 00:00:00 2001 From: Nupur Goyal Date: Wed, 17 Sep 2025 18:10:42 +0530 Subject: [PATCH 06/10] fixing readme --- tools/access-management/README.md | 2 ++ tools/minimal-demo-prod-scripts/README.md | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/tools/access-management/README.md b/tools/access-management/README.md index 47fde9bd..6f1ca6f6 100644 --- a/tools/access-management/README.md +++ b/tools/access-management/README.md @@ -5,6 +5,8 @@ How to run: ``` +ibmcloud login --apikey -g + chmod +x permissions.sh ./permissions.sh diff --git a/tools/minimal-demo-prod-scripts/README.md b/tools/minimal-demo-prod-scripts/README.md index dd8ba4bc..309f6cfa 100644 --- a/tools/minimal-demo-prod-scripts/README.md +++ b/tools/minimal-demo-prod-scripts/README.md @@ -94,3 +94,10 @@ sh submit.sh bjobs lshosts -w ``` + +#### 5. To destroy the created infrastructure + +``` +chmod +x destroy.sh + ./destroy.sh +``` From d2ce3965a7566fb2aad0329afb383ac43110e931 Mon Sep 17 00:00:00 2001 From: Nupur Goyal Date: Fri, 19 Sep 2025 21:50:08 +0530 Subject: [PATCH 07/10] fixing existing admin issues --- tools/access-management/permissions.sh | 61 ++++++++++++++++---------- 1 file changed, 39 insertions(+), 22 deletions(-) diff --git a/tools/access-management/permissions.sh b/tools/access-management/permissions.sh index d8576ba7..60666e40 100755 --- a/tools/access-management/permissions.sh +++ b/tools/access-management/permissions.sh @@ -48,13 +48,30 @@ echo "πŸ” Checking if $ADMIN_EMAIL can assign IAM permissions..." has_permission=false check_policies() { - echo "$1" | jq -e ' - .[]? | - select(.roles[]?.display_name == "Administrator") | - select(.resources[]?.attributes[]? | select(.name == "serviceName" and .value == "iam-identity")) | - select(.resources[]?.attributes[]?.name == "accountId") | - select(all(.resources[]?.attributes[]?.name; . != "resourceGroupId")) - ' >/dev/null + local policies="$1" + + # Check Administrator role for serviceType=service + local has_admin + has_admin=$(echo "$policies" | jq -e ' + .[] | + select(.roles? != null) | + select(any(.roles[]?.display_name; . == "Administrator")) | + select(any(.resources[].attributes[]?; .name == "accountId")) | + select(any(.resources[].attributes[]?; .name == "serviceType" and .value == "service")) + ' >/dev/null 2>&1 && echo "true" || echo "false") + + # Check role for serviceType=platform_service (Viewer, Editor, or Administrator) + local has_platform_role + has_platform_role=$(echo "$policies" | jq -e ' + .[] | + select(.roles? != null) | + select(any(.roles[]?.display_name; . == "Viewer" or . == "Editor" or . == "Administrator")) | + select(any(.resources[].attributes[]?; .name == "accountId")) | + select(any(.resources[].attributes[]?; .name == "serviceType" and .value == "platform_service")) + ' >/dev/null 2>&1 && echo "true" || echo "false") + + # Return true only if both checks pass + [[ "$has_admin" == "true" && "$has_platform_role" == "true" ]] } USER_POLICIES=$(ibmcloud iam user-policies "$ADMIN_EMAIL" --output json 2>/dev/null || echo "[]") @@ -64,25 +81,27 @@ if echo "$USER_POLICIES" | jq empty 2>/dev/null; then fi fi -ACCESS_GROUPS_FOR_ADMIN=$(ibmcloud iam access-groups --ibm-id "$ADMIN_EMAIL" --output json 2>/dev/null || echo "[]") -if echo "$ACCESS_GROUPS_FOR_ADMIN" | jq empty 2>/dev/null; then - for GROUP_ID in $(echo "$ACCESS_GROUPS_FOR_ADMIN" | jq -r '.[].id // empty'); do - GROUP_POLICIES=$(ibmcloud iam access-group-policies "$GROUP_ID" --output json 2>/dev/null || echo "[]") - if echo "$GROUP_POLICIES" | jq empty 2>/dev/null; then - if check_policies "$GROUP_POLICIES"; then - has_permission=true - break - fi - fi - done +if [ "$has_permission" != true ]; then + ACCESS_GROUPS_FOR_ADMIN=$(ibmcloud iam access-groups -u "$ADMIN_EMAIL" --output json 2>/dev/null || echo "[]") + + # Collect all policies from all access groups into a single array + ALL_GROUP_POLICIES="[]" + while IFS= read -r GROUP_NAME; do + GROUP_POLICIES=$(ibmcloud iam access-group-policies "$GROUP_NAME" --output json 2>/dev/null || echo "[]") + ALL_GROUP_POLICIES=$(echo "$ALL_GROUP_POLICIES $GROUP_POLICIES" | jq -s 'add') + done < <(echo "$ACCESS_GROUPS_FOR_ADMIN" | jq -r '.[].name // empty') + # Check all group policies at once + if check_policies "$ALL_GROUP_POLICIES"; then + has_permission=true + fi fi if [ "$has_permission" != true ]; then - echo "❌ $ADMIN_EMAIL does NOT have account-level iam-identity Administrator rights β€” cannot assign permissions." + echo "❌ $ADMIN_EMAIL does NOT have account-level Administrator rights β€” cannot assign permissions." exit 1 fi -echo "βœ… $ADMIN_EMAIL has account-level iam-identity Administrator rights β€” proceeding." +echo "βœ… $ADMIN_EMAIL has account-level Administrator rights β€” proceeding." ##################################### # 3. Role assignment definitions @@ -94,7 +113,6 @@ sysdig-monitor|Administrator|Manager kms|Service Configuration Reader|Manager secrets-manager|Administrator|Manager sysdig-secure|Administrator| -iam-identity|Administrator| is|Editor|" # New friendly names list (service|friendly name) @@ -105,7 +123,6 @@ sysdig-monitor|Cloud Monitoring kms|Key Protect secrets-manager|Secrets Manager sysdig-secure|Security and Compliance Center Workload Protection -iam-identity|IAM Identity Service is|VPC Infrastructure Services" get_friendly_name() { From 9e0c64709dcca32c883339e8e1bd8bc9e44701ee Mon Sep 17 00:00:00 2001 From: Nupur Goyal Date: Fri, 19 Sep 2025 22:53:32 +0530 Subject: [PATCH 08/10] fixing detect secrets --- .secrets.baseline | 3 --- 1 file changed, 3 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index af25d937..c26ba777 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -21,9 +21,6 @@ { "name": "BasicAuthDetector" }, - { - "name": "BoxDetector" - }, { "name": "CloudantDetector" }, From f9fedce7337869a606d7daa5690d9022774420b4 Mon Sep 17 00:00:00 2001 From: Nupur Goyal Date: Fri, 19 Sep 2025 23:07:10 +0530 Subject: [PATCH 09/10] fixing pre-commit issue --- .secrets.baseline | 3 +++ common-dev-assets | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.secrets.baseline b/.secrets.baseline index c26ba777..af25d937 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -21,6 +21,9 @@ { "name": "BasicAuthDetector" }, + { + "name": "BoxDetector" + }, { "name": "CloudantDetector" }, diff --git a/common-dev-assets b/common-dev-assets index 6739b3a0..325cfd0d 160000 --- a/common-dev-assets +++ b/common-dev-assets @@ -1 +1 @@ -Subproject commit 6739b3a089aa08a072dd83c8b594311e42fc96d4 +Subproject commit 325cfd0d91902e08079644092bbf298c4872f388 From 840fdbef0fc55d3a5c5af777cb2b9031df8dcdff Mon Sep 17 00:00:00 2001 From: Nupur Goyal Date: Fri, 19 Sep 2025 23:21:50 +0530 Subject: [PATCH 10/10] fixing catalog.json --- ibm_catalog.json | 1 + 1 file changed, 1 insertion(+) diff --git a/ibm_catalog.json b/ibm_catalog.json index 8bd5af3e..adb3b716 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -48,6 +48,7 @@ "name": "Cluster-with-LSF", "install_type": "fullstack", "working_directory": "solutions/lsf", + "terraform_version": "1.10.5", "compliance": { "authority": "scc-v3", "profiles": [