Route alert buttons through Rancher proxy instead of Harvester IP

Author: HiranAdikari

modules/monitoring/README.md

@@ -24,9 +24,11 @@ module "monitoring" {
   kubeconfig_context      = "local"
   google_chat_webhook_url = var.google_chat_webhook_url
 
-  # Optional — show a "View Alert" deep-link button in each notification card.
-  # Find this URL: Harvester UI → Add-ons → rancher-monitoring → alert-manager
-  # alertmanager_url = "https://<harvester-ip>/api/v1/namespaces/cattle-monitoring-system/services/http:rancher-monitoring-alertmanager:9093/proxy"
+  # Optional — adds "View Alert" and "View in Prometheus" buttons to each card.
+  # Both URLs are routed through Rancher's authenticated proxy so users don't
+  # need a separate session directly on the Harvester IP.
+  # rancher_url          = "https://rancher.example.com"
+  # harvester_cluster_id = "c-xxxxx"   # Rancher UI → Cluster Management → cluster row
 }
 ```
 
@@ -267,16 +269,15 @@ stored in the `calert-config` Secret and rendered by calert at runtime.
 └─────────────────────────────────────────────────────┘
 ```
 
-**"View Alert" button** is only rendered when `alertmanager_url` is set. It
-links to:
+Both buttons are rendered only when `rancher_url` and `harvester_cluster_id`
+are set. URLs are constructed at `terraform apply` time and routed through
+Rancher's authenticated proxy — no separate Harvester session required.
+
 ```text
-<alertmanager_url>/#/alerts?filter={alertname="<name>"}
+View Alert    → <rancher_url>/k8s/clusters/<id>/…/rancher-monitoring-alertmanager:9093/proxy/#/alerts?filter={alertname="<name>"}
+View in Prometheus → <rancher_url>/k8s/clusters/<id>/…/rancher-monitoring-prometheus:9090/proxy/alerts?search=<name>
 ```
 
-**"View in Prometheus" button** is only rendered when the alert carries a
-`GeneratorURL` (set automatically by Prometheus when a rule fires for real;
-absent in synthetic test-fires).
-
 ### Template evaluation: Terraform vs Go
 
 The card template is a Go template evaluated by calert at runtime — but it
@@ -312,7 +313,8 @@ calert ignores `${ }` delimiters.
 
 | Name | Type | Default | Description |
 |---|---|---|---|
-| `alertmanager_url` | string | `""` | Alertmanager UI base URL — enables "View Alert" button. Leave empty to omit. |
+| `rancher_url` | string | `""` | Base URL of the Rancher server (e.g. `https://rancher.example.com`). Combined with `harvester_cluster_id` to build Rancher-authenticated proxy URLs for both buttons. Leave empty to omit buttons. |
+| `harvester_cluster_id` | string | `""` | Rancher cluster ID for the Harvester cluster (e.g. `c-v7gvt`). Found in Rancher UI → Cluster Management. Required when `rancher_url` is set. |
 | `monitoring_namespace` | string | `cattle-monitoring-system` | Namespace where rancher-monitoring runs |
 | `dashboards_namespace` | string | `cattle-dashboards` | Namespace where Grafana picks up dashboard ConfigMaps |
 | `runbook_base_url` | string | `https://wiki.internal/runbooks/harvester` | Base URL prepended to each alert's `runbook_url` annotation |

modules/monitoring/main.tf

@@ -15,6 +15,15 @@ locals {
   # AlertmanagerConfig and template resources go in the monitoring namespace.
   ns  = var.monitoring_namespace
   dns = var.dashboards_namespace
+
+  # Rancher-authenticated proxy base for this Harvester cluster.
+  # e.g. https://rancher.example.com/k8s/clusters/c-v7gvt
+  # Both button URLs are derived from this so they pass through Rancher auth
+  # rather than hitting the Harvester IP directly (which returns 403 to users
+  # who are not separately authenticated to Harvester).
+  rancher_proxy_base    = (var.rancher_url != "" && var.harvester_cluster_id != "") ? "${var.rancher_url}/k8s/clusters/${var.harvester_cluster_id}" : ""
+  alertmanager_base_url = local.rancher_proxy_base != "" ? "${local.rancher_proxy_base}/api/v1/namespaces/${var.monitoring_namespace}/services/http:rancher-monitoring-alertmanager:9093/proxy" : ""
+  prometheus_base_url   = local.rancher_proxy_base != "" ? "${local.rancher_proxy_base}/api/v1/namespaces/${var.monitoring_namespace}/services/http:rancher-monitoring-prometheus:9090/proxy" : ""
 }
 
 # ── Alertmanager config + calert (Google Chat webhook forwarder) ──────────────
@@ -63,9 +72,10 @@ locals {
   # Section-level headers (strings) only — card-level header objects are not
   # supported by Google Chat webhooks and produce blank cards.
   # Available functions: toUpper, Title, SortedPairs (calert v2.3.0)
-  # var.alertmanager_url is a Terraform interpolation — baked in at apply time as
-  # a literal string. {{.GeneratorURL}} is a Go template variable — resolved at
-  # runtime per alert. Both coexist safely in the same heredoc.
+  # local.alertmanager_base_url / local.prometheus_base_url are Terraform
+  # interpolations — baked in at apply time as literal strings. The Go template
+  # vars ({{.Labels.alertname}} etc.) are resolved at runtime per alert.
+  # Both coexist safely in the same heredoc.
   calert_message_tmpl = <<-TMPL
     {{- define "cardsV2" -}}
     {
@@ -78,20 +88,12 @@ locals {
               {{- if ne $i 0 -}},{{- end -}}
               {"decoratedText": {"text": "{{ $pair.Name | Title }}: {{ $pair.Value }}"}}
               {{- end -}}
-              %{~if var.alertmanager_url != ""~}
-              ,{"buttonList": {"buttons": [
-                {"text": "View Alert", "onClick": {"openLink": {"url": "${var.alertmanager_url}/#/alerts?filter=%7Balertname%3D%22{{.Labels.alertname}}%22%7D"}}}
-                {{- if .GeneratorURL -}}
-                ,{"text": "View in Prometheus", "onClick": {"openLink": {"url": "{{.GeneratorURL}}"}}}
-                {{- end -}}
-              ]}}
-              %{~else~}
-              {{- if .GeneratorURL -}}
+              %{~ if local.rancher_proxy_base != "" ~}
               ,{"buttonList": {"buttons": [
-                {"text": "View in Prometheus", "onClick": {"openLink": {"url": "{{.GeneratorURL}}"}}}
+                {"text": "View Alert", "onClick": {"openLink": {"url": "${local.alertmanager_base_url}/#/alerts?filter=%7Balertname%3D%22{{.Labels.alertname}}%22%7D"}}},
+                {"text": "View in Prometheus", "onClick": {"openLink": {"url": "${local.prometheus_base_url}/alerts?search={{.Labels.alertname}}"}}}
               ]}}
-              {{- end -}}
-              %{~endif~}
+              %{~ endif ~}
             ]
           },
           {

modules/monitoring/variables.tf

@@ -21,10 +21,16 @@ variable "google_chat_webhook_url" {
   description = "Google Chat incoming webhook URL for alert notifications."
 }
 
-variable "alertmanager_url" {
+variable "rancher_url" {
   type        = string
   default     = ""
-  description = "Base URL of the Alertmanager UI (e.g. https://<rancher>/api/v1/namespaces/cattle-monitoring-system/services/http:rancher-monitoring-alertmanager:9093/proxy). Used for the 'View Alert' button in Google Chat cards. Leave empty to omit the button."
+  description = "Base URL of the Rancher server (e.g. https://rancher.example.com). Combined with harvester_cluster_id to build Rancher-authenticated proxy URLs for the 'View Alert' and 'View in Prometheus' buttons. Leave empty to omit both buttons."
+}
+
+variable "harvester_cluster_id" {
+  type        = string
+  default     = ""
+  description = "Rancher cluster ID for the Harvester cluster (e.g. c-v7gvt). Found in Rancher UI → Cluster Management → cluster row. Required when rancher_url is set."
 }
 
 # ── Optional (monitoring namespaces) ─────────────────────────────────────────