Merge branch 'main' into bbr-docs

Author: srampal

.github/workflows/kal.yml

@@ -20,9 +20,5 @@ jobs:
           persist-credentials: false
       - name: Set up Go
         uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # tag=v5.5.0
-      - name: Install Golang CI Lint
-        run: go install github.com/golangci/golangci-lint/v2/cmd/[email protected]
-      - name: Build KAL
-        run: golangci-lint custom
-      - name: run api linter
-        run: ./bin/golangci-kube-api-linter run -c ./.golangci-kal.yml ./...
+      - name: Run API Linter
+        run: make api-lint

.github/workflows/non-main-gatekeeper.yml

@@ -0,0 +1,18 @@
+name: Label non-main PRs
+
+on:
+  pull_request:
+    types: [opened, edited, synchronize, reopened]
+
+jobs:
+  add-label:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Add labels when base branch is not main
+        if: github.event.pull_request.base.ref != 'main'
+        uses: actions-ecosystem/action-add-labels@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          labels: |
+            do-not-merge/hold
+            do-not-merge/cherry-pick-not-approved

Makefile

@@ -160,8 +160,12 @@ lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes
 ci-lint: golangci-lint
 	$(GOLANGCI_LINT) run --timeout 15m0s
 
+.PHONY: api-lint
+api-lint: golangci-api-lint
+	$(GOLANGCI_API_LINT) run -c .golangci-kal.yml --timeout 15m0s ./...
+
 .PHONY: verify
-verify: vet fmt-verify generate ci-lint verify-all
+verify: vet fmt-verify generate ci-lint api-lint verify-all
 	git --no-pager diff --exit-code config api client-go
 
 .PHONY: verify-crds
@@ -366,6 +370,7 @@ CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
 ENVTEST ?= $(LOCALBIN)/setup-envtest
 CRD_REF_DOCS ?= $(LOCALBIN)/crd-ref-docs
 GOLANGCI_LINT = $(LOCALBIN)/golangci-lint
+GOLANGCI_API_LINT = $(LOCALBIN)/golangci-kube-api-linter
 HELM = $(PROJECT_DIR)/bin/helm
 YQ = $(PROJECT_DIR)/bin/yq
 KUBECTL_VALIDATE = $(PROJECT_DIR)/bin/kubectl-validate
@@ -407,6 +412,11 @@ golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
 $(GOLANGCI_LINT): $(LOCALBIN)
 	$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/v2/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION))
 
+.PHONY: golangci-api-lint
+golangci-api-lint: golangci-lint $(GOLANGCI_API_LINT) ## Download golangci-lint locally if necessary before building KAL
+$(GOLANGCI_API_LINT):
+	$(GOLANGCI_LINT) custom
+
 .PHONY: yq
 yq: ## Download yq locally if necessary.
 	GOBIN=$(PROJECT_DIR)/bin GO111MODULE=on go install github.com/mikefarah/yq/v4@$(YQ_VERSION)

README.md

@@ -10,9 +10,6 @@ This is achieved by leveraging Envoy's [External Processing] (ext-proc) to exten
 
 [Inference Gateway]:#concepts-and-definitions
 
-
-> ***NOTE*** : As we prep for our `v1` release, some of our docs may fall out of scope, we are working hard to get these up to date and they will be ready by the time we launch `v1`. Thanks!
-
 ## New!
 Inference Gateway has partnered with vLLM to accelerate LLM serving optimizations with [llm-d](https://llm-d.ai/blog/llm-d-announce)!
 

bbr.Dockerfile

@@ -18,6 +18,7 @@ RUN go mod download
 COPY cmd/bbr ./cmd
 COPY pkg ./pkg
 COPY internal ./internal
+COPY api ./api
 WORKDIR /src/cmd
 RUN go build -o /bbr
 

cmd/epp/runner/runner.go

@@ -80,7 +80,7 @@ var (
 	enablePprof    = flag.Bool("enable-pprof", runserver.DefaultEnablePprof, "Enables pprof handlers. Defaults to true. Set to false to disable pprof handlers.")
 	poolName       = flag.String("pool-name", runserver.DefaultPoolName, "Name of the InferencePool this Endpoint Picker is associated with.")
 	poolGroup      = flag.String("pool-group", runserver.DefaultPoolGroup, "group of the InferencePool this Endpoint Picker is associated with.")
-	poolNamespace  = flag.String("pool-namespace", runserver.DefaultPoolNamespace, "Namespace of the InferencePool this Endpoint Picker is associated with.")
+	poolNamespace  = flag.String("pool-namespace", "", "Namespace of the InferencePool this Endpoint Picker is associated with.")
 	logVerbosity   = flag.Int("v", logging.DEFAULT, "number for the log level verbosity")
 	secureServing  = flag.Bool("secure-serving", runserver.DefaultSecureServing, "Enables secure serving. Defaults to true.")
 	healthChecking = flag.Bool("health-checking", runserver.DefaultHealthChecking, "Enables health checking")
@@ -188,9 +188,20 @@ func (r *Runner) Run(ctx context.Context) error {
 		FilterProvider: filters.WithAuthenticationAndAuthorization,
 	}
 
+	// Determine pool namespace: if --pool-namespace is non-empty, use it; else NAMESPACE env var; else default
+	resolvePoolNamespace := func() string {
+		if *poolNamespace != "" {
+			return *poolNamespace
+		}
+		if nsEnv := os.Getenv("NAMESPACE"); nsEnv != "" {
+			return nsEnv
+		}
+		return runserver.DefaultPoolNamespace
+	}
+	resolvedPoolNamespace := resolvePoolNamespace()
 	poolNamespacedName := types.NamespacedName{
 		Name:      *poolName,
-		Namespace: *poolNamespace,
+		Namespace: resolvedPoolNamespace,
 	}
 	poolGroupKind := schema.GroupKind{
 		Group: *poolGroup,

config/charts/inferencepool/README.md

@@ -117,6 +117,30 @@ Then apply it with:
 helm install vllm-llama3-8b-instruct ./config/charts/inferencepool -f values.yaml
 ```
 
+### Install with Monitoring
+
+To enable metrics collection and monitoring for the EndpointPicker, you can configure Prometheus ServiceMonitor creation:
+
+```yaml
+inferenceExtension:
+  monitoring:
+    interval: "10s"
+    prometheus:
+      enabled: true
+    secret:
+      name: inference-gateway-sa-metrics-reader-secret
+```
+
+**Note:** Prometheus monitoring requires the Prometheus Operator and ServiceMonitor CRD to be installed in the cluster.
+
+For GKE environments, monitoring is automatically configured when `provider.name` is set to `gke`.
+
+Then apply it with:
+
+```txt
+helm install vllm-llama3-8b-instruct ./config/charts/inferencepool -f values.yaml
+```
+
 ## Uninstall
 
 Run the following command to uninstall the chart:
@@ -147,6 +171,9 @@ The following table list the configurable parameters of the chart.
 | `inferenceExtension.affinity`               | Affinity for the endpoint picker. Defaults to `{}`.                                                                    |
 | `inferenceExtension.tolerations`            | Tolerations for the endpoint picker. Defaults to `[]`.                                                                 |
 | `inferenceExtension.flags.has-enable-leader-election` | Enable leader election for high availability. When enabled, only one EPP pod (the leader) will be ready to serve traffic.       |
+| `inferenceExtension.monitoring.interval`   | Metrics scraping interval for monitoring. Defaults to `10s`.                                                           |
+| `inferenceExtension.monitoring.secret.name` | Name of the service account token secret for metrics authentication. Defaults to `inference-gateway-sa-metrics-reader-secret`. |
+| `inferenceExtension.monitoring.prometheus.enabled` | Enable Prometheus ServiceMonitor creation for EPP metrics collection. Defaults to `false`.                      |
 | `inferenceExtension.pluginsCustomConfig`    | Custom config that is passed to EPP as inline yaml.      |
 | `provider.name`                             | Name of the Inference Gateway implementation being used. Possible values: `gke`. Defaults to `none`.                   |
 

config/charts/inferencepool/templates/epp-sa-token-secret.yaml

@@ -0,0 +1,12 @@
+{{- if or .Values.inferenceExtension.monitoring.prometheus.enabled .Values.inferenceExtension.monitoring.gke.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.inferenceExtension.monitoring.secret.name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "gateway-api-inference-extension.labels" . | nindent 4 }}
+  annotations:
+    kubernetes.io/service-account.name: {{ include "gateway-api-inference-extension.name" . }}
+type: kubernetes.io/service-account-token
+{{- end }}

config/charts/inferencepool/templates/epp-servicemonitor.yaml

@@ -0,0 +1,25 @@
+{{- if .Values.inferenceExtension.monitoring.prometheus.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "gateway-api-inference-extension.name" . }}-monitor
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "gateway-api-inference-extension.labels" . | nindent 4 }}
+spec:
+  endpoints:
+  - interval: {{ .Values.inferenceExtension.monitoring.interval }}
+    port: "http-metrics"
+    path: "/metrics"
+    authorization:
+      credentials:
+        key: token
+        name: {{ .Values.inferenceExtension.monitoring.secret.name }}
+  jobLabel: {{ include "gateway-api-inference-extension.name" . }}
+  namespaceSelector:
+    matchNames:
+    - {{ .Release.Namespace }}
+  selector:
+    matchLabels:
+      {{- include "gateway-api-inference-extension.labels" . | nindent 6 }}
+{{- end }}

config/charts/inferencepool/templates/gke.yaml

@@ -46,15 +46,15 @@ spec:
   endpoints:
   - port: metrics
     scheme: http
-    interval: 5s
+    interval: {{ .Values.inferenceExtension.monitoring.interval }}
     path: /metrics
     authorization:
       type: Bearer
       credentials:
         secret:
-          name: {{ .Values.gke.monitoringSecret.name }}
+          name: {{ .Values.inferenceExtension.monitoring.secret.name }}
           key: token
-          namespace: {{ .Values.gke.monitoringSecret.namespace }}
+          namespace: {{ .Release.Namespace }}
   selector:
     matchLabels:
       {{- include "gateway-api-inference-extension.selectorLabels" . | nindent 8 }}