KVCache and Intelligent Routing for HyperPod Inference (#290)

* feat: KVCache and Intelligent Routing for HyperPod Inference

* Initial commit for kv-cache and intelligent support in inference SDK/CLI (#252)

* Initial commit for kv-cache and intelligent support in inference SDK

* Initial commit for kv-cache & intelligent routing CLI changes

* [Fix] clean up test code

* Separate out kv-cache and intelligent routing spec update into v1_1 inference cli

* Fix ut and black format

* Fix template.py extra white lines

* fix: update inference template.py to reflect init experience template submission changes

---------

Co-authored-by: jiapinw <[email protected]>

Author: rvasahu-amazon

helm_chart/HyperPodHelmChart/charts/inference-operator/Chart.lock

@@ -13,9 +13,9 @@ dependencies:
   version: 1.13.2
 - name: cert-manager
   repository: https://charts.jetstack.io
-  version: v1.18.0
+  version: v1.18.2
 - name: keda
   repository: https://kedacore.github.io/charts
   version: 2.17.1
-digest: sha256:5f877809dfd7c4d13b13f3de92e0824c28f80ed3abcf7c54f11764d9aeabbeba
-generated: "2025-06-19T22:21:36.075156362Z"
+digest: sha256:f54ece80a00cb4da98440551765d9c660a0704d6b59f4f9030a5a9e86eab4eea
+generated: "2025-10-27T17:20:29.746399171Z"

helm_chart/HyperPodHelmChart/charts/inference-operator/Chart.yaml

@@ -15,13 +15,13 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 1.0.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.16.0"
+appVersion: "2.0"
 
 dependencies:
 - name: aws-mountpoint-s3-csi-driver
@@ -45,7 +45,7 @@ dependencies:
   condition: alb.enabled
 - name: cert-manager
   alias: cert-manager
-  version: v1.18.0
+  version: v1.18.2
   repository: "https://charts.jetstack.io"
   condition: cert-manager.enabled
 - name: keda

helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_inferenceendpointconfigs.yaml

@@ -1,11 +1,21 @@
----
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
+    cert-manager.io/inject-ca-from: '{{ .Values.shortPrefix }}-system/serving-cert'
     controller-gen.kubebuilder.io/version: v0.16.4
   name: sagemakerendpointregistrations.inference.sagemaker.aws.amazon.com
 spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: '{{ .Values.namePrefix }}-conversion-webhook'
+          namespace: '{{ .Values.shortPrefix }}-system'
+          path: /convert
+      conversionReviewVersions:
+      - v1
   group: inference.sagemaker.aws.amazon.com
   names:
     kind: SageMakerEndpointRegistration
@@ -14,7 +24,7 @@ spec:
     singular: sagemakerendpointregistration
   scope: Namespaced
   versions:
-  - name: v1alpha1
+  - name: v1
     schema:
       openAPIV3Schema:
         description: SageMakerEndpointRegistration is the Schema for the sagemakerendpointregistrations
@@ -88,6 +98,10 @@ spec:
                 description: InstanceType is the ML compute instance type used for
                   EndpointConfig creation
                 type: string
+              invocationEndpoint:
+                default: invocations
+                description: The invocation endpoint path used by the model server
+                type: string
               loadBalancerHostName:
                 description: Needed to embed the LB Host Name
                 type: string
@@ -248,3 +262,241 @@ spec:
     storage: true
     subresources:
       status: {}
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: SageMakerEndpointRegistration is the Schema for the sagemakerendpointregistrations
+          API
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: SageMakerEndpointRegistrationSpec defines the desired state
+              of SageMakerEndpointRegistration
+            properties:
+              eksClusterDetails:
+                properties:
+                  arn:
+                    description: Stores cluster ARN
+                    type: string
+                  clusterSecurityGroupId:
+                    description: Stores ClusterSecurityGroup of the EKS Cluster
+                    type: string
+                  name:
+                    description: Stores cluster name
+                    type: string
+                  securityGroupIds:
+                    description: Stores AdditionalSecurityGroupIds of the EKS Cluster
+                    items:
+                      type: string
+                    type: array
+                  subnetIds:
+                    description: Stores SubnetIDs of the EKS Cluster
+                    items:
+                      type: string
+                    type: array
+                  vpcId:
+                    description: Stores VPC Id of the EKS Cluster
+                    type: string
+                required:
+                - arn
+                - clusterSecurityGroupId
+                - name
+                - securityGroupIds
+                - subnetIds
+                - vpcId
+                type: object
+              executionRole:
+                description: The Amazon Resource Name (ARN) of an IAM role that will
+                  be used to create model, endpoint config, and the endpoint
+                maxLength: 2048
+                minLength: 20
+                pattern: ^arn:aws[a-z\-]*:iam::\d{12}:role/?[a-zA-Z_0-9+=,.@\-_/]+$
+                type: string
+              imageUri:
+                description: The ImageUri where inference code is stored
+                maxLength: 255
+                type: string
+              instanceType:
+                description: InstanceType is the ML compute instance type used for
+                  EndpointConfig creation
+                type: string
+              invocationEndpoint:
+                default: invocations
+                description: The invocation endpoint path used by the model server
+                type: string
+              loadBalancerHostName:
+                description: Needed to embed the LB Host Name
+                type: string
+              name:
+                description: Name used for AWS resource creation
+                maxLength: 63
+                pattern: ^[a-zA-Z0-9](-*[a-zA-Z0-9]){0,62}
+                type: string
+              restApiId:
+                description: REST API Gateway identifier that proxies requests to
+                  the HyperPod endpoint (via NLB/ALB)
+                type: string
+              tlsConfig:
+                properties:
+                  tlsCertificateOutputS3Bucket:
+                    description: S3 bucket that stores the certificate that needs
+                      to be trusted
+                    type: string
+                  tlsCertificateS3Keys:
+                    description: The output tls certificate S3 key that points to
+                      the .pem file
+                    items:
+                      type: string
+                    type: array
+                  tlsServerNameOverride:
+                    description: The server name override for tls certificate selection
+                    type: string
+                required:
+                - tlsCertificateOutputS3Bucket
+                - tlsCertificateS3Keys
+                type: object
+            required:
+            - eksClusterDetails
+            - executionRole
+            - imageUri
+            - instanceType
+            - loadBalancerHostName
+            - name
+            - restApiId
+            - tlsConfig
+            type: object
+          status:
+            description: SageMakerEndpointRegistrationStatus defines the observed
+              state of SageMakerEndpointRegistration
+            properties:
+              conditions:
+                description: Detailed conditions representing the state of the deployment
+                items:
+                  description: Condition contains details for one aspect of the current
+                    state of this API Resource.
+                  properties:
+                    lastTransitionTime:
+                      description: |-
+                        lastTransitionTime is the last time the condition transitioned from one status to another.
+                        This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: |-
+                        message is a human readable message indicating details about the transition.
+                        This may be an empty string.
+                      maxLength: 32768
+                      type: string
+                    observedGeneration:
+                      description: |-
+                        observedGeneration represents the .metadata.generation that the condition was set based upon.
+                        For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                        with respect to the current state of the instance.
+                      format: int64
+                      minimum: 0
+                      type: integer
+                    reason:
+                      description: |-
+                        reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                        Producers of specific condition types may define expected values and meanings for this field,
+                        and whether the values are considered a guaranteed API.
+                        The value should be a CamelCase string.
+                        This field may not be empty.
+                      maxLength: 1024
+                      minLength: 1
+                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      type: string
+                    status:
+                      description: status of the condition, one of True, False, Unknown.
+                      enum:
+                      - "True"
+                      - "False"
+                      - Unknown
+                      type: string
+                    type:
+                      description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                      maxLength: 316
+                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - message
+                  - reason
+                  - status
+                  - type
+                  type: object
+                type: array
+              endpoint:
+                description: Endpoint Metadata
+                properties:
+                  arn:
+                    description: The Amazon Resource Name (ARN) of the SageMaker endpoint
+                    pattern: (arn:aws[a-z\-]*:sagemaker:[a-z0-9\-]*:[0-9]{12}:endpoint/.*|^$)
+                    type: string
+                  configArn:
+                    description: The Amazon Resource Name (ARN) of the endpoint configuration.
+                    pattern: (arn:aws[a-z\-]*:sagemaker:[a-z0-9\-]*:[0-9]{12}:endpoint-config/.*|^$)
+                    type: string
+                  lastModifiedTime:
+                    description: The last modified time of SageMaker endpoint.
+                    format: date-time
+                    type: string
+                  modelArn:
+                    description: The ARN of the model created in SageMaker.
+                    pattern: (arn:aws[a-z\-]*:sagemaker:[a-z0-9\-]*:[0-9]{12}:model/.*|^$)
+                    type: string
+                required:
+                - arn
+                - configArn
+                - modelArn
+                type: object
+              loadBalancer:
+                description: LoadBalancer Metadata
+                properties:
+                  hostName:
+                    description: Hostname of LoadBalancer
+                    type: string
+                required:
+                - hostName
+                type: object
+              observedGeneration:
+                description: Latest generation reconciled by controller
+                format: int64
+                type: integer
+              state:
+                description: Current phase of the Endpoint creation Step
+                enum:
+                - CreationInProgress
+                - CreationFailed
+                - CreationCompleted
+                - DeletionInProgress
+                - DeletionFailed
+                - DeletionCompleted
+                - UpdateInProgress
+                - UpdateFailed
+                - UpdateCompleted
+                type: string
+            required:
+            - state
+            type: object
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}

helm_chart/HyperPodHelmChart/charts/inference-operator/config/crd/inference.sagemaker.aws.amazon.com_jumpstartmodels.yaml

@@ -55,9 +55,14 @@ spec:
           - --metrics-bind-address=:8443
           - --leader-elect
           - --health-probe-bind-address=:8081
+          - --webhook-cert-path=/tmp/k8s-webhook-server/serving-certs
         image: "{{ .Values.image.repository }}/hyperpod-inference-operator:{{ .Values.image.tag }}"
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         name: manager
+        ports:
+        - containerPort: 9443
+          name: webhook-server
+          protocol: TCP
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:
@@ -73,8 +78,10 @@ spec:
           httpGet:
             path: /healthz
             port: 8081
+          initialDelaySeconds: 180
           failureThreshold: 120
           periodSeconds: 60
+          timeoutSeconds: 5
         readinessProbe:
           httpGet:
             path: /readyz
@@ -90,6 +97,10 @@ spec:
           requests:
             cpu: 10m
             memory: 64Mi
+        volumeMounts:
+        - mountPath: /tmp/k8s-webhook-server/serving-certs
+          name: webhook-certs
+          readOnly: true
         env:
           - name: HYPERPOD_CLUSTER_ARN
             value: {{ .Values.hyperpodClusterArn }}
@@ -103,5 +114,15 @@ spec:
             value: {{ .Values.eksClusterName }}
           - name: TLS_CERTIFICATE_OUTPUT_S3URI
             value: {{ .Values.tlsCertificateS3Bucket }}
+          - name: ENABLE_WEBHOOKS
+            value: "{{ .Values.enableWebhooks }}"
+          - name: CHART_VERSION
+            value: {{ .Chart.Version | quote }}
+          - name: APP_VERSION
+            value: {{ .Chart.AppVersion | quote }}
       serviceAccountName: {{ .Values.namePrefix }}-controller-manager
       terminationGracePeriodSeconds: 10
+      volumes:
+      - name: webhook-certs
+        secret:
+          secretName: webhook-server-cert