Merge pull request #488 from ffromani/config-attributes

noderesourcetopology: overhaul Topology Manager configuration management

Author: k8s-ci-robot

go.mod

@@ -8,8 +8,8 @@ require (
 	github.com/dustin/go-humanize v1.0.0
 	github.com/go-logr/logr v1.2.3
 	github.com/google/go-cmp v0.5.8
-	github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.0.13
-	github.com/k8stopologyawareschedwg/podfingerprint v0.1.1
+	github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.1.0
+	github.com/k8stopologyawareschedwg/podfingerprint v0.1.2
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/paypal/load-watcher v0.2.2
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -321,10 +321,10 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.0.13 h1:Y1RjPskyGMkVtNL8lq75bEdjqgq8gi+JJ1oWaz/mIJE=
-github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.0.13/go.mod h1:AkACMQGiTgCt0lQw3m7TTU8PLH9lYKNK5e9DqFf5VuM=
-github.com/k8stopologyawareschedwg/podfingerprint v0.1.1 h1:uNEj+avp3yJkJMvkmk6iosibVauSo+owEKV2JyuKNsQ=
-github.com/k8stopologyawareschedwg/podfingerprint v0.1.1/go.mod h1:C23pM15t06dXg/OihGlqBvnYzLr+MXDXJ7zMfbNAyXI=
+github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.1.0 h1:2uCRJbv+A+fmaUaO0wLZ8oYd6cLE1dRzBQcFNxggH3s=
+github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.1.0/go.mod h1:AkACMQGiTgCt0lQw3m7TTU8PLH9lYKNK5e9DqFf5VuM=
+github.com/k8stopologyawareschedwg/podfingerprint v0.1.2 h1:Db5KLJjPg2mKaCoeEliMlea+JMyDMWdbNPXnWbPNDyM=
+github.com/k8stopologyawareschedwg/podfingerprint v0.1.2/go.mod h1:C23pM15t06dXg/OihGlqBvnYzLr+MXDXJ7zMfbNAyXI=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=

manifests/noderesourcetopology/crd.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    api-approved.kubernetes.io: https://github.com/kubernetes/enhancements/pull/1870 # edited manually
+    api-approved.kubernetes.io: https://github.com/kubernetes/enhancements/pull/1870
     controller-gen.kubebuilder.io/version: v0.11.1
   creationTimestamp: null
   name: noderesourcetopologies.topology.node.k8s.io
@@ -135,4 +135,137 @@ spec:
         - zones
         type: object
     served: true
+    storage: false
+  - name: v1alpha2
+    schema:
+      openAPIV3Schema:
+        description: NodeResourceTopology describes node resources and their topology.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          attributes:
+            description: AttributeList contains an array of AttributeInfo objects.
+            items:
+              description: AttributeInfo contains one attribute of a Zone.
+              properties:
+                name:
+                  type: string
+                value:
+                  type: string
+              required:
+              - name
+              - value
+              type: object
+            type: array
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          topologyPolicies:
+            description: 'DEPRECATED (to be removed in v1beta1): use top level attributes
+              if needed'
+            items:
+              type: string
+            type: array
+          zones:
+            description: ZoneList contains an array of Zone objects.
+            items:
+              description: Zone represents a resource topology zone, e.g. socket,
+                node, die or core.
+              properties:
+                attributes:
+                  description: AttributeList contains an array of AttributeInfo objects.
+                  items:
+                    description: AttributeInfo contains one attribute of a Zone.
+                    properties:
+                      name:
+                        type: string
+                      value:
+                        type: string
+                    required:
+                    - name
+                    - value
+                    type: object
+                  type: array
+                costs:
+                  description: CostList contains an array of CostInfo objects.
+                  items:
+                    description: CostInfo describes the cost (or distance) between
+                      two Zones.
+                    properties:
+                      name:
+                        type: string
+                      value:
+                        format: int64
+                        type: integer
+                    required:
+                    - name
+                    - value
+                    type: object
+                  type: array
+                name:
+                  type: string
+                parent:
+                  type: string
+                resources:
+                  description: ResourceInfoList contains an array of ResourceInfo
+                    objects.
+                  items:
+                    description: ResourceInfo contains information about one resource
+                      type.
+                    properties:
+                      allocatable:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        description: Allocatable quantity of the resource, corresponding
+                          to allocatable in node status, i.e. total amount of this
+                          resource available to be used by pods.
+                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                        x-kubernetes-int-or-string: true
+                      available:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        description: Available is the amount of this resource currently
+                          available for new (to be scheduled) pods, i.e. Allocatable
+                          minus the resources reserved by currently running pods.
+                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                        x-kubernetes-int-or-string: true
+                      capacity:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        description: Capacity of the resource, corresponding to capacity
+                          in node status, i.e. total amount of this resource that
+                          the node has.
+                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                        x-kubernetes-int-or-string: true
+                      name:
+                        description: Name of the resource.
+                        type: string
+                    required:
+                    - allocatable
+                    - available
+                    - capacity
+                    - name
+                    type: object
+                  type: array
+                type:
+                  type: string
+              required:
+              - name
+              - type
+              type: object
+            type: array
+        required:
+        - zones
+        type: object
+    served: true
     storage: true

pkg/noderesourcetopology/README.md

@@ -29,7 +29,7 @@ NodeResourceTopologyMatch plugin to work properly requires specific version of N
 
 | Scheduler Plugins | NodeResourceTopology CRD version |
 |-------------------|----------------------------------|
-| master            | v0.0.13                          |
+| master            | v0.1.0                           |
 | v0.24.9           | v0.0.12                          |
 | v0.23.10          | v0.0.12                          |
 | v0.22.6           | v0.0.12                          |
@@ -41,7 +41,7 @@ In case NodeResourceTopology CRD is being installed and advertised by [NFD](http
 
 | Scheduler Plugins | NodeResourceTopology CRD version | NFD version |
 |-------------------|----------------------------------|-------------|
-| master            | v0.0.13                          | master      |
+| master            | v0.1.0                           | master      |
 | v0.24.9           | v0.0.12                          | > v0.10.0   |
 | v0.23.10          | v0.0.12                          | > v0.10.0   |
 | v0.22.6           | v0.0.12                          | > v0.10.0   |
@@ -155,6 +155,27 @@ Should the cluster need to have different settings (e.g. topology manager) or NU
 using [affinity](https://kubernetes.io/docs/user-guide/node-selection/#node-affinity-beta-feature) or also
 [taints](https://kubernetes.io/docs/user-guide/node-selection/#taints-and-toleations-beta-feature).
 
+#### Topology Manager configuration
+
+***Target audience: developers and operators of topology updaters (NodeResourceTopology producers)***
+
+In addition to logically partitioning a cluster like explained above, the topology-aware scheduler needs to know key node-specific configuration settings like Topology manager policy and scope.
+This data is expected to be provided as top-level `Attributes` of the NodeResourceTopology objects:
+
+NodeResourceTopology producers should add top-level `Attributes` in the following format
+- For `Name` and `Value` of attributes, words should be `snakeCase`
+- The `Name` of each attribute should be **the same of the corresponding kubelet configuration option**.
+  - example: `--topology-manager-scope` becomes `topologyManagerScope`
+  - example: `topologyManagerPolicy` becomes `topologyManagerPolicy`
+- The `Value` of each attribute should be **one of the value of the corresponding kubelet configuration option, VERBATIM**.
+  - example: `single-numa-node` becomes `single-numa-node`
+- Should `topologyManagerOptions` be exposed:
+  - they should be expanded in key-value pairs, using the `String()` representation
+  - each key-value pair should be preceded by the `topologyManagerOption` prefix
+  - every other provision described above applies
+  - example: the `prefer-closest-numa-nodes` option becomes `topologyManagerOptionPreferClosestNumaNodes`, accepting exactly one of either `true` and `false`.
+  - **RATIONALE**: this representation wants to guarantee all the Attribute Names are unique (no aliasing). It must be noted this is a stricter requirement with respect to the Attribute representation
+    in NRT objects, and this requirement could be lifted in the future (an upgrade path will be provided).
 
 ### Demo
 
@@ -168,7 +189,7 @@ For configuring your cluster with [NFD-topology updater](https://github.com/kube
 
 ```yaml
 # Worker Node A CRD spec
-apiVersion: topology.node.k8s.io/v1alpha1
+apiVersion: topology.node.k8s.io/v1alpha2
 kind: NodeResourceTopology
 metadata:
   name: worker-node-A
@@ -202,7 +223,7 @@ zones:
 
 ```yaml
 # Worker Node B CRD spec
-apiVersion: topology.node.k8s.io/v1alpha1
+apiVersion: topology.node.k8s.io/v1alpha2
 kind: NodeResourceTopology
 metadata:
   name: worker-node-B

pkg/noderesourcetopology/cache/cache.go

@@ -19,7 +19,7 @@ package cache
 import (
 	corev1 "k8s.io/api/core/v1"
 
-	topologyv1alpha1 "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha1"
+	topologyv1alpha2 "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha2"
 )
 
 type Interface interface {
@@ -30,7 +30,7 @@ type Interface interface {
 	// The pod argument is used only for logging purposes.
 	// Returns a boolean to signal the caller if the NRT data is clean. If false, then the node has foreign
 	// Pods detected - so it should be ignored or handled differently by the caller.
-	GetCachedNRTCopy(nodeName string, pod *corev1.Pod) (*topologyv1alpha1.NodeResourceTopology, bool)
+	GetCachedNRTCopy(nodeName string, pod *corev1.Pod) (*topologyv1alpha2.NodeResourceTopology, bool)
 
 	// NodeMaybeOverReserved declares a node was filtered out for not enough resources available.
 	// This means this node is eligible for a resync. When a node is marked discarded (dirty), it matters not

pkg/noderesourcetopology/cache/overreserve.go

@@ -28,8 +28,8 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/pkg/scheduler/framework"
 
-	topologyv1alpha1 "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha1"
-	listerv1alpha1 "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/generated/listers/topology/v1alpha1"
+	topologyv1alpha2 "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha2"
+	listerv1alpha2 "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/generated/listers/topology/v1alpha2"
 
 	"sigs.k8s.io/scheduler-plugins/pkg/noderesourcetopology/stringify"
 
@@ -44,11 +44,11 @@ type OverReserve struct {
 	// to resync nodes. See The documentation of Resync() below for more details.
 	nodesMaybeOverreserved counter
 	nodesWithForeignPods   counter
-	nrtLister              listerv1alpha1.NodeResourceTopologyLister
+	nrtLister              listerv1alpha2.NodeResourceTopologyLister
 	nodeIndexer            NodeIndexer
 }
 
-func NewOverReserve(lister listerv1alpha1.NodeResourceTopologyLister, indexer NodeIndexer) (*OverReserve, error) {
+func NewOverReserve(lister listerv1alpha2.NodeResourceTopologyLister, indexer NodeIndexer) (*OverReserve, error) {
 	if lister == nil || indexer == nil {
 		return nil, fmt.Errorf("nrtcache: received nil references")
 	}
@@ -70,7 +70,7 @@ func NewOverReserve(lister listerv1alpha1.NodeResourceTopologyLister, indexer No
 	return obj, nil
 }
 
-func (ov *OverReserve) GetCachedNRTCopy(nodeName string, pod *corev1.Pod) (*topologyv1alpha1.NodeResourceTopology, bool) {
+func (ov *OverReserve) GetCachedNRTCopy(nodeName string, pod *corev1.Pod) (*topologyv1alpha2.NodeResourceTopology, bool) {
 	ov.lock.Lock()
 	defer ov.lock.Unlock()
 	if ov.nodesWithForeignPods.IsSet(nodeName) {
@@ -197,7 +197,7 @@ func (ov *OverReserve) Resync() {
 	klog.V(6).InfoS("nrtcache: resync NodeTopology cache starting", "logID", logID)
 	defer klog.V(6).InfoS("nrtcache: resync NodeTopology cache complete", "logID", logID)
 
-	var nrtUpdates []*topologyv1alpha1.NodeResourceTopology
+	var nrtUpdates []*topologyv1alpha2.NodeResourceTopology
 	for _, nodeName := range nodeNames {
 		nrtCandidate, err := ov.nrtLister.Get(nodeName)
 		if err != nil {
@@ -237,7 +237,7 @@ func (ov *OverReserve) Resync() {
 }
 
 // FlushNodes drops all the cached information about a given node, resetting its state clean.
-func (ov *OverReserve) FlushNodes(logID string, nrts ...*topologyv1alpha1.NodeResourceTopology) {
+func (ov *OverReserve) FlushNodes(logID string, nrts ...*topologyv1alpha2.NodeResourceTopology) {
 	ov.lock.Lock()
 	defer ov.lock.Unlock()
 	for _, nrt := range nrts {