Skip to content

Commit bf1ddce

Browse files
zetxqxzetxqx
committed
more efficient rewrite fetching per request.
1 parent 3e9939f commit bf1ddce

File tree

9 files changed

+535
-163
lines changed

9 files changed

+535
-163
lines changed

‎apix/v1alpha2/inferencemodelrewrite_types.go‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -57,20 +57,25 @@ type InferenceModelRewriteSpec struct {
5757
// If multiple InferenceModelRewrite resources target the same
5858
// InferencePool, the controller will merge them based on precedence.
5959
//
60-
// **Timestamp Wins:** If two rules from different rewrites all matches,
61-
// the rule from the *oldest*
62-
// InferenceModelRewrite resource (determined by
63-
// metadata.creationTimestamp) will be used.
60+
// Across all rules specified on applicable rewrites, precedence MUST be
61+
// given to the match having an "Exact" model match over a generic match
62+
// (a rule with an empty `matches` array).
63+
//
64+
// If ties still exist across multiple InferenceModelRewrite resources (e.g.
65+
// two rewrites both have an exact match for the same model), matching
66+
// precedence MUST be determined by the oldest resource based on
67+
// creation timestamp.
68+
//
69+
// If ties still exist within a single InferenceModelRewrite resource, the
70+
// FIRST matching rule (in list order) is used.
6471
// +required
6572
Rules []InferenceModelRewriteRule `json:"rules"`
6673
}
6774

6875
// InferenceModelRewriteRule defines the match criteria and corresponding action.
69-
//
70-
// A specific model name can only be matched by one rule across all
71-
// rules attached to the same InferencePool. If multiple rules attempt
72-
// to match the same model name, the oldest rule (by creationTimestamp)
73-
// will be the only one considered valid.
76+
// For details on how precedence is determined across multiple rules and
77+
// InferenceModelRewrite resources, see the "Precedence and Conflict Resolution"
78+
// section in InferenceModelRewriteSpec.
7479
type InferenceModelRewriteRule struct {
7580
// Matches defines the criteria for matching a request.
7681
// If multiple match criteria are specified, a request matches if

‎config/crd/bases/inference.networking.x-k8s.io_inferencemodelrewrites.yaml‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -74,11 +74,9 @@ spec:
7474
items:
7575
description: |-
7676
InferenceModelRewriteRule defines the match criteria and corresponding action.
77-
78-
A specific model name can only be matched by one rule across all
79-
rules attached to the same InferencePool. If multiple rules attempt
80-
to match the same model name, the oldest rule (by creationTimestamp)
81-
will be the only one considered valid.
77+
For details on how precedence is determined across multiple rules and
78+
InferenceModelRewrite resources, see the "Precedence and Conflict Resolution"
79+
section in InferenceModelRewriteSpec.
8280
properties:
8381
matches:
8482
items:

‎docs/proposals/1816-inferenceomodelrewrite/README.md‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -64,20 +64,25 @@ type InferenceModelRewriteSpec struct {
6464
// If multiple InferenceModelRewrite resources target the same
6565
// InferencePool, the controller will merge them based on precedence.
6666
//
67-
// **Timestamp Wins:** If two rules from different rewrite all matches,
68-
// the rule from the *oldest*
69-
// InferenceModelRewrite resource (determined by
70-
// metadata.creationTimestamp) will be used.
67+
// Across all rules specified on applicable rewrites, precedence MUST be
68+
// given to the match having an "Exact" model match over a generic match
69+
// (a rule with an empty `matches` array).
70+
//
71+
// If ties still exist across multiple InferenceModelRewrite resources (e.g.
72+
// two rewrites both have an exact match for the same model), matching
73+
// precedence MUST be determined by the oldest resource based on
74+
// creation timestamp.
75+
//
76+
// If ties still exist within a single InferenceModelRewrite resource, the
77+
// FIRST matching rule (in list order) is used.
7178
// +required
7279
Rules []InferenceModelRewriteRule `json:"rules"`
7380
}
7481

7582
// InferenceModelRewriteRule defines the match criteria and corresponding action.
76-
//
77-
// A specific model name can only be matched by one rule across all
78-
// rewrites attached to the same InferencePool. If multiple rules attempt
79-
// to match the same model name, the oldest rule (by creationTimestamp)
80-
// will be the only one considered valid.
83+
// For details on how precedence is determined across multiple rules and
84+
// InferenceModelRewrite resources, see the "Precedence and Conflict Resolution"
85+
// section in InferenceModelRewriteSpec.
8186
type InferenceModelRewriteRule struct {
8287
// Matches defines the criteria for matching a request.
8388
// If multiple match criteria are specified, a request matches if

‎pkg/epp/controller/inferencemodelrewrite_reconciler_test.go‎

Lines changed: 52 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -41,79 +41,59 @@ import (
4141

4242
var (
4343
poolForRewrite = utiltest.MakeInferencePool("test-pool1").Namespace("ns1").ObjRef()
44-
rewrite1 = makeInferenceModelRewrite("rewrite1").
45-
Namespace(poolForRewrite.Namespace).
46-
PoolName(poolForRewrite.Name).
47-
CreationTimestamp(metav1.Unix(1000, 0)).
48-
ObjRef()
49-
rewrite1Pool2 = makeInferenceModelRewrite(rewrite1.Name).
50-
Namespace(rewrite1.Namespace).
51-
PoolName("test-pool2").
52-
CreationTimestamp(metav1.Unix(1001, 0)).
53-
ObjRef()
54-
rewrite1Updated = makeInferenceModelRewrite(rewrite1.Name).
55-
Namespace(rewrite1.Namespace).
56-
PoolName(poolForRewrite.Name).
57-
CreationTimestamp(metav1.Unix(1003, 0)).
58-
Rules([]v1alpha2.InferenceModelRewriteRule{{}}).
59-
ObjRef()
60-
rewrite1Deleted = makeInferenceModelRewrite(rewrite1.Name).
61-
Namespace(rewrite1.Namespace).
62-
PoolName(poolForRewrite.Name).
63-
CreationTimestamp(metav1.Unix(1004, 0)).
64-
DeletionTimestamp().
65-
ObjRef()
66-
rewrite2 = makeInferenceModelRewrite("rewrite2").
67-
Namespace(poolForRewrite.Namespace).
68-
PoolName(poolForRewrite.Name).
69-
CreationTimestamp(metav1.Unix(1000, 0)).
70-
ObjRef()
71-
)
72-
73-
type inferenceModelRewriteBuilder struct {
74-
*v1alpha2.InferenceModelRewrite
75-
}
76-
77-
func makeInferenceModelRewrite(name string) *inferenceModelRewriteBuilder {
78-
return &inferenceModelRewriteBuilder{
79-
&v1alpha2.InferenceModelRewrite{
80-
ObjectMeta: metav1.ObjectMeta{
81-
Name: name,
82-
},
44+
rewrite1 = &v1alpha2.InferenceModelRewrite{
45+
ObjectMeta: metav1.ObjectMeta{
46+
Name: "rewrite1",
47+
Namespace: poolForRewrite.Namespace,
48+
CreationTimestamp: metav1.Unix(1000, 0),
49+
},
50+
Spec: v1alpha2.InferenceModelRewriteSpec{
51+
PoolRef: &v1alpha2.PoolObjectReference{Name: v1alpha2.ObjectName(poolForRewrite.Name)},
8352
},
8453
}
85-
}
86-
87-
func (b *inferenceModelRewriteBuilder) Namespace(ns string) *inferenceModelRewriteBuilder {
88-
b.ObjectMeta.Namespace = ns
89-
return b
90-
}
91-
92-
func (b *inferenceModelRewriteBuilder) PoolName(name string) *inferenceModelRewriteBuilder {
93-
b.Spec.PoolRef = &v1alpha2.PoolObjectReference{}
94-
b.Spec.PoolRef.Name = v1alpha2.ObjectName(name)
95-
return b
96-
}
97-
98-
func (b *inferenceModelRewriteBuilder) CreationTimestamp(t metav1.Time) *inferenceModelRewriteBuilder {
99-
b.ObjectMeta.CreationTimestamp = t
100-
return b
101-
}
102-
103-
func (b *inferenceModelRewriteBuilder) DeletionTimestamp() *inferenceModelRewriteBuilder {
104-
now := metav1.Now()
105-
b.ObjectMeta.DeletionTimestamp = &now
106-
return b
107-
}
108-
109-
func (b *inferenceModelRewriteBuilder) Rules(rules []v1alpha2.InferenceModelRewriteRule) *inferenceModelRewriteBuilder {
110-
b.Spec.Rules = rules
111-
return b
112-
}
113-
114-
func (b *inferenceModelRewriteBuilder) ObjRef() *v1alpha2.InferenceModelRewrite {
115-
return b.InferenceModelRewrite
116-
}
54+
rewrite1Pool2 = &v1alpha2.InferenceModelRewrite{
55+
ObjectMeta: metav1.ObjectMeta{
56+
Name: rewrite1.Name,
57+
Namespace: rewrite1.Namespace,
58+
CreationTimestamp: metav1.Unix(1001, 0),
59+
},
60+
Spec: v1alpha2.InferenceModelRewriteSpec{
61+
PoolRef: &v1alpha2.PoolObjectReference{Name: "test-pool2"},
62+
},
63+
}
64+
rewrite1Updated = &v1alpha2.InferenceModelRewrite{
65+
ObjectMeta: metav1.ObjectMeta{
66+
Name: rewrite1.Name,
67+
Namespace: rewrite1.Namespace,
68+
CreationTimestamp: metav1.Unix(1003, 0),
69+
},
70+
Spec: v1alpha2.InferenceModelRewriteSpec{
71+
PoolRef: &v1alpha2.PoolObjectReference{Name: v1alpha2.ObjectName(poolForRewrite.Name)},
72+
Rules: []v1alpha2.InferenceModelRewriteRule{{}},
73+
},
74+
}
75+
rewrite1Deleted = &v1alpha2.InferenceModelRewrite{
76+
ObjectMeta: metav1.ObjectMeta{
77+
Name: rewrite1.Name,
78+
Namespace: rewrite1.Namespace,
79+
CreationTimestamp: metav1.Unix(1004, 0),
80+
DeletionTimestamp: &metav1.Time{Time: time.Now()},
81+
},
82+
Spec: v1alpha2.InferenceModelRewriteSpec{
83+
PoolRef: &v1alpha2.PoolObjectReference{Name: v1alpha2.ObjectName(poolForRewrite.Name)},
84+
},
85+
}
86+
rewrite2 = &v1alpha2.InferenceModelRewrite{
87+
ObjectMeta: metav1.ObjectMeta{
88+
Name: "rewrite2",
89+
Namespace: poolForRewrite.Namespace,
90+
CreationTimestamp: metav1.Unix(1001, 0),
91+
},
92+
Spec: v1alpha2.InferenceModelRewriteSpec{
93+
PoolRef: &v1alpha2.PoolObjectReference{Name: v1alpha2.ObjectName(poolForRewrite.Name)},
94+
},
95+
}
96+
)
11797

11898
func TestInferenceModelRewriteReconciler(t *testing.T) {
11999
tests := []struct {
@@ -233,4 +213,4 @@ func diffStoreRewrites(ds datastore.Datastore, wantRewrites []*v1alpha2.Inferenc
233213
return "rewrites:" + diff
234214
}
235215
return ""
236-
}
216+
}

0 commit comments

Comments
 (0)