Skip to content

Commit 91549f5

Browse files
irar2irar2
authored
Code reorganization: moved configuration related code to common (#109)
* Moved configuration related code to common Signed-off-by: Ira <[email protected]> * Made NewConfig private and changed simulator test Signed-off-by: Ira <[email protected]> --------- Signed-off-by: Ira <[email protected]>
1 parent 67b90e0 commit 91549f5

File tree

6 files changed

+166
-168
lines changed

6 files changed

+166
-168
lines changed

‎pkg/common/common_suite_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package common_test
1+
package common
22

33
import (
44
"testing"

‎pkg/common/config.go‎

Lines changed: 122 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,15 @@ package common
1919
import (
2020
"encoding/json"
2121
"errors"
22+
"flag"
2223
"fmt"
2324
"os"
2425
"strings"
2526
"time"
2627

28+
"github.com/spf13/pflag"
2729
"gopkg.in/yaml.v3"
30+
"k8s.io/klog/v2"
2831
)
2932

3033
const (
@@ -119,24 +122,24 @@ type LoraModule struct {
119122
}
120123

121124
// Needed to parse values that contain multiple strings
122-
type MultiString struct {
123-
Values []string
125+
type multiString struct {
126+
values []string
124127
}
125128

126-
func (l *MultiString) String() string {
127-
return strings.Join(l.Values, " ")
129+
func (l *multiString) String() string {
130+
return strings.Join(l.values, " ")
128131
}
129132

130-
func (l *MultiString) Set(val string) error {
131-
l.Values = append(l.Values, val)
133+
func (l *multiString) Set(val string) error {
134+
l.values = append(l.values, val)
132135
return nil
133136
}
134137

135-
func (l *MultiString) Type() string {
138+
func (l *multiString) Type() string {
136139
return "strings"
137140
}
138141

139-
func (c *Configuration) UnmarshalLoras() error {
142+
func (c *Configuration) unmarshalLoras() error {
140143
c.LoraModules = make([]LoraModule, 0)
141144
for _, jsonStr := range c.LoraModulesString {
142145
var lora LoraModule
@@ -148,7 +151,7 @@ func (c *Configuration) UnmarshalLoras() error {
148151
return nil
149152
}
150153

151-
func NewConfig() *Configuration {
154+
func newConfig() *Configuration {
152155
return &Configuration{
153156
Port: vLLMDefaultPort,
154157
MaxLoras: 1,
@@ -165,7 +168,7 @@ func NewConfig() *Configuration {
165168
}
166169
}
167170

168-
func (c *Configuration) Load(configFile string) error {
171+
func (c *Configuration) load(configFile string) error {
169172
configBytes, err := os.ReadFile(configFile)
170173
if err != nil {
171174
return fmt.Errorf("failed to read configuration file: %s", err)
@@ -175,10 +178,10 @@ func (c *Configuration) Load(configFile string) error {
175178
return fmt.Errorf("failed to unmarshal configuration: %s", err)
176179
}
177180

178-
return c.UnmarshalLoras()
181+
return c.unmarshalLoras()
179182
}
180183

181-
func (c *Configuration) Validate() error {
184+
func (c *Configuration) validate() error {
182185
if c.Model == "" {
183186
return errors.New("model parameter is empty")
184187
}
@@ -265,3 +268,110 @@ func (c *Configuration) Validate() error {
265268
}
266269
return nil
267270
}
271+
272+
// ParseCommandParamsAndLoadConfig loads configuration, parses command line parameters, merges the values
273+
// (command line values overwrite the config file ones), and validates the configuration
274+
func ParseCommandParamsAndLoadConfig() (*Configuration, error) {
275+
config := newConfig()
276+
277+
configFileValues := getParamValueFromArgs("config")
278+
if len(configFileValues) == 1 {
279+
if err := config.load(configFileValues[0]); err != nil {
280+
return nil, err
281+
}
282+
}
283+
284+
servedModelNames := getParamValueFromArgs("served-model-name")
285+
loraModuleNames := getParamValueFromArgs("lora-modules")
286+
287+
f := pflag.NewFlagSet("llm-d-inference-sim flags", pflag.ContinueOnError)
288+
289+
f.IntVar(&config.Port, "port", config.Port, "Port")
290+
f.StringVar(&config.Model, "model", config.Model, "Currently 'loaded' model")
291+
f.IntVar(&config.MaxNumSeqs, "max-num-seqs", config.MaxNumSeqs, "Maximum number of inference requests that could be processed at the same time (parameter to simulate requests waiting queue)")
292+
f.IntVar(&config.MaxLoras, "max-loras", config.MaxLoras, "Maximum number of LoRAs in a single batch")
293+
f.IntVar(&config.MaxCPULoras, "max-cpu-loras", config.MaxCPULoras, "Maximum number of LoRAs to store in CPU memory")
294+
f.IntVar(&config.MaxModelLen, "max-model-len", config.MaxModelLen, "Model's context window, maximum number of tokens in a single request including input and output")
295+
296+
f.StringVar(&config.Mode, "mode", config.Mode, "Simulator mode, echo - returns the same text that was sent in the request, for chat completion returns the last message, random - returns random sentence from a bank of pre-defined sentences")
297+
f.IntVar(&config.InterTokenLatency, "inter-token-latency", config.InterTokenLatency, "Time to generate one token (in milliseconds)")
298+
f.IntVar(&config.TimeToFirstToken, "time-to-first-token", config.TimeToFirstToken, "Time to first token (in milliseconds)")
299+
f.IntVar(&config.KVCacheTransferLatency, "kv-cache-transfer-latency", config.KVCacheTransferLatency, "Time for KV-cache transfer from a remote vLLM (in milliseconds)")
300+
f.IntVar(&config.InterTokenLatencyStdDev, "inter-token-latency-std-dev", config.InterTokenLatencyStdDev, "Standard deviation for time between generated tokens (in milliseconds)")
301+
f.IntVar(&config.TimeToFirstTokenStdDev, "time-to-first-token-std-dev", config.TimeToFirstTokenStdDev, "Standard deviation for time before the first token will be returned (in milliseconds)")
302+
f.IntVar(&config.KVCacheTransferLatencyStdDev, "kv-cache-transfer-latency-std-dev", config.KVCacheTransferLatencyStdDev, "Standard deviation for time for KV-cache transfer from a remote vLLM (in milliseconds)")
303+
f.Int64Var(&config.Seed, "seed", config.Seed, "Random seed for operations (if not set, current Unix time in nanoseconds is used)")
304+
305+
f.IntVar(&config.MaxToolCallIntegerParam, "max-tool-call-integer-param", config.MaxToolCallIntegerParam, "Maximum possible value of integer parameters in a tool call")
306+
f.IntVar(&config.MinToolCallIntegerParam, "min-tool-call-integer-param", config.MinToolCallIntegerParam, "Minimum possible value of integer parameters in a tool call")
307+
f.Float64Var(&config.MaxToolCallNumberParam, "max-tool-call-number-param", config.MaxToolCallNumberParam, "Maximum possible value of number (float) parameters in a tool call")
308+
f.Float64Var(&config.MinToolCallNumberParam, "min-tool-call-number-param", config.MinToolCallNumberParam, "Minimum possible value of number (float) parameters in a tool call")
309+
f.IntVar(&config.MaxToolCallArrayParamLength, "max-tool-call-array-param-length", config.MaxToolCallArrayParamLength, "Maximum possible length of array parameters in a tool call")
310+
f.IntVar(&config.MinToolCallArrayParamLength, "min-tool-call-array-param-length", config.MinToolCallArrayParamLength, "Minimum possible length of array parameters in a tool call")
311+
f.IntVar(&config.ToolCallNotRequiredParamProbability, "tool-call-not-required-param-probability", config.ToolCallNotRequiredParamProbability, "Probability to add a parameter, that is not required, in a tool call")
312+
f.IntVar(&config.ObjectToolCallNotRequiredParamProbability, "object-tool-call-not-required-field-probability", config.ObjectToolCallNotRequiredParamProbability, "Probability to add a field, that is not required, in an object in a tool call")
313+
314+
// These values were manually parsed above in getParamValueFromArgs, we leave this in order to get these flags in --help
315+
var dummyString string
316+
f.StringVar(&dummyString, "config", "", "The path to a yaml configuration file. The command line values overwrite the configuration file values")
317+
var dummyMultiString multiString
318+
f.Var(&dummyMultiString, "served-model-name", "Model names exposed by the API (a list of space-separated strings)")
319+
f.Var(&dummyMultiString, "lora-modules", "List of LoRA adapters (a list of space-separated JSON strings)")
320+
// In order to allow empty arguments, we set a dummy NoOptDefVal for these flags
321+
f.Lookup("served-model-name").NoOptDefVal = "dummy"
322+
f.Lookup("lora-modules").NoOptDefVal = "dummy"
323+
324+
flagSet := flag.NewFlagSet("simFlagSet", flag.ExitOnError)
325+
klog.InitFlags(flagSet)
326+
f.AddGoFlagSet(flagSet)
327+
328+
if err := f.Parse(os.Args[1:]); err != nil {
329+
if err == pflag.ErrHelp {
330+
// --help - exit without printing an error message
331+
os.Exit(0)
332+
}
333+
return nil, err
334+
}
335+
336+
// Need to read in a variable to avoid merging the values with the config file ones
337+
if loraModuleNames != nil {
338+
config.LoraModulesString = loraModuleNames
339+
if err := config.unmarshalLoras(); err != nil {
340+
return nil, err
341+
}
342+
}
343+
if servedModelNames != nil {
344+
config.ServedModelNames = servedModelNames
345+
}
346+
347+
if err := config.validate(); err != nil {
348+
return nil, err
349+
}
350+
351+
return config, nil
352+
}
353+
354+
func getParamValueFromArgs(param string) []string {
355+
var values []string
356+
var readValues bool
357+
for _, arg := range os.Args[1:] {
358+
if readValues {
359+
if strings.HasPrefix(arg, "--") {
360+
break
361+
}
362+
if arg != "" {
363+
values = append(values, arg)
364+
}
365+
} else {
366+
if arg == "--"+param {
367+
readValues = true
368+
values = make([]string, 0)
369+
} else if strings.HasPrefix(arg, "--"+param+"=") {
370+
// Handle --param=value
371+
values = append(values, strings.TrimPrefix(arg, "--"+param+"="))
372+
break
373+
}
374+
}
375+
}
376+
return values
377+
}

pkg/llm-d-inference-sim/simulator_startup_test.go renamed to pkg/common/config_test.go

Lines changed: 14 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -14,40 +14,32 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17-
package llmdinferencesim
17+
package common
1818

1919
import (
2020
"os"
2121

22-
"github.com/llm-d/llm-d-inference-sim/pkg/common"
2322
. "github.com/onsi/ginkgo/v2"
2423
. "github.com/onsi/gomega"
25-
"k8s.io/klog/v2"
2624
)
2725

2826
const (
2927
qwenModelName = "Qwen/Qwen2-0.5B"
28+
model = "test-model"
3029
)
3130

32-
func createSimConfig(args []string) (*common.Configuration, error) {
31+
func createSimConfig(args []string) (*Configuration, error) {
3332
oldArgs := os.Args
3433
defer func() {
3534
os.Args = oldArgs
3635
}()
3736
os.Args = args
3837

39-
s, err := New(klog.Background())
40-
if err != nil {
41-
return nil, err
42-
}
43-
if err := s.parseCommandParamsAndLoadConfig(); err != nil {
44-
return nil, err
45-
}
46-
return s.config, nil
38+
return ParseCommandParamsAndLoadConfig()
4739
}
4840

49-
func createDefaultConfig(model string) *common.Configuration {
50-
c := common.NewConfig()
41+
func createDefaultConfig(model string) *Configuration {
42+
c := newConfig()
5143

5244
c.Model = model
5345
c.ServedModelNames = []string{c.Model}
@@ -58,29 +50,29 @@ func createDefaultConfig(model string) *common.Configuration {
5850
c.InterTokenLatency = 1000
5951
c.KVCacheTransferLatency = 100
6052
c.Seed = 100100100
61-
c.LoraModules = []common.LoraModule{}
53+
c.LoraModules = []LoraModule{}
6254

6355
return c
6456
}
6557

6658
type testCase struct {
6759
name string
6860
args []string
69-
expectedConfig *common.Configuration
61+
expectedConfig *Configuration
7062
}
7163

7264
var _ = Describe("Simulator configuration", func() {
7365
tests := make([]testCase, 0)
7466

7567
// Simple config with a few parameters
76-
c := common.NewConfig()
68+
c := newConfig()
7769
c.Model = model
7870
c.ServedModelNames = []string{c.Model}
7971
c.MaxCPULoras = 1
8072
c.Seed = 100
8173
test := testCase{
8274
name: "simple",
83-
args: []string{"cmd", "--model", model, "--mode", common.ModeRandom, "--seed", "100"},
75+
args: []string{"cmd", "--model", model, "--mode", ModeRandom, "--seed", "100"},
8476
expectedConfig: c,
8577
}
8678
tests = append(tests, test)
@@ -89,7 +81,7 @@ var _ = Describe("Simulator configuration", func() {
8981
c = createDefaultConfig(qwenModelName)
9082
c.Port = 8001
9183
c.ServedModelNames = []string{"model1", "model2"}
92-
c.LoraModules = []common.LoraModule{{Name: "lora1", Path: "/path/to/lora1"}, {Name: "lora2", Path: "/path/to/lora2"}}
84+
c.LoraModules = []LoraModule{{Name: "lora1", Path: "/path/to/lora1"}, {Name: "lora2", Path: "/path/to/lora2"}}
9385
test = testCase{
9486
name: "config file",
9587
args: []string{"cmd", "--config", "../../manifests/config.yaml"},
@@ -106,7 +98,7 @@ var _ = Describe("Simulator configuration", func() {
10698
c.Port = 8002
10799
c.ServedModelNames = []string{"alias1", "alias2"}
108100
c.Seed = 100
109-
c.LoraModules = []common.LoraModule{{Name: "lora3", Path: "/path/to/lora3"}, {Name: "lora4", Path: "/path/to/lora4"}}
101+
c.LoraModules = []LoraModule{{Name: "lora3", Path: "/path/to/lora3"}, {Name: "lora4", Path: "/path/to/lora4"}}
110102
c.LoraModulesString = []string{
111103
"{\"name\":\"lora3\",\"path\":\"/path/to/lora3\"}",
112104
"{\"name\":\"lora4\",\"path\":\"/path/to/lora4\"}",
@@ -124,7 +116,7 @@ var _ = Describe("Simulator configuration", func() {
124116
// Config from config.yaml file plus command line args with different format
125117
c = createDefaultConfig(model)
126118
c.Port = 8002
127-
c.LoraModules = []common.LoraModule{{Name: "lora3", Path: "/path/to/lora3"}}
119+
c.LoraModules = []LoraModule{{Name: "lora3", Path: "/path/to/lora3"}}
128120
c.LoraModulesString = []string{
129121
"{\"name\":\"lora3\",\"path\":\"/path/to/lora3\"}",
130122
}
@@ -141,7 +133,7 @@ var _ = Describe("Simulator configuration", func() {
141133
// Config from config.yaml file plus command line args with empty string
142134
c = createDefaultConfig(model)
143135
c.Port = 8002
144-
c.LoraModules = []common.LoraModule{{Name: "lora3", Path: "/path/to/lora3"}}
136+
c.LoraModules = []LoraModule{{Name: "lora3", Path: "/path/to/lora3"}}
145137
c.LoraModulesString = []string{
146138
"{\"name\":\"lora3\",\"path\":\"/path/to/lora3\"}",
147139
}

pkg/common/test-helpers.go renamed to pkg/common/test_helpers.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,16 +27,16 @@ func IsValidText(text string) bool {
2727
textToCheck := text[charsTested:]
2828
found := false
2929

30-
for _, fakeSentense := range chatCompletionFakeResponses {
31-
if len(textToCheck) <= len(fakeSentense) {
32-
if strings.HasPrefix(fakeSentense, textToCheck) {
30+
for _, fakeSentence := range chatCompletionFakeResponses {
31+
if len(textToCheck) <= len(fakeSentence) {
32+
if strings.HasPrefix(fakeSentence, textToCheck) {
3333
found = true
3434
charsTested = len(text)
3535
break
3636
}
3737
} else {
38-
if strings.HasPrefix(textToCheck, fakeSentense) {
39-
charsTested += len(fakeSentense)
38+
if strings.HasPrefix(textToCheck, fakeSentence) {
39+
charsTested += len(fakeSentence)
4040
// during generation sentences are connected by space, skip it
4141
// additional space at the end of the string is invalid
4242
if text[charsTested] == ' ' && charsTested < len(text)-1 {

0 commit comments

Comments
 (0)