Skip to content

Commit 04abfd4

Browse files
wanghh2000HaiHui886
wanghh2000
authored and
HaiHui886
committed
Add engine args support for runtime frameworks in deploy process
1 parent 2df3a78 commit 04abfd4

File tree

13 files changed

+236
-40
lines changed

13 files changed

+236
-40
lines changed

‎builder/deploy/common/utils.go‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,23 @@
11
package common
22

33
import (
4-
"encoding/json"
4+
"fmt"
55
"strings"
66
)
77

8-
func JsonStrToMap(jsonStr string) (map[string]string, error) {
9-
var resMap map[string]string
10-
if len(strings.Trim(jsonStr, " ")) == 0 {
11-
return map[string]string{}, nil
8+
func GetNamespaceAndNameFromGitPath(gitpath string) (string, string, error) {
9+
if gitpath == "" {
10+
return "", "", fmt.Errorf("empty git path %s", gitpath)
1211
}
13-
err := json.Unmarshal([]byte(jsonStr), &resMap)
14-
return resMap, err
12+
var fields []string
13+
idx := strings.Index(gitpath, "_")
14+
if idx > -1 && idx+1 < len(gitpath) {
15+
fields = strings.Split(gitpath[idx+1:], "/")
16+
if len(fields) != 2 {
17+
return "", "", fmt.Errorf("empty git path %s", gitpath)
18+
}
19+
} else {
20+
return "", "", fmt.Errorf("empty git path %s", gitpath)
21+
}
22+
return fields[0], fields[1], nil
1523
}

‎builder/deploy/deployer.go‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"opencsg.com/csghub-server/builder/deploy/scheduler"
1919
"opencsg.com/csghub-server/builder/store/database"
2020
"opencsg.com/csghub-server/common/types"
21+
hubcom "opencsg.com/csghub-server/common/utils/common"
2122
)
2223

2324
type Deployer interface {
@@ -529,10 +530,29 @@ func (d *deployer) UpdateDeploy(ctx context.Context, dur *types.DeployUpdateReq,
529530
if dur.SecureLevel != nil {
530531
deploy.SecureLevel = *dur.SecureLevel
531532
}
533+
532534
if dur.ClusterID != nil {
533535
deploy.ClusterID = *dur.ClusterID
534536
}
535537

538+
if dur.EngineArgs != nil {
539+
deploy.EngineArgs = *dur.EngineArgs
540+
}
541+
542+
if dur.Entrypoint != nil {
543+
if deploy.RuntimeFramework == string(types.LlamaCpp) {
544+
newVarStr, err := buildVariables(dur)
545+
if err != nil {
546+
return fmt.Errorf("build variables for llama cpp error: %w", err)
547+
}
548+
dur.Variables = &newVarStr
549+
}
550+
}
551+
552+
if dur.Variables != nil {
553+
deploy.Variables = *dur.Variables
554+
}
555+
536556
// update deploy table
537557
err = d.deployTaskStore.UpdateDeploy(ctx, deploy)
538558
if err != nil {
@@ -542,6 +562,24 @@ func (d *deployer) UpdateDeploy(ctx context.Context, dur *types.DeployUpdateReq,
542562
return nil
543563
}
544564

565+
func buildVariables(dur *types.DeployUpdateReq) (string, error) {
566+
varStr := ""
567+
if dur.Variables != nil {
568+
varStr = *dur.Variables
569+
}
570+
varMap, err := hubcom.JsonStrToMap(varStr)
571+
if err != nil {
572+
return "", fmt.Errorf("invalid json format of variables error: %w", err)
573+
}
574+
varMap[types.GGUFEntryPoint] = *dur.Entrypoint
575+
varBytes, err := json.Marshal(varMap)
576+
if err != nil {
577+
return "", fmt.Errorf("marshal variables error: %w", err)
578+
}
579+
varStr = string(varBytes)
580+
return varStr, nil
581+
}
582+
545583
func (d *deployer) StartDeploy(ctx context.Context, deploy *database.Deploy) error {
546584
deploy.Status = common.Pending
547585
// update deploy table

‎builder/deploy/scheduler/deploy_runner.go‎

Lines changed: 60 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -14,28 +14,31 @@ import (
1414
"opencsg.com/csghub-server/builder/deploy/imagerunner"
1515
"opencsg.com/csghub-server/builder/store/database"
1616
"opencsg.com/csghub-server/common/types"
17+
hubcom "opencsg.com/csghub-server/common/utils/common"
1718
)
1819

1920
// DeployRunner defines a k8s image running task
2021
type DeployRunner struct {
21-
repo *RepoInfo
22-
task *database.DeployTask
23-
ir imagerunner.Runner
24-
store database.DeployTaskStore
25-
tokenStore database.AccessTokenStore
26-
deployStartTime time.Time
27-
deployCfg common.DeployConfig
22+
repo *RepoInfo
23+
task *database.DeployTask
24+
ir imagerunner.Runner
25+
store database.DeployTaskStore
26+
tokenStore database.AccessTokenStore
27+
deployStartTime time.Time
28+
deployCfg common.DeployConfig
29+
runtimeFrameworksStore database.RuntimeFrameworksStore
2830
}
2931

3032
func NewDeployRunner(ir imagerunner.Runner, r *RepoInfo, t *database.DeployTask, deployCfg common.DeployConfig) Runner {
3133
return &DeployRunner{
32-
repo: r,
33-
task: t,
34-
ir: ir,
35-
store: database.NewDeployTaskStore(),
36-
deployStartTime: time.Now(),
37-
tokenStore: database.NewAccessTokenStore(),
38-
deployCfg: deployCfg,
34+
repo: r,
35+
task: t,
36+
ir: ir,
37+
store: database.NewDeployTaskStore(),
38+
deployStartTime: time.Now(),
39+
tokenStore: database.NewAccessTokenStore(),
40+
deployCfg: deployCfg,
41+
runtimeFrameworksStore: database.NewRuntimeFrameworksStore(),
3942
}
4043

4144
}
@@ -209,17 +212,34 @@ func (t *DeployRunner) runtimeError(msg string) {
209212
}
210213

211214
func (t *DeployRunner) makeDeployRequest() (*types.RunRequest, error) {
212-
token, err := t.tokenStore.FindByUID(context.Background(), t.task.Deploy.UserID)
215+
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
216+
defer cancel()
217+
218+
token, err := t.tokenStore.FindByUID(ctx, t.task.Deploy.UserID)
213219
if err != nil {
214220
return nil, fmt.Errorf("cant get git access token:%w", err)
215221
}
216222
fields := strings.Split(t.repo.Path, "/")
217-
deploy, err := t.store.GetDeployByID(context.Background(), t.task.DeployID)
223+
deploy, err := t.store.GetDeployByID(ctx, t.task.DeployID)
218224
if err != nil {
219-
return nil, fmt.Errorf("fail to get deploy with error :%w", err)
225+
return nil, fmt.Errorf("fail to get deploy with error: %w", err)
220226
}
221227

222-
annoMap, err := common.JsonStrToMap(deploy.Annotation)
228+
var engineArgsTemplate []types.EngineArg
229+
if len(deploy.RuntimeFramework) > 0 {
230+
frame, err := t.runtimeFrameworksStore.FindEnabledByName(ctx, deploy.RuntimeFramework)
231+
if err != nil {
232+
return nil, fmt.Errorf("get runtime framework by name %s error: %w", deploy.RuntimeFramework, err)
233+
}
234+
if len(strings.TrimSpace(frame.EngineArgs)) > 0 {
235+
err = json.Unmarshal([]byte(frame.EngineArgs), &engineArgsTemplate)
236+
if err != nil {
237+
return nil, fmt.Errorf("unmarshal engine args error: %w", err)
238+
}
239+
}
240+
}
241+
242+
annoMap, err := hubcom.JsonStrToMap(deploy.Annotation)
223243
if err != nil {
224244
slog.Error("deploy annotation is invalid json data", slog.Any("Annotation", deploy.Annotation))
225245
return nil, err
@@ -233,7 +253,7 @@ func (t *DeployRunner) makeDeployRequest() (*types.RunRequest, error) {
233253
return nil, err
234254
}
235255

236-
envMap := t.makeDeployEnv(hardware, token, deploy)
256+
envMap := t.makeDeployEnv(hardware, token, deploy, engineArgsTemplate)
237257

238258
targetID := deploy.SpaceID
239259
// deployID is unique for space and model
@@ -267,14 +287,16 @@ func (t *DeployRunner) makeDeployRequest() (*types.RunRequest, error) {
267287

268288
func (t *DeployRunner) makeDeployEnv(
269289
hardware types.HardWare,
270-
token *database.AccessToken, deploy *database.Deploy,
290+
token *database.AccessToken,
291+
deploy *database.Deploy,
292+
engineArgsTemplate []types.EngineArg,
271293
) map[string]string {
272-
envMap, err := common.JsonStrToMap(deploy.Env)
294+
envMap, err := hubcom.JsonStrToMap(deploy.Env)
273295
if err != nil {
274296
slog.Error("deploy env is invalid json data", slog.Any("deploy", deploy))
275297
}
276298

277-
varMap, err := common.JsonStrToMap(deploy.Variables)
299+
varMap, err := hubcom.JsonStrToMap(deploy.Variables)
278300
if err != nil {
279301
slog.Error("deploy variables is invalid json data", slog.Any("deploy", deploy))
280302
} else {
@@ -289,7 +311,22 @@ func (t *DeployRunner) makeDeployEnv(
289311
envMap["ACCESS_TOKEN"] = token.Token
290312
envMap["REPO_ID"] = t.repo.Path // "namespace/name"
291313
envMap["REVISION"] = deploy.GitBranch // branch
292-
envMap["ENGINE_ARGS"] = deploy.EngineArgs
314+
if len(engineArgsTemplate) > 0 {
315+
ENGINE_ARGS := ""
316+
argValuesMap, err := hubcom.JsonStrToMap(deploy.EngineArgs)
317+
if err != nil {
318+
slog.Error("deploy engine args is invalid json data", slog.Any("deploy", *deploy), slog.Any("error", err))
319+
} else {
320+
for _, arg := range engineArgsTemplate {
321+
if value, ok := argValuesMap[arg.Name]; ok {
322+
ENGINE_ARGS += " " + fmt.Sprintf(arg.Format, value)
323+
}
324+
}
325+
}
326+
slog.Debug("makeDeployEnv", slog.Any("ENGINE_ARGS", ENGINE_ARGS))
327+
envMap["ENGINE_ARGS"] = ENGINE_ARGS
328+
}
329+
293330
if hardware.Gpu.Num != "" {
294331
envMap["GPU_NUM"] = hardware.Gpu.Num
295332
}

‎builder/store/database/migrations/20250226052439_add_column_engine_args_to_runtime_framework.down.sql‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
SET statement_timeout = 0;
2+
3+
--bun:split
4+
5+
ALTER TABLE runtime_frameworks DROP COLUMN IF EXISTS engine_args;

‎builder/store/database/migrations/20250226052439_add_column_engine_args_to_runtime_framework.up.sql‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
SET statement_timeout = 0;
2+
3+
--bun:split
4+
5+
ALTER TABLE runtime_frameworks ADD COLUMN IF NOT EXISTS engine_args VARCHAR;

‎builder/store/database/runtime_framework.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ type RuntimeFramework struct {
4646
Enabled int64 `bun:",notnull" json:"enabled"`
4747
ContainerPort int `bun:",notnull" json:"container_port"`
4848
Type int `bun:",notnull" json:"type"` // 0-space, 1-inference, 2-finetune
49+
EngineArgs string `bun:",notnull" json:"engine_args"`
4950
times
5051
}
5152

‎common/types/model.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,17 @@ const (
205205
ModelWidgetTypeChat ModelWidgetType = "chat"
206206
)
207207

208+
type ModelType string
209+
210+
const (
211+
GGUF ModelType = "gguf"
212+
Safetensors ModelType = "safetensors"
213+
)
214+
215+
const (
216+
GGUFEntryPoint = "GGUF_ENTRY_POINT"
217+
)
218+
208219
type ModelRunReq struct {
209220
DeployName string `json:"deploy_name"`
210221
ClusterID string `json:"cluster_id"`
@@ -216,6 +227,8 @@ type ModelRunReq struct {
216227
Revision string `json:"revision"`
217228
SecureLevel int `json:"secure_level"`
218229
OrderDetailID int64 `json:"order_detail_id"`
230+
Entrypoint string `json:"entrypoint"` // model file name for gguf model
231+
EngineArgs string `json:"engine_args"`
219232
}
220233

221234
var _ SensitiveRequestV2 = (*ModelRunReq)(nil)
@@ -238,6 +251,8 @@ type InstanceRunReq struct {
238251
ResourceID int64 `json:"resource_id"`
239252
RuntimeFrameworkID int64 `json:"runtime_framework_id"`
240253
Revision string `json:"revision"`
254+
OrderDetailID int64 `json:"order_detail_id"`
255+
EngineArgs string `json:"engine_args"`
241256
}
242257

243258
var _ SensitiveRequestV2 = (*InstanceRunReq)(nil)
@@ -304,6 +319,9 @@ type DeployUpdateReq struct {
304319
MaxReplica *int `json:"max_replica" validate:"min=1,gtefield=MinReplica"`
305320
Revision *string `json:"revision"`
306321
SecureLevel *int `json:"secure_level"`
322+
Entrypoint *string `json:"entrypoint"`
323+
Variables *string `json:"variables"`
324+
EngineArgs *string `json:"engine_args"`
307325
}
308326

309327
type RelationModels struct {

‎common/types/repo.go‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,10 @@ func (s SensitiveCheckStatus) String() string {
3535
}
3636

3737
const (
38-
ResTypeKey string = "hub-res-type"
39-
ResNameKey string = "hub-res-name"
40-
ResDeployID string = "hub-deploy-id"
38+
ResTypeKey string = "hub-res-type"
39+
ResNameKey string = "hub-res-name"
40+
ResDeployID string = "hub-deploy-id"
41+
ResDeployUser string = "hub-deploy-user"
4142

4243
ModelRepo RepositoryType = "model"
4344
DatasetRepo RepositoryType = "dataset"
@@ -164,6 +165,7 @@ type DeployRepo struct {
164165
Task string `json:"task,omitempty"`
165166
EngineArgs string `json:"engine_args,omitempty"`
166167
Variables string `json:"variables,omitempty"`
168+
Entrypoint string `json:"entrypoint,omitempty"`
167169
}
168170

169171
type RuntimeFrameworkReq struct {
@@ -174,6 +176,7 @@ type RuntimeFrameworkReq struct {
174176
Enabled int64 `json:"enabled"`
175177
ContainerPort int `json:"container_port"`
176178
Type int `json:"type"`
179+
EngineArgs string `json:"engine_args"`
177180
CurrentUser string `json:"-"`
178181
}
179182

@@ -186,6 +189,7 @@ type RuntimeFramework struct {
186189
Enabled int64 `json:"enabled"`
187190
ContainerPort int `json:"container_port"`
188191
Type int `json:"type"`
192+
EngineArgs string `json:"engine_args"`
189193
}
190194

191195
type RuntimeFrameworkModels struct {

‎common/types/service_runner.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,4 +188,10 @@ type (
188188
OrderDetailID int64 `json:"order_detail_id"`
189189
SrvName string `json:"-"`
190190
}
191+
192+
EngineArg struct {
193+
Name string `json:"name"`
194+
Value string `json:"value"`
195+
Format string `json:"format"`
196+
}
191197
)

‎common/utils/common/string.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,19 @@ package common
33
import (
44
"crypto/md5"
55
"encoding/hex"
6+
"encoding/json"
7+
"strings"
68
)
79

10+
func JsonStrToMap(jsonStr string) (map[string]string, error) {
11+
var resMap map[string]string
12+
if len(strings.Trim(jsonStr, " ")) == 0 {
13+
return map[string]string{}, nil
14+
}
15+
err := json.Unmarshal([]byte(jsonStr), &resMap)
16+
return resMap, err
17+
}
18+
819
// TruncString return a substring if the input string is larger than limit size, truncated string ends with "..."
920
func TruncString(s string, limit int) string {
1021
if len(s) <= limit {

0 commit comments

Comments
 (0)