Skip to content

Commit a610062

Browse files
miaow999miaodanyanghannatao
authored
update app core config add http speed limit (#590)
Co-authored-by: miaodanyang <miaodanyang@baidu.com> Co-authored-by: hannatao <413024870@qq.com>
1 parent 47a4899 commit a610062

File tree

7 files changed

+99
-35
lines changed

7 files changed

+99
-35
lines changed

‎api/node.go‎

Lines changed: 63 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,14 @@ const (
5151
HookUpdateNodeDmp = "hookUpdateNodeDmp"
5252
HookDeleteNodeDmp = "hookDeleteNodeDmp"
5353

54-
BaetylCoreLogLevel = "BaetylCoreLogLevel"
55-
LogLevelDebug = "debug"
56-
UserID = "UserId"
54+
BaetylCoreLogLevel = "BaetylCoreLogLevel"
55+
BaetylCoreByteUnit = "BaetylCoreByteUint"
56+
BaetylCoreSpeedLimit = "BaetylCoreSpeedLimit"
57+
LogLevelDebug = "debug"
58+
UserID = "UserId"
59+
60+
ByteUnitKB = "KB"
61+
ByteUnitMB = "MB"
5762
)
5863

5964
var (
@@ -717,12 +722,16 @@ func (api *API) UpdateCoreApp(c *common.Context) (interface{}, error) {
717722
}
718723

719724
logLevel := api.getLogLevel(node)
725+
byteUnit := api.getByteUnit(node)
726+
speedLimit := api.getSpeedLimit(node)
720727

721728
if coreConfig.Version == version &&
722729
coreConfig.Frequency == freq &&
723730
coreConfig.APIPort == port &&
724731
coreConfig.AgentPort == agentPort &&
725-
coreConfig.LogLevel == logLevel {
732+
coreConfig.LogLevel == logLevel &&
733+
coreConfig.ByteUnit == byteUnit &&
734+
coreConfig.SpeedLimit == speedLimit {
726735
return api.ToApplicationView(app)
727736
}
728737

@@ -740,8 +749,10 @@ func (api *API) UpdateCoreApp(c *common.Context) (interface{}, error) {
740749
}
741750
node.Attributes[v1.BaetylCoreAPIPort] = fmt.Sprintf("%d", coreConfig.APIPort)
742751
node.Attributes[BaetylCoreLogLevel] = coreConfig.LogLevel
752+
node.Attributes[BaetylCoreByteUnit] = coreConfig.ByteUnit
753+
node.Attributes[BaetylCoreSpeedLimit] = coreConfig.SpeedLimit
743754

744-
err = api.updateCoreAppConfig(app, node, coreConfig.Frequency, coreConfig.AgentPort, coreConfig.LogLevel)
755+
err = api.updateCoreAppConfig(app, node, coreConfig)
745756
if err != nil {
746757
return nil, err
747758
}
@@ -860,6 +871,10 @@ func (api *API) GetCoreAppConfigs(c *common.Context) (interface{}, error) {
860871
}
861872

862873
coreInfo.LogLevel = api.getLogLevel(node)
874+
// get byte unit
875+
coreInfo.ByteUnit = api.getByteUnit(node)
876+
// get s
877+
coreInfo.SpeedLimit = api.getSpeedLimit(node)
863878

864879
return coreInfo, nil
865880
}
@@ -1239,21 +1254,23 @@ func (api *API) updateCoreVersions(node *v1.Node, currentVersion, updateVersion
12391254
node.Attributes[BaetylCorePrevVersion] = currentVersion
12401255
}
12411256

1242-
func (api *API) updateCoreAppConfig(app *v1.Application, node *v1.Node, freq, agentPort int, logLevel string) error {
1257+
func (api *API) updateCoreAppConfig(app *v1.Application, node *v1.Node, coreConfig *models.NodeCoreConfigs) error {
12431258
config, err := api.getAppConfig(app, BaetylCoreConfPrefix)
12441259
if err != nil {
12451260
return err
12461261
}
12471262
params := map[string]interface{}{
1248-
"CoreConfName": config.Name,
1249-
"CoreAppName": app.Name,
1250-
"NodeMode": node.NodeMode,
1251-
"CoreFrequency": fmt.Sprintf("%ds", freq),
1252-
"AgentPort": fmt.Sprintf("%d", agentPort),
1253-
"GPUStats": node.Accelerator != "",
1254-
"DiskNetStats": node.NodeMode == context.RunModeKube,
1255-
"QPSStats": node.NodeMode == context.RunModeKube,
1256-
BaetylCoreLogLevel: logLevel,
1263+
"CoreConfName": config.Name,
1264+
"CoreAppName": app.Name,
1265+
"NodeMode": node.NodeMode,
1266+
"CoreFrequency": fmt.Sprintf("%ds", coreConfig.Frequency),
1267+
"AgentPort": fmt.Sprintf("%d", coreConfig.AgentPort),
1268+
"GPUStats": node.Accelerator != "",
1269+
"DiskNetStats": node.NodeMode == context.RunModeKube,
1270+
"QPSStats": node.NodeMode == context.RunModeKube,
1271+
BaetylCoreLogLevel: coreConfig.LogLevel,
1272+
BaetylCoreSpeedLimit: coreConfig.SpeedLimit,
1273+
BaetylCoreByteUnit: coreConfig.ByteUnit,
12571274
}
12581275
res, err := api.Init.GetResource(config.Namespace, node.Name, service.TemplateCoreConfYaml, params)
12591276
if err != nil {
@@ -1428,6 +1445,26 @@ func (api *API) getLogLevel(node *v1.Node) string {
14281445
return node.Attributes[BaetylCoreLogLevel].(string)
14291446
}
14301447

1448+
func (api *API) getByteUnit(node *v1.Node) string {
1449+
if node.Attributes == nil {
1450+
return ByteUnitKB
1451+
}
1452+
if _, ok := node.Attributes[BaetylCoreByteUnit]; !ok {
1453+
node.Attributes[BaetylCoreByteUnit] = ByteUnitKB
1454+
}
1455+
return node.Attributes[BaetylCoreByteUnit].(string)
1456+
}
1457+
1458+
func (api *API) getSpeedLimit(node *v1.Node) int {
1459+
if node.Attributes == nil {
1460+
return 0
1461+
}
1462+
if _, ok := node.Attributes[BaetylCoreSpeedLimit]; !ok {
1463+
node.Attributes[BaetylCoreSpeedLimit] = 0
1464+
}
1465+
return node.Attributes[BaetylCoreSpeedLimit].(int)
1466+
}
1467+
14311468
func (api *API) updateAgentAppPort(ns string, agent *v1.Application, oldPort, newPort int) error {
14321469
for i, v := range agent.Services[0].Ports {
14331470
if v.HostPort == int32(oldPort) {
@@ -1505,7 +1542,17 @@ func (api *API) UpdateConfigByAccelerator(ns string, node *v1.Node) error {
15051542
return err
15061543
}
15071544
logLevel := api.getLogLevel(node)
1508-
err = api.updateCoreAppConfig(core, node, freq, agentPort, logLevel)
1545+
speedLimit := api.getSpeedLimit(node)
1546+
byteUint := api.getByteUnit(node)
1547+
err = api.updateCoreAppConfig(core, node, &models.NodeCoreConfigs{
1548+
Version: "",
1549+
Frequency: freq,
1550+
APIPort: 0,
1551+
AgentPort: agentPort,
1552+
LogLevel: logLevel,
1553+
ByteUnit: byteUint,
1554+
SpeedLimit: speedLimit,
1555+
})
15091556
if err != nil {
15101557
return err
15111558
}

‎api/node_test.go‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1115,6 +1115,8 @@ func TestUpdateNodeAccelerator(t *testing.T) {
11151115
specV1.BaetylCoreFrequency: common.DefaultCoreFrequency,
11161116
specV1.BaetylAgentPort: common.DefaultAgentPort,
11171117
BaetylCoreLogLevel: LogLevelDebug,
1118+
BaetylCoreByteUnit: ByteUnitKB,
1119+
BaetylCoreSpeedLimit: 0,
11181120
UserID: "",
11191121
},
11201122
Desire: specV1.Desire{
@@ -2013,15 +2015,17 @@ func TestAPI_UpdateCoreApp(t *testing.T) {
20132015
mockConfig.EXPECT().Get(ns, "baetyl-program-config-baetyl-core", "").Return(pconfig, nil).Times(1)
20142016

20152017
pparams := map[string]interface{}{
2016-
"CoreAppName": "baetyl-core-1",
2017-
"CoreConfName": "baetyl-core-conf-ialplsycd",
2018-
"CoreFrequency": "40s",
2019-
"NodeMode": "native",
2020-
"AgentPort": "30080",
2021-
"GPUStats": true,
2022-
"DiskNetStats": false,
2023-
"QPSStats": false,
2024-
BaetylCoreLogLevel: LogLevelDebug,
2018+
"CoreAppName": "baetyl-core-1",
2019+
"CoreConfName": "baetyl-core-conf-ialplsycd",
2020+
"CoreFrequency": "40s",
2021+
"NodeMode": "native",
2022+
"AgentPort": "30080",
2023+
"GPUStats": true,
2024+
"DiskNetStats": false,
2025+
"QPSStats": false,
2026+
BaetylCoreLogLevel: LogLevelDebug,
2027+
BaetylCoreByteUnit: ByteUnitKB,
2028+
BaetylCoreSpeedLimit: 0,
20252029
}
20262030

20272031
params := map[string]interface{}{
@@ -2159,6 +2163,8 @@ func TestAPI_GetCoreAppConfigs(t *testing.T) {
21592163
specV1.BaetylCoreFrequency: common.DefaultCoreFrequency,
21602164
specV1.BaetylCoreAPIPort: common.DefaultCoreAPIPort,
21612165
specV1.BaetylAgentPort: common.DefaultAgentPort,
2166+
BaetylCoreByteUnit: ByteUnitKB,
2167+
BaetylCoreSpeedLimit: 100,
21622168
},
21632169
Report: map[string]interface{}{"1": "1"},
21642170
Desire: map[string]interface{}{"2": "2"},

‎go.mod‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ require (
66
github.com/VictoriaMetrics/fastcache v1.12.1
77
github.com/ZZMarquis/gm v1.3.2
88
github.com/aws/aws-sdk-go v1.34.0
9-
github.com/baetyl/baetyl-go/v2 v2.2.4-0.20230625073200-0f48322528d2
9+
github.com/baetyl/baetyl-go/v2 v2.2.4-0.20230626031834-a39dbb3259dd
1010
github.com/gin-contrib/cache v1.1.0
1111
github.com/gin-gonic/gin v1.9.0
1212
github.com/go-playground/validator/v10 v10.11.2
@@ -35,6 +35,7 @@ require (
3535
github.com/bytedance/sonic v1.8.0 // indirect
3636
github.com/cespare/xxhash/v2 v2.2.0 // indirect
3737
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
38+
github.com/conduitio/bwlimit v0.1.0 // indirect
3839
github.com/containerd/containerd v1.5.18 // indirect
3940
github.com/creasty/defaults v1.4.0 // indirect
4041
github.com/davecgh/go-spew v1.1.1 // indirect
@@ -96,7 +97,7 @@ require (
9697
golang.org/x/sys v0.5.0 // indirect
9798
golang.org/x/term v0.5.0 // indirect
9899
golang.org/x/text v0.7.0 // indirect
99-
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
100+
golang.org/x/time v0.3.0 // indirect
100101
google.golang.org/appengine v1.6.5 // indirect
101102
google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
102103
google.golang.org/grpc v1.33.2 // indirect

‎go.sum‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9Pq
3737
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
3838
github.com/aws/aws-sdk-go v1.34.0 h1:brux2dRrlwCF5JhTL7MUT3WUwo9zfDHZZp3+g3Mvlmo=
3939
github.com/aws/aws-sdk-go v1.34.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
40-
github.com/baetyl/baetyl-go/v2 v2.2.4-0.20230625073200-0f48322528d2 h1:tcn+pg0PaUPk+vGmaPe8lYwyELUYZbi4bO+xJL6u3vc=
41-
github.com/baetyl/baetyl-go/v2 v2.2.4-0.20230625073200-0f48322528d2/go.mod h1:gHgoxwl5T59XxJodoEyPJrL8GDRrJLyxhxxscDsqyEs=
40+
github.com/baetyl/baetyl-go/v2 v2.2.4-0.20230626031834-a39dbb3259dd h1:bnBip/ygpP6EYVS4LEhtQzlHlzXT9/zQFJOIM4PSY9I=
41+
github.com/baetyl/baetyl-go/v2 v2.2.4-0.20230626031834-a39dbb3259dd/go.mod h1:+cGX9Pxi6IRlDF1+A10BIN1fjPxPQ9JnDqtp/a8DxDE=
4242
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
4343
github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
4444
github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737 h1:rRISKWyXfVxvoa702s91Zl5oREZTrR3yv+tXrrX7G/g=
@@ -57,6 +57,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
5757
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
5858
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
5959
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
60+
github.com/conduitio/bwlimit v0.1.0 h1:x3ijON0TSghQob4tFKaEvKixFmYKfVJQeSpXluC2JvE=
61+
github.com/conduitio/bwlimit v0.1.0/go.mod h1:E+ASZ1/5L33MTb8hJTERs5Xnmh6Ulq3jbRh7LrdbXWU=
6062
github.com/containerd/containerd v1.5.18 h1:doHr6cNxfOLTotWmZs6aZF6LrfJFcjmYFcWlRmQgYPM=
6163
github.com/containerd/containerd v1.5.18/go.mod h1:7IN9MtIzTZH4WPEmD1gNH8bbTQXVX68yd3ZXxSHYCis=
6264
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -469,8 +471,8 @@ golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
469471
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
470472
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
471473
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
472-
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s=
473-
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
474+
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
475+
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
474476
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
475477
golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
476478
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

‎models/node.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,12 @@ type NodePropertiesMetadata struct {
6060
type NodeCoreConfigs struct {
6161
Version string `yaml:"version,omitempty" json:"version,omitempty"`
6262
// unit: seconds
63-
Frequency int `yaml:"frequency,omitempty" json:"frequency,omitempty"`
64-
APIPort int `yaml:"apiport,omitempty" json:"apiport,omitempty"`
65-
AgentPort int `yaml:"agentport,omitempty" json:"agentport,omitempty" default:"30080"`
66-
LogLevel string `yaml:"logLevel,omitempty" json:"logLevel,omitempty" default:"debug" binding:"omitempty,oneof=debug info warn error"`
63+
Frequency int `yaml:"frequency,omitempty" json:"frequency,omitempty"`
64+
APIPort int `yaml:"apiport,omitempty" json:"apiport,omitempty"`
65+
AgentPort int `yaml:"agentport,omitempty" json:"agentport,omitempty" default:"30080"`
66+
LogLevel string `yaml:"logLevel,omitempty" json:"logLevel,omitempty" default:"debug" binding:"omitempty,oneof=debug info warn error"`
67+
ByteUnit string `yaml:"byteUnit,omitempty" json:"byteUnit,omitempty" default:"KB" `
68+
SpeedLimit int `yaml:"speedLimit,omitempty" json:"speedLimit,omitempty" default:"0"`
6769
}
6870

6971
type NodeCoreVersions struct {

‎scripts/native/templates/baetyl-core-conf.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ data:
1717
sync:
1818
download:
1919
timeout: 30m
20+
speedLimit: {{.BaetylCoreSpeedLimit}}
21+
byteUint: {{.BaetylCoreByteUint}}
2022
report:
2123
interval: {{.CoreFrequency}}
2224
httplink:

‎service/template_test.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ var params = map[string]interface{}{
3333
"NodeCertCa": "---node cert ca---",
3434
"CoreFrequency": "20s",
3535
"CoreAPIPort": 30050,
36+
"BaetylCoreByteUint": "KB",
37+
"BaetylCoreSpeedLimit": 0,
3638
context.KeyBaetylHostPathLib: "{{." + context.KeyBaetylHostPathLib + "}}",
3739
}
3840

@@ -139,6 +141,8 @@ data:
139141
sync:
140142
download:
141143
timeout: 30m
144+
speedLimit: 0
145+
byteUint: KB
142146
report:
143147
interval: 20s
144148
httplink:

0 commit comments

Comments
 (0)