Skip to content

Commit 18a90db

Browse files
BSWANGBSWANG
authored
Merge pull request #520 from l1b0k/feat/daemon
optimize daemon setup
2 parents 32f66fb + abfdaab commit 18a90db

File tree

9 files changed

+395
-334
lines changed

9 files changed

+395
-334
lines changed

‎daemon/config.go‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
package daemon
22

3+
import (
4+
"github.com/AliyunContainerService/terway/pkg/aliyun"
5+
"github.com/AliyunContainerService/terway/pkg/utils"
6+
"github.com/AliyunContainerService/terway/types"
7+
"github.com/AliyunContainerService/terway/types/daemon"
8+
)
9+
310
// getDynamicConfig returns (config, label, error) specified in node
411
// ("", "", nil) for no dynamic config for this node
512
func getDynamicConfig(k8s Kubernetes) (string, string, error) {
@@ -12,3 +19,122 @@ func getDynamicConfig(k8s Kubernetes) (string, string, error) {
1219

1320
return cfg, label, err
1421
}
22+
23+
// the actual size for pool is minIdle and maxIdle
24+
func getPoolConfig(cfg *daemon.Config, daemonMode string, limit *aliyun.Limits) (*types.PoolConfig, error) {
25+
poolConfig := &types.PoolConfig{
26+
SecurityGroupIDs: cfg.GetSecurityGroups(),
27+
VSwitchSelectionPolicy: cfg.VSwitchSelectionPolicy,
28+
DisableSecurityGroupCheck: cfg.DisableSecurityGroupCheck,
29+
}
30+
if cfg.ENITags == nil {
31+
cfg.ENITags = make(map[string]string)
32+
}
33+
cfg.ENITags[types.NetworkInterfaceTagCreatorKey] = types.NetworkInterfaceTagCreatorValue
34+
35+
poolConfig.ENITags = cfg.ENITags
36+
37+
capacity := 0
38+
maxENI := 0
39+
maxMemberENI := 0
40+
41+
switch daemonMode {
42+
case daemonModeVPC, daemonModeENIOnly:
43+
maxENI = limit.Adapters
44+
maxENI = int(float64(maxENI)*cfg.EniCapRatio) + cfg.EniCapShift - 1
45+
46+
// set max eni node can use
47+
if cfg.MaxENI > 0 && cfg.MaxENI < maxENI {
48+
maxENI = cfg.MaxENI
49+
}
50+
51+
capacity = maxENI
52+
53+
if cfg.MaxPoolSize > maxENI {
54+
poolConfig.MaxPoolSize = maxENI
55+
} else {
56+
poolConfig.MaxPoolSize = cfg.MaxPoolSize
57+
}
58+
59+
if cfg.MinENI > 0 {
60+
poolConfig.MinPoolSize = cfg.MinENI
61+
}
62+
63+
if poolConfig.MinPoolSize > poolConfig.MaxPoolSize {
64+
poolConfig.MinPoolSize = poolConfig.MaxPoolSize
65+
}
66+
67+
maxMemberENI = limit.MemberAdapterLimit
68+
if cfg.ENICapPolicy == types.ENICapPolicyPreferTrunk {
69+
maxMemberENI = limit.MaxMemberAdapterLimit
70+
}
71+
72+
poolConfig.MaxIPPerENI = 1
73+
case daemonModeENIMultiIP:
74+
maxENI = limit.Adapters
75+
maxENI = int(float64(maxENI)*cfg.EniCapRatio) + cfg.EniCapShift - 1
76+
77+
// set max eni node can use
78+
if cfg.MaxENI > 0 && cfg.MaxENI < maxENI {
79+
maxENI = cfg.MaxENI
80+
}
81+
82+
ipPerENI := limit.IPv4PerAdapter
83+
if utils.IsWindowsOS() {
84+
// NB(thxCode): don't assign the primary IP of one assistant eni.
85+
ipPerENI--
86+
}
87+
88+
capacity = maxENI * ipPerENI
89+
if cfg.MaxPoolSize > capacity {
90+
poolConfig.MaxPoolSize = capacity
91+
} else {
92+
poolConfig.MaxPoolSize = cfg.MaxPoolSize
93+
}
94+
95+
if cfg.MinENI > 0 {
96+
poolConfig.MinPoolSize = cfg.MinENI * ipPerENI
97+
}
98+
if poolConfig.MinPoolSize > poolConfig.MaxPoolSize {
99+
poolConfig.MinPoolSize = poolConfig.MaxPoolSize
100+
}
101+
102+
maxMemberENI = limit.MemberAdapterLimit
103+
104+
poolConfig.MaxIPPerENI = ipPerENI
105+
}
106+
107+
poolConfig.ENITags = cfg.ENITags
108+
109+
requireMeta := true
110+
if cfg.IPAMType == types.IPAMTypeCRD {
111+
poolConfig.MaxPoolSize = 0
112+
poolConfig.MinPoolSize = 0
113+
114+
if cfg.DisableDevicePlugin {
115+
requireMeta = false
116+
}
117+
}
118+
119+
if requireMeta {
120+
ins := aliyun.GetInstanceMeta()
121+
zone := ins.ZoneID
122+
if cfg.VSwitches != nil {
123+
zoneVswitchs, ok := cfg.VSwitches[zone]
124+
if ok && len(zoneVswitchs) > 0 {
125+
poolConfig.VSwitchOptions = cfg.VSwitches[zone]
126+
}
127+
}
128+
if len(poolConfig.VSwitchOptions) == 0 {
129+
poolConfig.VSwitchOptions = []string{ins.VSwitchID}
130+
}
131+
poolConfig.ZoneID = zone
132+
poolConfig.InstanceID = ins.InstanceID
133+
}
134+
135+
poolConfig.Capacity = capacity
136+
poolConfig.MaxENI = maxENI
137+
poolConfig.MaxMemberENI = maxMemberENI
138+
139+
return poolConfig, nil
140+
}

‎daemon/daemon.go‎

Lines changed: 110 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,13 @@ import (
1212
"sync"
1313
"time"
1414

15+
"github.com/AliyunContainerService/terway/deviceplugin"
1516
"github.com/AliyunContainerService/terway/pkg/aliyun"
1617
"github.com/AliyunContainerService/terway/pkg/aliyun/client"
1718
"github.com/AliyunContainerService/terway/pkg/aliyun/credential"
1819
podENITypes "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1"
1920
"github.com/AliyunContainerService/terway/pkg/backoff"
21+
vswpool "github.com/AliyunContainerService/terway/pkg/controller/vswitch"
2022
terwayIP "github.com/AliyunContainerService/terway/pkg/ip"
2123
"github.com/AliyunContainerService/terway/pkg/link"
2224
"github.com/AliyunContainerService/terway/pkg/logger"
@@ -1335,7 +1337,7 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) (
13351337
if ipFamily.IPv6 {
13361338
if !limit.SupportIPv6() {
13371339
ipFamily.IPv6 = false
1338-
serviceLog.Warnf("instance %s is not support ipv6", aliyun.GetInstanceMeta().InstanceType)
1340+
serviceLog.Warnf("instance %s is not support ipv6", config.InstanceType)
13391341
} else if daemonMode == daemonModeENIMultiIP && !limit.SupportMultiIPIPv6() {
13401342
ipFamily.IPv6 = false
13411343
serviceLog.Warnf("instance %s is not support ipv6", config.InstanceType)
@@ -1344,22 +1346,6 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) (
13441346

13451347
ecs := aliyun.NewAliyunImpl(aliyunClient, config.EnableENITrunking && !config.WaitTrunkENI, ipFamily, config.ENITagFilter)
13461348

1347-
netSrv.enableTrunk = config.EnableENITrunking
1348-
1349-
ipNetSet := &types.IPNetSet{}
1350-
if config.ServiceCIDR != "" {
1351-
cidrs := strings.Split(config.ServiceCIDR, ",")
1352-
1353-
for _, cidr := range cidrs {
1354-
ipNetSet.SetIPNet(cidr)
1355-
}
1356-
}
1357-
1358-
err = netSrv.k8s.SetSvcCidr(ipNetSet)
1359-
if err != nil {
1360-
return nil, errors.Wrapf(err, "error set k8s svcCidr")
1361-
}
1362-
13631349
netSrv.resourceDB, err = storage.NewDiskStorage(
13641350
resDBName, utils.NormalizePath(resDBPath), json.Marshal, func(bytes []byte) (interface{}, error) {
13651351
resourceRel := &types.PodResources{}
@@ -1373,13 +1359,108 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) (
13731359
return nil, errors.Wrapf(err, "error init resource manager storage")
13741360
}
13751361

1362+
nodeAnnotations := map[string]string{}
1363+
13761364
// get pool config
1377-
poolConfig, err := getPoolConfig(config)
1365+
poolConfig, err := getPoolConfig(config, daemonMode, limit)
13781366
if err != nil {
13791367
return nil, errors.Wrapf(err, "error get pool config")
13801368
}
13811369
serviceLog.Infof("init pool config: %+v", poolConfig)
13821370

1371+
vswPool, err := vswpool.NewSwitchPool(100, "10m")
1372+
if err != nil {
1373+
return nil, fmt.Errorf("error init vsw pool, %w", err)
1374+
}
1375+
1376+
// init trunk
1377+
if config.EnableENITrunking {
1378+
preferTrunkID := netSrv.k8s.GetTrunkID()
1379+
if preferTrunkID == "" && config.WaitTrunkENI {
1380+
preferTrunkID, err = netSrv.k8s.WaitTrunkReady()
1381+
if err != nil {
1382+
return nil, fmt.Errorf("error wait trunk ready, %w", err)
1383+
}
1384+
}
1385+
1386+
if !config.WaitTrunkENI {
1387+
enis, err := ecs.GetAttachedENIs(ctx, false, preferTrunkID)
1388+
if err != nil {
1389+
return nil, fmt.Errorf("error get attached eni, %w", err)
1390+
}
1391+
found := false
1392+
for _, eni := range enis {
1393+
if eni.Trunk && eni.ID == preferTrunkID {
1394+
found = true
1395+
1396+
poolConfig.TrunkENIID = preferTrunkID
1397+
netSrv.enableTrunk = true
1398+
1399+
nodeAnnotations[types.TrunkOn] = preferTrunkID
1400+
nodeAnnotations[string(types.MemberENIIPTypeIPs)] = strconv.Itoa(poolConfig.MaxMemberENI)
1401+
break
1402+
}
1403+
}
1404+
if !found {
1405+
if poolConfig.MaxENI > len(enis) {
1406+
vsw, err := vswPool.GetOne(ctx, ecs, poolConfig.ZoneID, poolConfig.VSwitchOptions, &vswpool.SelectOptions{
1407+
VSwitchSelectPolicy: vswpool.VSwitchSelectionPolicyMost,
1408+
})
1409+
if err != nil {
1410+
return nil, fmt.Errorf("error get vsw, %w", err)
1411+
}
1412+
1413+
eni, err := ecs.AllocateENI(ctx, vsw.ID, poolConfig.SecurityGroupIDs, poolConfig.InstanceID, true, 1, poolConfig.ENITags)
1414+
if err != nil {
1415+
return nil, fmt.Errorf("error allocate eni, %w", err)
1416+
}
1417+
1418+
poolConfig.TrunkENIID = eni.ID
1419+
netSrv.enableTrunk = true
1420+
1421+
nodeAnnotations[types.TrunkOn] = eni.ID
1422+
nodeAnnotations[string(types.MemberENIIPTypeIPs)] = strconv.Itoa(poolConfig.MaxMemberENI)
1423+
} else {
1424+
serviceLog.Warnf("no trunk eni found, fallback to non-trunk mode")
1425+
1426+
config.EnableENITrunking = false
1427+
config.DisableDevicePlugin = true
1428+
}
1429+
}
1430+
} else {
1431+
// WaitTrunkENI enabled, we believe what we got.
1432+
poolConfig.TrunkENIID = preferTrunkID
1433+
netSrv.enableTrunk = true
1434+
1435+
nodeAnnotations[types.TrunkOn] = preferTrunkID
1436+
nodeAnnotations[string(types.MemberENIIPTypeIPs)] = strconv.Itoa(poolConfig.MaxMemberENI)
1437+
}
1438+
}
1439+
1440+
if daemonMode != daemonModeVPC {
1441+
nodeAnnotations[string(types.NormalIPTypeIPs)] = strconv.Itoa(poolConfig.Capacity)
1442+
}
1443+
1444+
if !(daemonMode == daemonModeENIMultiIP && !config.EnableENITrunking) {
1445+
if !config.DisableDevicePlugin {
1446+
res := deviceplugin.ENITypeENI
1447+
capacity := poolConfig.MaxENI
1448+
if config.EnableENITrunking {
1449+
res = deviceplugin.ENITypeMember
1450+
capacity = poolConfig.MaxMemberENI
1451+
}
1452+
1453+
dp := deviceplugin.NewENIDevicePlugin(capacity, res)
1454+
go dp.Serve()
1455+
}
1456+
}
1457+
1458+
// ensure node annotations
1459+
err = netSrv.k8s.PatchNodeAnnotations(nodeAnnotations)
1460+
if err != nil {
1461+
return nil, fmt.Errorf("error patch node annotations, %w", err)
1462+
}
1463+
13831464
localResource := make(map[string]map[string]resourceManagerInitItem)
13841465
resObjList, err := netSrv.resourceDB.List()
13851466
if err != nil {
@@ -1454,16 +1535,18 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) (
14541535
panic("unsupported daemon mode" + daemonMode)
14551536
}
14561537

1457-
//start gc loop
1458-
netSrv.startGarbageCollectionLoop()
1459-
period := poolCheckPeriod
1460-
periodCfg := os.Getenv("POOL_CHECK_PERIOD_SECONDS")
1461-
periodSeconds, err := strconv.Atoi(periodCfg)
1462-
if err == nil {
1463-
period = time.Duration(periodSeconds) * time.Second
1464-
}
1538+
if config.IPAMType != types.IPAMTypeCRD {
1539+
//start gc loop
1540+
netSrv.startGarbageCollectionLoop()
1541+
period := poolCheckPeriod
1542+
periodCfg := os.Getenv("POOL_CHECK_PERIOD_SECONDS")
1543+
periodSeconds, err := strconv.Atoi(periodCfg)
1544+
if err == nil {
1545+
period = time.Duration(periodSeconds) * time.Second
1546+
}
14651547

1466-
go wait.JitterUntil(netSrv.startPeriodCheck, period, 1, true, wait.NeverStop)
1548+
go wait.JitterUntil(netSrv.startPeriodCheck, period, 1, true, wait.NeverStop)
1549+
}
14671550

14681551
// register for tracing
14691552
_ = tracing.Register(tracing.ResourceTypeNetworkService, "default", netSrv)
@@ -1473,49 +1556,6 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) (
14731556
return netSrv, nil
14741557
}
14751558

1476-
func getPoolConfig(cfg *daemon.Config) (*types.PoolConfig, error) {
1477-
poolConfig := &types.PoolConfig{
1478-
MaxPoolSize: cfg.MaxPoolSize,
1479-
MinPoolSize: cfg.MinPoolSize,
1480-
MaxENI: cfg.MaxENI,
1481-
MinENI: cfg.MinENI,
1482-
EniCapRatio: cfg.EniCapRatio,
1483-
EniCapShift: cfg.EniCapShift,
1484-
SecurityGroups: cfg.GetSecurityGroups(),
1485-
VSwitchSelectionPolicy: cfg.VSwitchSelectionPolicy,
1486-
EnableENITrunking: cfg.EnableENITrunking,
1487-
ENICapPolicy: cfg.ENICapPolicy,
1488-
DisableDevicePlugin: cfg.DisableDevicePlugin,
1489-
WaitTrunkENI: cfg.WaitTrunkENI,
1490-
DisableSecurityGroupCheck: cfg.DisableSecurityGroupCheck,
1491-
}
1492-
if len(poolConfig.SecurityGroups) > 5 {
1493-
return nil, fmt.Errorf("security groups should not be more than 5, current %d", len(poolConfig.SecurityGroups))
1494-
}
1495-
ins := aliyun.GetInstanceMeta()
1496-
zone := ins.ZoneID
1497-
if cfg.VSwitches != nil {
1498-
zoneVswitchs, ok := cfg.VSwitches[zone]
1499-
if ok && len(zoneVswitchs) > 0 {
1500-
poolConfig.VSwitch = cfg.VSwitches[zone]
1501-
}
1502-
}
1503-
if len(poolConfig.VSwitch) == 0 {
1504-
poolConfig.VSwitch = []string{ins.VSwitchID}
1505-
}
1506-
poolConfig.ENITags = cfg.ENITags
1507-
poolConfig.VPC = ins.VPCID
1508-
poolConfig.InstanceID = ins.InstanceID
1509-
1510-
if cfg.IPAMType == types.IPAMTypeCRD {
1511-
poolConfig.MaxPoolSize = 0
1512-
poolConfig.MinPoolSize = 0
1513-
poolConfig.MaxENI = 0
1514-
poolConfig.MinENI = 0
1515-
}
1516-
return poolConfig, nil
1517-
}
1518-
15191559
func parseExtraRoute(routes []podENITypes.Route) []*rpc.Route {
15201560
if routes == nil {
15211561
return nil

0 commit comments

Comments
 (0)