Skip to content

Commit a3ed686

Browse files
authored
If service discovery from the registry is not used, the architecture allows decoupling from the Higress Controller. (higress-group#506)
1 parent e26558b commit a3ed686

File tree

10 files changed

+298
-31
lines changed

10 files changed

+298
-31
lines changed

backend/console/src/main/java/com/alibaba/higress/console/config/SdkConfig.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,9 @@ public class SdkConfig {
7070
@Value("${" + SystemConfigKey.CONTROLLER_ACCESS_TOKEN_KEY + ":}")
7171
private String controllerAccessToken;
7272

73+
@Value("${" + SystemConfigKey.CLUSTER_DOMAIN_SUFFIX + ":" + HigressConstants.CLUSTER_DOMAIN_SUFFIX_DEFAULT + "}")
74+
private String clusterDomainSuffix;
75+
7376
private HigressServiceProvider serviceProvider;
7477

7578
@PostConstruct
@@ -81,6 +84,7 @@ public void initialize() throws IOException {
8184
.withControllerServiceName(controllerServiceName).withControllerServiceHost(controllerServiceHost)
8285
.withControllerServicePort(controllerServicePort).withControllerJwtPolicy(controllerJwtPolicy)
8386
.withControllerAccessToken(controllerAccessToken)
87+
.withClusterDomainSuffix(clusterDomainSuffix)
8488
.withWasmPluginServiceConfig(WasmPluginServiceConfig.buildFromEnv()).build();
8589
serviceProvider = HigressServiceProvider.create(config);
8690
}

backend/console/src/main/java/com/alibaba/higress/console/constant/SystemConfigKey.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,4 +116,6 @@ public class SystemConfigKey {
116116
public static final String AI_PROXY_SOCKET_TIMEOUT_KEY = CONFIG_KEY_PREFIX + "ai-proxy.socket-timeout";
117117

118118
public static final int AI_PROXY_SOCKET_TIMEOUT_DEFAULT = 2 * 60 * 1000;
119+
120+
public static final String CLUSTER_DOMAIN_SUFFIX = "CLUSTER_DOMAIN_SUFFIX";
119121
}

backend/sdk/src/main/java/com/alibaba/higress/sdk/config/HigressServiceConfig.java

Lines changed: 43 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,15 @@
1212
*/
1313
package com.alibaba.higress.sdk.config;
1414

15+
import java.util.Objects;
16+
import java.util.Optional;
17+
18+
import org.apache.commons.lang3.StringUtils;
19+
1520
import com.alibaba.higress.sdk.constant.HigressConstants;
1621
import com.alibaba.higress.sdk.model.wasmplugin.WasmPluginServiceConfig;
17-
import lombok.Data;
18-
import org.apache.commons.lang3.StringUtils;
1922

20-
import java.util.Objects;
21-
import java.util.Optional;
23+
import lombok.Data;
2224

2325
/**
2426
* @author CH3CHO
@@ -36,6 +38,26 @@ public class HigressServiceConfig {
3638
private final String controllerJwtPolicy;
3739
private final String controllerAccessToken;
3840
private final WasmPluginServiceConfig wasmPluginServiceConfig;
41+
/**
42+
* Does the service list interface support registry?
43+
*
44+
* <p>
45+
* If null, use the default value. {@link HigressConstants#SERVICE_LIST_SUPPORT_REGISTRY_DEFAULT}
46+
* </p>
47+
* <p>
48+
* If true, the service list interface will support registry and depend on the controller API.
49+
* {@link com.alibaba.higress.sdk.service.ServiceServiceImpl}
50+
* </p>
51+
* <p>
52+
* If false, the service list implementation will not support registry and will interact with the API server
53+
* directly. {@link com.alibaba.higress.sdk.service.ServiceServiceByApiServerImpl} Under the current configuration,
54+
* there is no need for the capability of service discovery. The sources related to the registry in the service
55+
* sources will be disabled
56+
* </p>
57+
*
58+
*/
59+
private final Boolean serviceListSupportRegistry;
60+
private final String clusterDomainSuffix;
3961

4062
/**
4163
* @deprecated use {@link #getControllerWatchedIngressClassName()} instead
@@ -60,15 +82,26 @@ public static final class Builder {
6082
private String controllerJwtPolicy = HigressConstants.CONTROLLER_JWT_POLICY_DEFAULT;
6183
private String controllerAccessToken;
6284
private WasmPluginServiceConfig wasmPluginServiceConfig;
85+
private Boolean serviceListSupportRegistry;
86+
private String clusterDomainSuffix;
6387

64-
private Builder() {
65-
}
88+
private Builder() {}
6689

6790
public Builder withWasmPluginServiceConfig(WasmPluginServiceConfig wasmPluginServiceConfig) {
6891
this.wasmPluginServiceConfig = wasmPluginServiceConfig;
6992
return this;
7093
}
7194

95+
public Builder withServiceListSupportRegistry(Boolean serviceListSupportRegistry) {
96+
this.serviceListSupportRegistry = serviceListSupportRegistry;
97+
return this;
98+
}
99+
100+
public Builder withClusterDomainSuffix(String clusterDomainSuffix) {
101+
this.clusterDomainSuffix = clusterDomainSuffix;
102+
return this;
103+
}
104+
72105
public Builder withKubeConfigPath(String kubeConfigPath) {
73106
this.kubeConfigPath = kubeConfigPath;
74107
return this;
@@ -131,7 +164,10 @@ public HigressServiceConfig build() {
131164
Optional.ofNullable(controllerServicePort).orElse(HigressConstants.CONTROLLER_SERVICE_PORT_DEFAULT),
132165
StringUtils.firstNonEmpty(controllerJwtPolicy, HigressConstants.CONTROLLER_JWT_POLICY_DEFAULT),
133166
controllerAccessToken,
134-
Objects.isNull(wasmPluginServiceConfig) ? new WasmPluginServiceConfig() : wasmPluginServiceConfig);
167+
Objects.isNull(wasmPluginServiceConfig) ? new WasmPluginServiceConfig() : wasmPluginServiceConfig,
168+
Optional.ofNullable(serviceListSupportRegistry)
169+
.orElse(HigressConstants.SERVICE_LIST_SUPPORT_REGISTRY_DEFAULT),
170+
StringUtils.firstNonEmpty(clusterDomainSuffix, HigressConstants.CLUSTER_DOMAIN_SUFFIX_DEFAULT));
135171
}
136172
}
137173
}

backend/sdk/src/main/java/com/alibaba/higress/sdk/constant/HigressConstants.java

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,10 @@
1212
*/
1313
package com.alibaba.higress.sdk.constant;
1414

15-
import com.google.common.collect.Sets;
16-
1715
import java.util.Set;
1816

17+
import com.google.common.collect.Sets;
18+
1919
public class HigressConstants {
2020
public static final String NS_DEFAULT = "higress-system";
2121
public static final String CONTROLLER_SERVICE_NAME_DEFAULT = "higress-controller";
@@ -29,6 +29,9 @@ public class HigressConstants {
2929
public static final String FALLBACK_ROUTE_NAME_SUFFIX = ".fallback";
3030
public static final String FALLBACK_FROM_HEADER = "x-higress-fallback-from";
3131
public static final String MODEL_ROUTING_HEADER = "x-higress-llm-model";
32-
public static final String INTERNAL_RESOURCE_COMMENT = "PLEASE DO NOT EDIT DIRECTLY. This resource is managed by Higress.";
32+
public static final String INTERNAL_RESOURCE_COMMENT =
33+
"PLEASE DO NOT EDIT DIRECTLY. This resource is managed by Higress.";
3334
public static final Set<String> VALID_FALLBACK_RESPONSE_CODES = Sets.newHashSet("4xx", "5xx");
35+
public static final Boolean SERVICE_LIST_SUPPORT_REGISTRY_DEFAULT = Boolean.TRUE;
36+
public static final String CLUSTER_DOMAIN_SUFFIX_DEFAULT = "cluster.local";
3437
}

backend/sdk/src/main/java/com/alibaba/higress/sdk/model/Service.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,4 +34,6 @@ public class Service {
3434
private Integer version;
3535

3636
private List<String> endpoints;
37+
38+
private String protocol;
3739
}

backend/sdk/src/main/java/com/alibaba/higress/sdk/service/HigressServiceProviderImpl.java

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@
1212
*/
1313
package com.alibaba.higress.sdk.service;
1414

15+
import java.io.IOException;
16+
17+
import org.apache.commons.lang3.BooleanUtils;
18+
1519
import com.alibaba.higress.sdk.config.HigressServiceConfig;
1620
import com.alibaba.higress.sdk.service.ai.AiRouteService;
1721
import com.alibaba.higress.sdk.service.ai.AiRouteServiceImpl;
@@ -22,8 +26,6 @@
2226
import com.alibaba.higress.sdk.service.kubernetes.KubernetesClientService;
2327
import com.alibaba.higress.sdk.service.kubernetes.KubernetesModelConverter;
2428

25-
import java.io.IOException;
26-
2729
/**
2830
* @author CH3CHO
2931
*/
@@ -45,23 +47,25 @@ class HigressServiceProviderImpl implements HigressServiceProvider {
4547
HigressServiceProviderImpl(HigressServiceConfig config) throws IOException {
4648
kubernetesClientService = new KubernetesClientService(config);
4749
kubernetesModelConverter = new KubernetesModelConverter(kubernetesClientService);
48-
serviceService = new ServiceServiceImpl(kubernetesClientService);
50+
if (BooleanUtils.isTrue(config.getServiceListSupportRegistry())) {
51+
serviceService = new ServiceServiceImpl(kubernetesClientService);
52+
} else {
53+
serviceService = new ServiceServiceByApiServerImpl(kubernetesClientService, kubernetesModelConverter);
54+
}
4955
serviceSourceService = new ServiceSourceServiceImpl(kubernetesClientService, kubernetesModelConverter);
5056
tlsCertificateService = new TlsCertificateServiceImpl(kubernetesClientService, kubernetesModelConverter);
5157
wasmPluginService = new WasmPluginServiceImpl(kubernetesClientService, kubernetesModelConverter,
5258
config.getWasmPluginServiceConfig());
5359
wasmPluginInstanceService =
5460
new WasmPluginInstanceServiceImpl(wasmPluginService, kubernetesClientService, kubernetesModelConverter);
5561
consumerService = new ConsumerServiceImpl(wasmPluginInstanceService);
56-
routeService =
57-
new RouteServiceImpl(kubernetesClientService, kubernetesModelConverter, wasmPluginInstanceService,
58-
consumerService);
62+
routeService = new RouteServiceImpl(kubernetesClientService, kubernetesModelConverter,
63+
wasmPluginInstanceService, consumerService);
5964
domainService = new DomainServiceImpl(kubernetesClientService, kubernetesModelConverter, routeService,
6065
wasmPluginInstanceService);
6166
llmProviderService = new LlmProviderServiceImpl(serviceSourceService, wasmPluginInstanceService);
62-
aiRouteService =
63-
new AiRouteServiceImpl(kubernetesModelConverter, kubernetesClientService, routeService, llmProviderService,
64-
wasmPluginInstanceService);
67+
aiRouteService = new AiRouteServiceImpl(kubernetesModelConverter, kubernetesClientService, routeService,
68+
llmProviderService, wasmPluginInstanceService);
6569
}
6670

6771
@Override
Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
/*
2+
* Copyright (c) 2022-2023 Alibaba Group Holding Ltd.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
5+
* the License. You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
10+
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
11+
* specific language governing permissions and limitations under the License.
12+
*/
13+
package com.alibaba.higress.sdk.service;
14+
15+
import java.util.Arrays;
16+
import java.util.Comparator;
17+
import java.util.List;
18+
import java.util.Map;
19+
import java.util.Objects;
20+
import java.util.function.Function;
21+
import java.util.stream.Collectors;
22+
23+
import org.apache.commons.collections4.CollectionUtils;
24+
import org.apache.commons.lang3.StringUtils;
25+
26+
import com.alibaba.higress.sdk.constant.Separators;
27+
import com.alibaba.higress.sdk.model.CommonPageQuery;
28+
import com.alibaba.higress.sdk.model.PaginatedResult;
29+
import com.alibaba.higress.sdk.model.Service;
30+
import com.alibaba.higress.sdk.service.kubernetes.KubernetesClientService;
31+
import com.alibaba.higress.sdk.service.kubernetes.KubernetesModelConverter;
32+
import com.alibaba.higress.sdk.service.kubernetes.crd.mcp.V1McpBridge;
33+
import com.alibaba.higress.sdk.service.kubernetes.crd.mcp.V1RegistryConfig;
34+
import com.google.common.collect.Lists;
35+
import com.google.common.collect.Maps;
36+
37+
import io.kubernetes.client.openapi.models.V1EndpointAddress;
38+
import io.kubernetes.client.openapi.models.V1Endpoints;
39+
import io.kubernetes.client.openapi.models.V1ObjectMeta;
40+
import io.kubernetes.client.openapi.models.V1Service;
41+
import lombok.SneakyThrows;
42+
import lombok.extern.slf4j.Slf4j;
43+
44+
/**
45+
* @author lvshui
46+
*/
47+
@Slf4j
48+
public class ServiceServiceByApiServerImpl implements ServiceService {
49+
public static final String MCP_SIMULATE_NS = "mcp";
50+
private final KubernetesClientService kubernetesClientService;
51+
private final KubernetesModelConverter kubernetesModelConverter;
52+
53+
public ServiceServiceByApiServerImpl(KubernetesClientService kubernetesClientService,
54+
KubernetesModelConverter kubernetesModelConverter) {
55+
this.kubernetesClientService = kubernetesClientService;
56+
this.kubernetesModelConverter = kubernetesModelConverter;
57+
}
58+
59+
@SneakyThrows
60+
@Override
61+
public PaginatedResult<Service> list(CommonPageQuery query) {
62+
List<Service> resultList = Lists.newArrayList();
63+
64+
List<V1Service> v1Services = kubernetesClientService.listAllServiceList();
65+
List<V1Endpoints> v1Endpoints = kubernetesClientService.listAllEndPointsList();
66+
final Map<String, V1Endpoints> v1EndpointsMap = Maps.newHashMap();
67+
if (CollectionUtils.isNotEmpty(v1Endpoints)) {
68+
Map<String, V1Endpoints> v1EndpointsMap1 = v1Endpoints.stream().collect(
69+
Collectors.toMap(i -> buildMetaUniqueKey(i.getMetadata()), Function.identity(), (k1, k2) -> k1));
70+
v1EndpointsMap.putAll(v1EndpointsMap1);
71+
}
72+
73+
if (CollectionUtils.isNotEmpty(v1Services)) {
74+
for (V1Service v1Service : v1Services) {
75+
try {
76+
Service service = kubernetesModelConverter.v1Service2Service(v1Service);
77+
String metaUniqueKey = buildMetaUniqueKey(v1Service.getMetadata());
78+
V1Endpoints v1Endpoints1 = null;
79+
if (v1EndpointsMap.containsKey(metaUniqueKey)) {
80+
v1Endpoints1 = v1EndpointsMap.get(metaUniqueKey);
81+
}
82+
if (Objects.nonNull(v1Endpoints1) && CollectionUtils.isNotEmpty(v1Endpoints1.getSubsets())) {
83+
try {
84+
List<String> ipList =
85+
v1Endpoints1.getSubsets().stream().filter(Objects::nonNull).map(v1Subset -> {
86+
if (v1Subset.getAddresses() != null) {
87+
return v1Subset.getAddresses().stream().map(V1EndpointAddress::getIp)
88+
.collect(Collectors.toList());
89+
}
90+
return null;
91+
}).filter(CollectionUtils::isNotEmpty).flatMap(List::stream)
92+
.collect(Collectors.toList());
93+
service.setEndpoints(ipList);
94+
} catch (Exception e) {
95+
log.error("deal service endpoints appear error. ", e);
96+
}
97+
}
98+
resultList.add(service);
99+
} catch (Exception e) {
100+
log.error("deal service appear error. ", e);
101+
}
102+
}
103+
}
104+
105+
List<V1McpBridge> v1McpBridges = kubernetesClientService.listMcpBridge();
106+
if (CollectionUtils.isNotEmpty(v1McpBridges)) {
107+
List<Service> externalServiceList = v1McpBridges.stream().map(i -> {
108+
List<V1RegistryConfig> registries = i.getSpec().getRegistries();
109+
if (CollectionUtils.isEmpty(registries)) {
110+
return null;
111+
}
112+
return registries.stream().map(r -> {
113+
Service service = new Service();
114+
service.setNamespace(MCP_SIMULATE_NS);
115+
service.setName(StringUtils.join(r.getName(), Separators.DOT, r.getType()));
116+
service.setProtocol(r.getProtocol());
117+
String[] endPoints = new String[] {};
118+
if (StringUtils.isNotBlank(r.getDomain())) {
119+
endPoints = StringUtils.split(r.getDomain(), V1McpBridge.REGISTRY_TYPE_STATIC_DNS_SEPARATOR);
120+
}
121+
service.setEndpoints(Arrays.asList(endPoints));
122+
switch (r.getType()) {
123+
case V1McpBridge.REGISTRY_TYPE_DNS:
124+
service.setPort(r.getPort());
125+
return service;
126+
case V1McpBridge.REGISTRY_TYPE_STATIC:
127+
service.setPort(V1McpBridge.STATIC_PORT);
128+
return service;
129+
default:
130+
return null;
131+
}
132+
}).filter(Objects::nonNull).collect(Collectors.toList());
133+
}).filter(CollectionUtils::isNotEmpty).flatMap(List::stream).collect(Collectors.toList());
134+
135+
if (CollectionUtils.isNotEmpty(externalServiceList)) {
136+
resultList.addAll(externalServiceList);
137+
}
138+
139+
}
140+
resultList.sort(Comparator.comparing(Service::getNamespace).thenComparing(Service::getName)
141+
.thenComparing(Service::getPort));
142+
143+
return PaginatedResult.createFromFullList(resultList, query);
144+
}
145+
146+
private String buildMetaUniqueKey(V1ObjectMeta metadata) {
147+
return StringUtils.join(metadata.getNamespace(), ".", metadata.getName());
148+
}
149+
}

0 commit comments

Comments
 (0)