diff --git a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java index 688626d9b3d..67c5626aff5 100644 --- a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java +++ b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java @@ -19,20 +19,31 @@ import static com.google.common.base.Preconditions.checkNotNull; import static io.grpc.ConnectivityState.TRANSIENT_FAILURE; import static io.grpc.xds.XdsLbPolicies.CDS_POLICY_NAME; -import static io.grpc.xds.XdsLbPolicies.CLUSTER_RESOLVER_POLICY_NAME; import static io.grpc.xds.XdsLbPolicies.PRIORITY_POLICY_NAME; +import com.google.common.collect.ImmutableMap; import com.google.errorprone.annotations.CheckReturnValue; +import io.grpc.Attributes; +import io.grpc.EquivalentAddressGroup; +import io.grpc.HttpConnectProxiedSocketAddress; import io.grpc.InternalLogId; import io.grpc.LoadBalancer; +import io.grpc.LoadBalancerProvider; import io.grpc.LoadBalancerRegistry; import io.grpc.NameResolver; import io.grpc.Status; import io.grpc.StatusOr; import io.grpc.util.GracefulSwitchLoadBalancer; +import io.grpc.util.OutlierDetectionLoadBalancer.OutlierDetectionLoadBalancerConfig; import io.grpc.xds.CdsLoadBalancerProvider.CdsConfig; -import io.grpc.xds.ClusterResolverLoadBalancerProvider.ClusterResolverConfig; -import io.grpc.xds.ClusterResolverLoadBalancerProvider.ClusterResolverConfig.DiscoveryMechanism; +import io.grpc.xds.ClusterImplLoadBalancerProvider.ClusterImplConfig; +import io.grpc.xds.Endpoints.DropOverload; +import io.grpc.xds.Endpoints.LbEndpoint; +import io.grpc.xds.Endpoints.LocalityLbEndpoints; +import io.grpc.xds.EnvoyServerProtoData.FailurePercentageEjection; +import io.grpc.xds.EnvoyServerProtoData.OutlierDetection; +import io.grpc.xds.EnvoyServerProtoData.SuccessRateEjection; +import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig; import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig.PriorityChildConfig; import io.grpc.xds.XdsClusterResource.CdsUpdate; import io.grpc.xds.XdsClusterResource.CdsUpdate.ClusterType; @@ -40,13 +51,22 @@ import io.grpc.xds.XdsConfig.XdsClusterConfig; import io.grpc.xds.XdsConfig.XdsClusterConfig.AggregateConfig; import io.grpc.xds.XdsConfig.XdsClusterConfig.EndpointConfig; +import io.grpc.xds.XdsEndpointResource.EdsUpdate; +import io.grpc.xds.client.Locality; import io.grpc.xds.client.XdsLogger; import io.grpc.xds.client.XdsLogger.XdsLogLevel; +import io.grpc.xds.internal.XdsInternalAttributes; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TreeMap; /** * Load balancer for cds_experimental LB policy. One instance per top-level cluster. @@ -57,6 +77,7 @@ final class CdsLoadBalancer2 extends LoadBalancer { private final XdsLogger logger; private final Helper helper; private final LoadBalancerRegistry lbRegistry; + private final ClusterState clusterState = new ClusterState(); private GracefulSwitchLoadBalancer delegate; // Following fields are effectively final. private String clusterName; @@ -100,7 +121,6 @@ public Status acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) { XdsClusterConfig clusterConfig = clusterConfigOr.getValue(); NameResolver.ConfigOrError configOrError; - Object gracefulConfig; if (clusterConfig.getChildren() instanceof EndpointConfig) { // The LB policy config is provided in service_config.proto/JSON format. configOrError = @@ -112,34 +132,38 @@ public Status acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) { return fail(Status.INTERNAL.withDescription( errorPrefix() + "Unable to parse the LB config: " + configOrError.getError())); } - CdsUpdate result = clusterConfig.getClusterResource(); - DiscoveryMechanism instance; - if (result.clusterType() == ClusterType.EDS) { - instance = DiscoveryMechanism.forEds( - clusterName, - result.edsServiceName(), - result.lrsServerInfo(), - result.maxConcurrentRequests(), - result.upstreamTlsContext(), - result.filterMetadata(), - result.outlierDetection(), - result.backendMetricPropagation()); - } else { - instance = DiscoveryMechanism.forLogicalDns( - clusterName, - result.dnsHostName(), - result.lrsServerInfo(), - result.maxConcurrentRequests(), - result.upstreamTlsContext(), - result.filterMetadata(), - result.backendMetricPropagation()); - } - gracefulConfig = GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig( - lbRegistry.getProvider(CLUSTER_RESOLVER_POLICY_NAME), - new ClusterResolverConfig( - instance, - configOrError.getConfig(), - clusterConfig.getClusterResource().isHttp11ProxyAvailable())); + + StatusOr edsUpdate = getEdsUpdate(xdsConfig, clusterName); + StatusOr statusOrResult = clusterState.edsUpdateToResult( + clusterName, + clusterConfig.getClusterResource(), + configOrError.getConfig(), + edsUpdate); + if (!statusOrResult.hasValue()) { + Status status = Status.UNAVAILABLE + .withDescription(statusOrResult.getStatus().getDescription()) + .withCause(statusOrResult.getStatus().getCause()); + delegate.handleNameResolutionError(status); + return status; + } + ClusterResolutionResult result = statusOrResult.getValue(); + List addresses = result.addresses; + if (addresses.isEmpty()) { + Status status = Status.UNAVAILABLE + .withDescription("No usable endpoint from cluster: " + clusterName); + delegate.handleNameResolutionError(status); + return status; + } + Object gracefulConfig = GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig( + lbRegistry.getProvider(PRIORITY_POLICY_NAME), + new PriorityLbConfig( + Collections.unmodifiableMap(result.priorityChildConfigs), + Collections.unmodifiableList(result.priorities))); + return delegate.acceptResolvedAddresses( + resolvedAddresses.toBuilder() + .setLoadBalancingPolicyConfig(gracefulConfig) + .setAddresses(Collections.unmodifiableList(addresses)) + .build()); } else if (clusterConfig.getChildren() instanceof AggregateConfig) { Map priorityChildConfigs = new HashMap<>(); List leafClusters = ((AggregateConfig) clusterConfig.getChildren()).getLeafNames(); @@ -151,18 +175,17 @@ public Status acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) { new CdsConfig(childCluster)), false)); } - gracefulConfig = GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig( + Object gracefulConfig = GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig( lbRegistry.getProvider(PRIORITY_POLICY_NAME), new PriorityLoadBalancerProvider.PriorityLbConfig( Collections.unmodifiableMap(priorityChildConfigs), leafClusters)); + return delegate.acceptResolvedAddresses( + resolvedAddresses.toBuilder().setLoadBalancingPolicyConfig(gracefulConfig).build()); } else { return fail(Status.INTERNAL.withDescription( errorPrefix() + "Unexpected cluster children type: " + clusterConfig.getChildren().getClass())); } - - return delegate.acceptResolvedAddresses( - resolvedAddresses.toBuilder().setLoadBalancingPolicyConfig(gracefulConfig).build()); } @Override @@ -198,4 +221,326 @@ private Status fail(Status error) { private String errorPrefix() { return "CdsLb for " + clusterName + ": "; } + + private static StatusOr getEdsUpdate(XdsConfig xdsConfig, String cluster) { + StatusOr clusterConfig = xdsConfig.getClusters().get(cluster); + if (clusterConfig == null) { + return StatusOr.fromStatus(Status.INTERNAL + .withDescription("BUG: cluster resolver could not find cluster in xdsConfig")); + } + if (!clusterConfig.hasValue()) { + return StatusOr.fromStatus(clusterConfig.getStatus()); + } + if (!(clusterConfig.getValue().getChildren() instanceof XdsClusterConfig.EndpointConfig)) { + return StatusOr.fromStatus(Status.INTERNAL + .withDescription("BUG: cluster resolver cluster with children of unknown type")); + } + XdsClusterConfig.EndpointConfig endpointConfig = + (XdsClusterConfig.EndpointConfig) clusterConfig.getValue().getChildren(); + return endpointConfig.getEndpoint(); + } + + /** + * Generates a string that represents the priority in the LB policy config. The string is unique + * across priorities in all clusters and priorityName(c, p1) < priorityName(c, p2) iff p1 < p2. + * The ordering is undefined for priorities in different clusters. + */ + private static String priorityName(String cluster, int priority) { + return cluster + "[child" + priority + "]"; + } + + /** + * Generates a string that represents the locality in the LB policy config. The string is unique + * across all localities in all clusters. + */ + private static String localityName(Locality locality) { + return "{region=\"" + locality.region() + + "\", zone=\"" + locality.zone() + + "\", sub_zone=\"" + locality.subZone() + + "\"}"; + } + + private final class ClusterState { + private Map localityPriorityNames = Collections.emptyMap(); + int priorityNameGenId = 1; + + StatusOr edsUpdateToResult( + String clusterName, + CdsUpdate discovery, + Object lbConfig, + StatusOr updateOr) { + if (!updateOr.hasValue()) { + return StatusOr.fromStatus(updateOr.getStatus()); + } + EdsUpdate update = updateOr.getValue(); + logger.log(XdsLogLevel.DEBUG, "Received endpoint update {0}", update); + if (logger.isLoggable(XdsLogLevel.INFO)) { + logger.log(XdsLogLevel.INFO, "Cluster {0}: {1} localities, {2} drop categories", + clusterName, update.localityLbEndpointsMap.size(), + update.dropPolicies.size()); + } + Map localityLbEndpoints = + update.localityLbEndpointsMap; + List dropOverloads = update.dropPolicies; + List addresses = new ArrayList<>(); + Map> prioritizedLocalityWeights = new HashMap<>(); + List sortedPriorityNames = + generatePriorityNames(clusterName, localityLbEndpoints); + for (Locality locality : localityLbEndpoints.keySet()) { + LocalityLbEndpoints localityLbInfo = localityLbEndpoints.get(locality); + String priorityName = localityPriorityNames.get(locality); + boolean discard = true; + for (LbEndpoint endpoint : localityLbInfo.endpoints()) { + if (endpoint.isHealthy()) { + discard = false; + long weight = localityLbInfo.localityWeight(); + if (endpoint.loadBalancingWeight() != 0) { + weight *= endpoint.loadBalancingWeight(); + } + String localityName = localityName(locality); + Attributes attr = + endpoint.eag().getAttributes().toBuilder() + .set(io.grpc.xds.XdsAttributes.ATTR_LOCALITY, locality) + .set(EquivalentAddressGroup.ATTR_LOCALITY_NAME, localityName) + .set(io.grpc.xds.XdsAttributes.ATTR_LOCALITY_WEIGHT, + localityLbInfo.localityWeight()) + .set(io.grpc.xds.XdsAttributes.ATTR_SERVER_WEIGHT, weight) + .set(XdsInternalAttributes.ATTR_ADDRESS_NAME, endpoint.hostname()) + .build(); + EquivalentAddressGroup eag; + if (discovery.isHttp11ProxyAvailable()) { + List rewrittenAddresses = new ArrayList<>(); + for (SocketAddress addr : endpoint.eag().getAddresses()) { + rewrittenAddresses.add(rewriteAddress( + addr, endpoint.endpointMetadata(), localityLbInfo.localityMetadata())); + } + eag = new EquivalentAddressGroup(rewrittenAddresses, attr); + } else { + eag = new EquivalentAddressGroup(endpoint.eag().getAddresses(), attr); + } + eag = AddressFilter.setPathFilter(eag, Arrays.asList(priorityName, localityName)); + addresses.add(eag); + } + } + if (discard) { + logger.log(XdsLogLevel.INFO, + "Discard locality {0} with 0 healthy endpoints", locality); + continue; + } + if (!prioritizedLocalityWeights.containsKey(priorityName)) { + prioritizedLocalityWeights.put(priorityName, new HashMap()); + } + prioritizedLocalityWeights.get(priorityName).put( + locality, localityLbInfo.localityWeight()); + } + if (prioritizedLocalityWeights.isEmpty()) { + // Will still update the result, as if the cluster resource is revoked. + logger.log(XdsLogLevel.INFO, + "Cluster {0} has no usable priority/locality/endpoint", clusterName); + } + sortedPriorityNames.retainAll(prioritizedLocalityWeights.keySet()); + Map priorityChildConfigs = + generatePriorityChildConfigs( + clusterName, discovery, lbConfig, lbRegistry, + prioritizedLocalityWeights, dropOverloads); + return StatusOr.fromValue(new ClusterResolutionResult(addresses, priorityChildConfigs, + sortedPriorityNames)); + } + + private SocketAddress rewriteAddress(SocketAddress addr, + ImmutableMap endpointMetadata, + ImmutableMap localityMetadata) { + if (!(addr instanceof InetSocketAddress)) { + return addr; + } + + SocketAddress proxyAddress; + try { + proxyAddress = (SocketAddress) endpointMetadata.get( + "envoy.http11_proxy_transport_socket.proxy_address"); + if (proxyAddress == null) { + proxyAddress = (SocketAddress) localityMetadata.get( + "envoy.http11_proxy_transport_socket.proxy_address"); + } + } catch (ClassCastException e) { + return addr; + } + + if (proxyAddress == null) { + return addr; + } + + return HttpConnectProxiedSocketAddress.newBuilder() + .setTargetAddress((InetSocketAddress) addr) + .setProxyAddress(proxyAddress) + .build(); + } + + private List generatePriorityNames(String name, + Map localityLbEndpoints) { + TreeMap> todo = new TreeMap<>(); + for (Locality locality : localityLbEndpoints.keySet()) { + int priority = localityLbEndpoints.get(locality).priority(); + if (!todo.containsKey(priority)) { + todo.put(priority, new ArrayList<>()); + } + todo.get(priority).add(locality); + } + Map newNames = new HashMap<>(); + Set usedNames = new HashSet<>(); + List ret = new ArrayList<>(); + for (Integer priority: todo.keySet()) { + String foundName = ""; + for (Locality locality : todo.get(priority)) { + if (localityPriorityNames.containsKey(locality) + && usedNames.add(localityPriorityNames.get(locality))) { + foundName = localityPriorityNames.get(locality); + break; + } + } + if ("".equals(foundName)) { + foundName = priorityName(name, priorityNameGenId++); + } + for (Locality locality : todo.get(priority)) { + newNames.put(locality, foundName); + } + ret.add(foundName); + } + localityPriorityNames = newNames; + return ret; + } + } + + private static class ClusterResolutionResult { + // Endpoint addresses. + private final List addresses; + // Config (include load balancing policy/config) for each priority in the cluster. + private final Map priorityChildConfigs; + // List of priority names ordered in descending priorities. + private final List priorities; + + ClusterResolutionResult(List addresses, + Map configs, List priorities) { + this.addresses = addresses; + this.priorityChildConfigs = configs; + this.priorities = priorities; + } + } + + /** + * Generates configs to be used in the priority LB policy for priorities in a cluster. + * + *

priority LB -> cluster_impl LB (one per priority) -> (weighted_target LB + * -> round_robin / least_request_experimental (one per locality)) / ring_hash_experimental + */ + private static Map generatePriorityChildConfigs( + String clusterName, + CdsUpdate discovery, + Object endpointLbConfig, + LoadBalancerRegistry lbRegistry, + Map> prioritizedLocalityWeights, + List dropOverloads) { + Map configs = new HashMap<>(); + for (String priority : prioritizedLocalityWeights.keySet()) { + ClusterImplConfig clusterImplConfig = + new ClusterImplConfig( + clusterName, discovery.edsServiceName(), discovery.lrsServerInfo(), + discovery.maxConcurrentRequests(), dropOverloads, endpointLbConfig, + discovery.upstreamTlsContext(), discovery.filterMetadata(), + discovery.backendMetricPropagation()); + LoadBalancerProvider clusterImplLbProvider = + lbRegistry.getProvider(XdsLbPolicies.CLUSTER_IMPL_POLICY_NAME); + Object priorityChildPolicy = GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig( + clusterImplLbProvider, clusterImplConfig); + + // If outlier detection has been configured we wrap the child policy in the outlier detection + // load balancer. + if (discovery.outlierDetection() != null) { + LoadBalancerProvider outlierDetectionProvider = lbRegistry.getProvider( + "outlier_detection_experimental"); + priorityChildPolicy = GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig( + outlierDetectionProvider, + buildOutlierDetectionLbConfig(discovery.outlierDetection(), priorityChildPolicy)); + } + + boolean isEds = discovery.clusterType() == ClusterType.EDS; + PriorityChildConfig priorityChildConfig = + new PriorityChildConfig(priorityChildPolicy, isEds /* ignoreReresolution */); + configs.put(priority, priorityChildConfig); + } + return configs; + } + + /** + * Converts {@link OutlierDetection} that represents the xDS configuration to {@link + * OutlierDetectionLoadBalancerConfig} that the {@link io.grpc.util.OutlierDetectionLoadBalancer} + * understands. + */ + private static OutlierDetectionLoadBalancerConfig buildOutlierDetectionLbConfig( + OutlierDetection outlierDetection, Object childConfig) { + OutlierDetectionLoadBalancerConfig.Builder configBuilder + = new OutlierDetectionLoadBalancerConfig.Builder(); + + configBuilder.setChildConfig(childConfig); + + if (outlierDetection.intervalNanos() != null) { + configBuilder.setIntervalNanos(outlierDetection.intervalNanos()); + } + if (outlierDetection.baseEjectionTimeNanos() != null) { + configBuilder.setBaseEjectionTimeNanos(outlierDetection.baseEjectionTimeNanos()); + } + if (outlierDetection.maxEjectionTimeNanos() != null) { + configBuilder.setMaxEjectionTimeNanos(outlierDetection.maxEjectionTimeNanos()); + } + if (outlierDetection.maxEjectionPercent() != null) { + configBuilder.setMaxEjectionPercent(outlierDetection.maxEjectionPercent()); + } + + SuccessRateEjection successRate = outlierDetection.successRateEjection(); + if (successRate != null) { + OutlierDetectionLoadBalancerConfig.SuccessRateEjection.Builder + successRateConfigBuilder = new OutlierDetectionLoadBalancerConfig + .SuccessRateEjection.Builder(); + + if (successRate.stdevFactor() != null) { + successRateConfigBuilder.setStdevFactor(successRate.stdevFactor()); + } + if (successRate.enforcementPercentage() != null) { + successRateConfigBuilder.setEnforcementPercentage(successRate.enforcementPercentage()); + } + if (successRate.minimumHosts() != null) { + successRateConfigBuilder.setMinimumHosts(successRate.minimumHosts()); + } + if (successRate.requestVolume() != null) { + successRateConfigBuilder.setRequestVolume(successRate.requestVolume()); + } + + configBuilder.setSuccessRateEjection(successRateConfigBuilder.build()); + } + + FailurePercentageEjection failurePercentage = outlierDetection.failurePercentageEjection(); + if (failurePercentage != null) { + OutlierDetectionLoadBalancerConfig.FailurePercentageEjection.Builder + failurePercentageConfigBuilder = new OutlierDetectionLoadBalancerConfig + .FailurePercentageEjection.Builder(); + + if (failurePercentage.threshold() != null) { + failurePercentageConfigBuilder.setThreshold(failurePercentage.threshold()); + } + if (failurePercentage.enforcementPercentage() != null) { + failurePercentageConfigBuilder.setEnforcementPercentage( + failurePercentage.enforcementPercentage()); + } + if (failurePercentage.minimumHosts() != null) { + failurePercentageConfigBuilder.setMinimumHosts(failurePercentage.minimumHosts()); + } + if (failurePercentage.requestVolume() != null) { + failurePercentageConfigBuilder.setRequestVolume(failurePercentage.requestVolume()); + } + + configBuilder.setFailurePercentageEjection(failurePercentageConfigBuilder.build()); + } + + return configBuilder.build(); + } } diff --git a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java b/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java deleted file mode 100644 index 61bfd3c0839..00000000000 --- a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java +++ /dev/null @@ -1,455 +0,0 @@ -/* - * Copyright 2020 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import static com.google.common.base.Preconditions.checkNotNull; -import static io.grpc.xds.XdsLbPolicies.PRIORITY_POLICY_NAME; - -import com.google.common.collect.ImmutableMap; -import io.grpc.Attributes; -import io.grpc.EquivalentAddressGroup; -import io.grpc.HttpConnectProxiedSocketAddress; -import io.grpc.InternalLogId; -import io.grpc.LoadBalancer; -import io.grpc.LoadBalancerProvider; -import io.grpc.LoadBalancerRegistry; -import io.grpc.Status; -import io.grpc.StatusOr; -import io.grpc.util.GracefulSwitchLoadBalancer; -import io.grpc.util.OutlierDetectionLoadBalancer.OutlierDetectionLoadBalancerConfig; -import io.grpc.xds.ClusterImplLoadBalancerProvider.ClusterImplConfig; -import io.grpc.xds.ClusterResolverLoadBalancerProvider.ClusterResolverConfig; -import io.grpc.xds.ClusterResolverLoadBalancerProvider.ClusterResolverConfig.DiscoveryMechanism; -import io.grpc.xds.Endpoints.DropOverload; -import io.grpc.xds.Endpoints.LbEndpoint; -import io.grpc.xds.Endpoints.LocalityLbEndpoints; -import io.grpc.xds.EnvoyServerProtoData.FailurePercentageEjection; -import io.grpc.xds.EnvoyServerProtoData.OutlierDetection; -import io.grpc.xds.EnvoyServerProtoData.SuccessRateEjection; -import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig; -import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig.PriorityChildConfig; -import io.grpc.xds.XdsConfig.XdsClusterConfig; -import io.grpc.xds.XdsEndpointResource.EdsUpdate; -import io.grpc.xds.client.Locality; -import io.grpc.xds.client.XdsLogger; -import io.grpc.xds.client.XdsLogger.XdsLogLevel; -import io.grpc.xds.internal.XdsInternalAttributes; -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; - -/** - * Load balancer for cluster_resolver_experimental LB policy. This LB policy is the child LB policy - * of the cds_experimental LB policy and the parent LB policy of the priority_experimental LB - * policy in the xDS load balancing hierarchy. This policy converts endpoints of non-aggregate - * clusters (e.g., EDS or Logical DNS) and groups endpoints in priorities and localities to be - * used in the downstream LB policies for fine-grained load balancing purposes. - */ -final class ClusterResolverLoadBalancer extends LoadBalancer { - private final XdsLogger logger; - private final LoadBalancerRegistry lbRegistry; - private final LoadBalancer delegate; - private ClusterState clusterState; - - ClusterResolverLoadBalancer(Helper helper, LoadBalancerRegistry lbRegistry) { - this.delegate = lbRegistry.getProvider(PRIORITY_POLICY_NAME).newLoadBalancer(helper); - this.lbRegistry = checkNotNull(lbRegistry, "lbRegistry"); - logger = XdsLogger.withLogId( - InternalLogId.allocate("cluster-resolver-lb", helper.getAuthority())); - logger.log(XdsLogLevel.INFO, "Created"); - } - - @Override - public void handleNameResolutionError(Status error) { - logger.log(XdsLogLevel.WARNING, "Received name resolution error: {0}", error); - delegate.handleNameResolutionError(error); - } - - @Override - public void shutdown() { - logger.log(XdsLogLevel.INFO, "Shutdown"); - delegate.shutdown(); - } - - @Override - public Status acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) { - logger.log(XdsLogLevel.DEBUG, "Received resolution result: {0}", resolvedAddresses); - ClusterResolverConfig config = - (ClusterResolverConfig) resolvedAddresses.getLoadBalancingPolicyConfig(); - XdsConfig xdsConfig = resolvedAddresses.getAttributes().get( - io.grpc.xds.XdsAttributes.XDS_CONFIG); - - DiscoveryMechanism instance = config.discoveryMechanism; - String cluster = instance.cluster; - if (clusterState == null) { - clusterState = new ClusterState(); - } - - StatusOr edsUpdate = getEdsUpdate(xdsConfig, cluster); - StatusOr statusOrResult = - clusterState.edsUpdateToResult(config, instance, edsUpdate); - if (!statusOrResult.hasValue()) { - Status status = Status.UNAVAILABLE - .withDescription(statusOrResult.getStatus().getDescription()) - .withCause(statusOrResult.getStatus().getCause()); - delegate.handleNameResolutionError(status); - return status; - } - ClusterResolutionResult result = statusOrResult.getValue(); - List addresses = result.addresses; - if (addresses.isEmpty()) { - Status status = Status.UNAVAILABLE - .withDescription("No usable endpoint from cluster: " + cluster); - delegate.handleNameResolutionError(status); - return status; - } - PriorityLbConfig childConfig = - new PriorityLbConfig( - Collections.unmodifiableMap(result.priorityChildConfigs), - Collections.unmodifiableList(result.priorities)); - return delegate.acceptResolvedAddresses( - resolvedAddresses.toBuilder() - .setLoadBalancingPolicyConfig(childConfig) - .setAddresses(Collections.unmodifiableList(addresses)) - .build()); - } - - private static StatusOr getEdsUpdate(XdsConfig xdsConfig, String cluster) { - StatusOr clusterConfig = xdsConfig.getClusters().get(cluster); - if (clusterConfig == null) { - return StatusOr.fromStatus(Status.INTERNAL - .withDescription("BUG: cluster resolver could not find cluster in xdsConfig")); - } - if (!clusterConfig.hasValue()) { - return StatusOr.fromStatus(clusterConfig.getStatus()); - } - if (!(clusterConfig.getValue().getChildren() instanceof XdsClusterConfig.EndpointConfig)) { - return StatusOr.fromStatus(Status.INTERNAL - .withDescription("BUG: cluster resolver cluster with children of unknown type")); - } - XdsClusterConfig.EndpointConfig endpointConfig = - (XdsClusterConfig.EndpointConfig) clusterConfig.getValue().getChildren(); - return endpointConfig.getEndpoint(); - } - - private final class ClusterState { - private Map localityPriorityNames = Collections.emptyMap(); - int priorityNameGenId = 1; - - StatusOr edsUpdateToResult( - ClusterResolverConfig config, DiscoveryMechanism discovery, StatusOr updateOr) { - if (!updateOr.hasValue()) { - return StatusOr.fromStatus(updateOr.getStatus()); - } - EdsUpdate update = updateOr.getValue(); - logger.log(XdsLogLevel.DEBUG, "Received endpoint update {0}", update); - if (logger.isLoggable(XdsLogLevel.INFO)) { - logger.log(XdsLogLevel.INFO, "Cluster {0}: {1} localities, {2} drop categories", - discovery.cluster, update.localityLbEndpointsMap.size(), - update.dropPolicies.size()); - } - Map localityLbEndpoints = - update.localityLbEndpointsMap; - List dropOverloads = update.dropPolicies; - List addresses = new ArrayList<>(); - Map> prioritizedLocalityWeights = new HashMap<>(); - List sortedPriorityNames = - generatePriorityNames(discovery.cluster, localityLbEndpoints); - for (Locality locality : localityLbEndpoints.keySet()) { - LocalityLbEndpoints localityLbInfo = localityLbEndpoints.get(locality); - String priorityName = localityPriorityNames.get(locality); - boolean discard = true; - for (LbEndpoint endpoint : localityLbInfo.endpoints()) { - if (endpoint.isHealthy()) { - discard = false; - long weight = localityLbInfo.localityWeight(); - if (endpoint.loadBalancingWeight() != 0) { - weight *= endpoint.loadBalancingWeight(); - } - String localityName = localityName(locality); - Attributes attr = - endpoint.eag().getAttributes().toBuilder() - .set(io.grpc.xds.XdsAttributes.ATTR_LOCALITY, locality) - .set(EquivalentAddressGroup.ATTR_LOCALITY_NAME, localityName) - .set(io.grpc.xds.XdsAttributes.ATTR_LOCALITY_WEIGHT, - localityLbInfo.localityWeight()) - .set(io.grpc.xds.XdsAttributes.ATTR_SERVER_WEIGHT, weight) - .set(XdsInternalAttributes.ATTR_ADDRESS_NAME, endpoint.hostname()) - .build(); - EquivalentAddressGroup eag; - if (config.isHttp11ProxyAvailable()) { - List rewrittenAddresses = new ArrayList<>(); - for (SocketAddress addr : endpoint.eag().getAddresses()) { - rewrittenAddresses.add(rewriteAddress( - addr, endpoint.endpointMetadata(), localityLbInfo.localityMetadata())); - } - eag = new EquivalentAddressGroup(rewrittenAddresses, attr); - } else { - eag = new EquivalentAddressGroup(endpoint.eag().getAddresses(), attr); - } - eag = AddressFilter.setPathFilter(eag, Arrays.asList(priorityName, localityName)); - addresses.add(eag); - } - } - if (discard) { - logger.log(XdsLogLevel.INFO, - "Discard locality {0} with 0 healthy endpoints", locality); - continue; - } - if (!prioritizedLocalityWeights.containsKey(priorityName)) { - prioritizedLocalityWeights.put(priorityName, new HashMap()); - } - prioritizedLocalityWeights.get(priorityName).put( - locality, localityLbInfo.localityWeight()); - } - if (prioritizedLocalityWeights.isEmpty()) { - // Will still update the result, as if the cluster resource is revoked. - logger.log(XdsLogLevel.INFO, - "Cluster {0} has no usable priority/locality/endpoint", discovery.cluster); - } - sortedPriorityNames.retainAll(prioritizedLocalityWeights.keySet()); - Map priorityChildConfigs = - generatePriorityChildConfigs( - discovery, config.lbConfig, lbRegistry, - prioritizedLocalityWeights, dropOverloads); - return StatusOr.fromValue(new ClusterResolutionResult(addresses, priorityChildConfigs, - sortedPriorityNames)); - } - - private SocketAddress rewriteAddress(SocketAddress addr, - ImmutableMap endpointMetadata, - ImmutableMap localityMetadata) { - if (!(addr instanceof InetSocketAddress)) { - return addr; - } - - SocketAddress proxyAddress; - try { - proxyAddress = (SocketAddress) endpointMetadata.get( - "envoy.http11_proxy_transport_socket.proxy_address"); - if (proxyAddress == null) { - proxyAddress = (SocketAddress) localityMetadata.get( - "envoy.http11_proxy_transport_socket.proxy_address"); - } - } catch (ClassCastException e) { - return addr; - } - - if (proxyAddress == null) { - return addr; - } - - return HttpConnectProxiedSocketAddress.newBuilder() - .setTargetAddress((InetSocketAddress) addr) - .setProxyAddress(proxyAddress) - .build(); - } - - private List generatePriorityNames(String name, - Map localityLbEndpoints) { - TreeMap> todo = new TreeMap<>(); - for (Locality locality : localityLbEndpoints.keySet()) { - int priority = localityLbEndpoints.get(locality).priority(); - if (!todo.containsKey(priority)) { - todo.put(priority, new ArrayList<>()); - } - todo.get(priority).add(locality); - } - Map newNames = new HashMap<>(); - Set usedNames = new HashSet<>(); - List ret = new ArrayList<>(); - for (Integer priority: todo.keySet()) { - String foundName = ""; - for (Locality locality : todo.get(priority)) { - if (localityPriorityNames.containsKey(locality) - && usedNames.add(localityPriorityNames.get(locality))) { - foundName = localityPriorityNames.get(locality); - break; - } - } - if ("".equals(foundName)) { - foundName = priorityName(name, priorityNameGenId++); - } - for (Locality locality : todo.get(priority)) { - newNames.put(locality, foundName); - } - ret.add(foundName); - } - localityPriorityNames = newNames; - return ret; - } - } - - private static class ClusterResolutionResult { - // Endpoint addresses. - private final List addresses; - // Config (include load balancing policy/config) for each priority in the cluster. - private final Map priorityChildConfigs; - // List of priority names ordered in descending priorities. - private final List priorities; - - ClusterResolutionResult(List addresses, - Map configs, List priorities) { - this.addresses = addresses; - this.priorityChildConfigs = configs; - this.priorities = priorities; - } - } - - /** - * Generates configs to be used in the priority LB policy for priorities in a cluster. - * - *

priority LB -> cluster_impl LB (one per priority) -> (weighted_target LB - * -> round_robin / least_request_experimental (one per locality)) / ring_hash_experimental - */ - private static Map generatePriorityChildConfigs( - DiscoveryMechanism discovery, - Object endpointLbConfig, - LoadBalancerRegistry lbRegistry, - Map> prioritizedLocalityWeights, - List dropOverloads) { - Map configs = new HashMap<>(); - for (String priority : prioritizedLocalityWeights.keySet()) { - ClusterImplConfig clusterImplConfig = - new ClusterImplConfig( - discovery.cluster, discovery.edsServiceName, discovery.lrsServerInfo, - discovery.maxConcurrentRequests, dropOverloads, endpointLbConfig, - discovery.tlsContext, discovery.filterMetadata, discovery.backendMetricPropagation); - LoadBalancerProvider clusterImplLbProvider = - lbRegistry.getProvider(XdsLbPolicies.CLUSTER_IMPL_POLICY_NAME); - Object priorityChildPolicy = GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig( - clusterImplLbProvider, clusterImplConfig); - - // If outlier detection has been configured we wrap the child policy in the outlier detection - // load balancer. - if (discovery.outlierDetection != null) { - LoadBalancerProvider outlierDetectionProvider = lbRegistry.getProvider( - "outlier_detection_experimental"); - priorityChildPolicy = GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig( - outlierDetectionProvider, - buildOutlierDetectionLbConfig(discovery.outlierDetection, priorityChildPolicy)); - } - - boolean isEds = discovery.type == DiscoveryMechanism.Type.EDS; - PriorityChildConfig priorityChildConfig = - new PriorityChildConfig(priorityChildPolicy, isEds /* ignoreReresolution */); - configs.put(priority, priorityChildConfig); - } - return configs; - } - - /** - * Converts {@link OutlierDetection} that represents the xDS configuration to {@link - * OutlierDetectionLoadBalancerConfig} that the {@link io.grpc.util.OutlierDetectionLoadBalancer} - * understands. - */ - private static OutlierDetectionLoadBalancerConfig buildOutlierDetectionLbConfig( - OutlierDetection outlierDetection, Object childConfig) { - OutlierDetectionLoadBalancerConfig.Builder configBuilder - = new OutlierDetectionLoadBalancerConfig.Builder(); - - configBuilder.setChildConfig(childConfig); - - if (outlierDetection.intervalNanos() != null) { - configBuilder.setIntervalNanos(outlierDetection.intervalNanos()); - } - if (outlierDetection.baseEjectionTimeNanos() != null) { - configBuilder.setBaseEjectionTimeNanos(outlierDetection.baseEjectionTimeNanos()); - } - if (outlierDetection.maxEjectionTimeNanos() != null) { - configBuilder.setMaxEjectionTimeNanos(outlierDetection.maxEjectionTimeNanos()); - } - if (outlierDetection.maxEjectionPercent() != null) { - configBuilder.setMaxEjectionPercent(outlierDetection.maxEjectionPercent()); - } - - SuccessRateEjection successRate = outlierDetection.successRateEjection(); - if (successRate != null) { - OutlierDetectionLoadBalancerConfig.SuccessRateEjection.Builder - successRateConfigBuilder = new OutlierDetectionLoadBalancerConfig - .SuccessRateEjection.Builder(); - - if (successRate.stdevFactor() != null) { - successRateConfigBuilder.setStdevFactor(successRate.stdevFactor()); - } - if (successRate.enforcementPercentage() != null) { - successRateConfigBuilder.setEnforcementPercentage(successRate.enforcementPercentage()); - } - if (successRate.minimumHosts() != null) { - successRateConfigBuilder.setMinimumHosts(successRate.minimumHosts()); - } - if (successRate.requestVolume() != null) { - successRateConfigBuilder.setRequestVolume(successRate.requestVolume()); - } - - configBuilder.setSuccessRateEjection(successRateConfigBuilder.build()); - } - - FailurePercentageEjection failurePercentage = outlierDetection.failurePercentageEjection(); - if (failurePercentage != null) { - OutlierDetectionLoadBalancerConfig.FailurePercentageEjection.Builder - failurePercentageConfigBuilder = new OutlierDetectionLoadBalancerConfig - .FailurePercentageEjection.Builder(); - - if (failurePercentage.threshold() != null) { - failurePercentageConfigBuilder.setThreshold(failurePercentage.threshold()); - } - if (failurePercentage.enforcementPercentage() != null) { - failurePercentageConfigBuilder.setEnforcementPercentage( - failurePercentage.enforcementPercentage()); - } - if (failurePercentage.minimumHosts() != null) { - failurePercentageConfigBuilder.setMinimumHosts(failurePercentage.minimumHosts()); - } - if (failurePercentage.requestVolume() != null) { - failurePercentageConfigBuilder.setRequestVolume(failurePercentage.requestVolume()); - } - - configBuilder.setFailurePercentageEjection(failurePercentageConfigBuilder.build()); - } - - return configBuilder.build(); - } - - /** - * Generates a string that represents the priority in the LB policy config. The string is unique - * across priorities in all clusters and priorityName(c, p1) < priorityName(c, p2) iff p1 < p2. - * The ordering is undefined for priorities in different clusters. - */ - private static String priorityName(String cluster, int priority) { - return cluster + "[child" + priority + "]"; - } - - /** - * Generates a string that represents the locality in the LB policy config. The string is unique - * across all localities in all clusters. - */ - private static String localityName(Locality locality) { - return "{region=\"" + locality.region() - + "\", zone=\"" + locality.zone() - + "\", sub_zone=\"" + locality.subZone() - + "\"}"; - } -} diff --git a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancerProvider.java b/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancerProvider.java deleted file mode 100644 index f6f92a7a9a7..00000000000 --- a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancerProvider.java +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Copyright 2020 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.common.base.MoreObjects; -import com.google.common.collect.ImmutableMap; -import com.google.protobuf.Struct; -import io.grpc.Internal; -import io.grpc.LoadBalancer; -import io.grpc.LoadBalancer.Helper; -import io.grpc.LoadBalancerProvider; -import io.grpc.LoadBalancerRegistry; -import io.grpc.NameResolver.ConfigOrError; -import io.grpc.Status; -import io.grpc.xds.EnvoyServerProtoData.OutlierDetection; -import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; -import io.grpc.xds.client.BackendMetricPropagation; -import io.grpc.xds.client.Bootstrapper.ServerInfo; -import java.util.Map; -import java.util.Objects; -import javax.annotation.Nullable; - -/** - * The provider for the cluster_resolver load balancing policy. This class should not be directly - * referenced in code. The policy should be accessed through - * {@link io.grpc.LoadBalancerRegistry#getProvider} with the name "cluster_resolver_experimental". - */ -@Internal -public final class ClusterResolverLoadBalancerProvider extends LoadBalancerProvider { - private final LoadBalancerRegistry lbRegistry; - - public ClusterResolverLoadBalancerProvider() { - this.lbRegistry = null; - } - - ClusterResolverLoadBalancerProvider(LoadBalancerRegistry lbRegistry) { - this.lbRegistry = checkNotNull(lbRegistry, "lbRegistry"); - } - - @Override - public boolean isAvailable() { - return true; - } - - @Override - public int getPriority() { - return 5; - } - - @Override - public String getPolicyName() { - return XdsLbPolicies.CLUSTER_RESOLVER_POLICY_NAME; - } - - @Override - public ConfigOrError parseLoadBalancingPolicyConfig(Map rawLoadBalancingPolicyConfig) { - return ConfigOrError.fromError( - Status.INTERNAL.withDescription(getPolicyName() + " cannot be used from service config")); - } - - @Override - public LoadBalancer newLoadBalancer(Helper helper) { - LoadBalancerRegistry lbRegistry = this.lbRegistry; - if (lbRegistry == null) { - lbRegistry = LoadBalancerRegistry.getDefaultRegistry(); - } - return new ClusterResolverLoadBalancer(helper, lbRegistry); - } - - static final class ClusterResolverConfig { - // Cluster to be resolved. - final DiscoveryMechanism discoveryMechanism; - // GracefulSwitch configuration - final Object lbConfig; - private final boolean isHttp11ProxyAvailable; - - ClusterResolverConfig(DiscoveryMechanism discoveryMechanism, Object lbConfig, - boolean isHttp11ProxyAvailable) { - this.discoveryMechanism = checkNotNull(discoveryMechanism, "discoveryMechanism"); - this.lbConfig = checkNotNull(lbConfig, "lbConfig"); - this.isHttp11ProxyAvailable = isHttp11ProxyAvailable; - } - - boolean isHttp11ProxyAvailable() { - return isHttp11ProxyAvailable; - } - - @Override - public int hashCode() { - return Objects.hash(discoveryMechanism, lbConfig, isHttp11ProxyAvailable); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ClusterResolverConfig that = (ClusterResolverConfig) o; - return discoveryMechanism.equals(that.discoveryMechanism) - && lbConfig.equals(that.lbConfig) - && isHttp11ProxyAvailable == that.isHttp11ProxyAvailable; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("discoveryMechanism", discoveryMechanism) - .add("lbConfig", lbConfig) - .add("isHttp11ProxyAvailable", isHttp11ProxyAvailable) - .toString(); - } - - // Describes the mechanism for a specific cluster. - static final class DiscoveryMechanism { - // Name of the cluster to resolve. - final String cluster; - // Type of the cluster. - final Type type; - // Load reporting server info. Null if not enabled. - @Nullable - final ServerInfo lrsServerInfo; - // Cluster-level max concurrent request threshold. Null if not specified. - @Nullable - final Long maxConcurrentRequests; - // TLS context for connections to endpoints in the cluster. - @Nullable - final UpstreamTlsContext tlsContext; - // Resource name for resolving endpoints via EDS. Only valid for EDS clusters. - @Nullable - final String edsServiceName; - // Hostname for resolving endpoints via DNS. Only valid for LOGICAL_DNS clusters. - @Nullable - final String dnsHostName; - @Nullable - final OutlierDetection outlierDetection; - final Map filterMetadata; - @Nullable - final BackendMetricPropagation backendMetricPropagation; - - enum Type { - EDS, - LOGICAL_DNS, - } - - private DiscoveryMechanism(String cluster, Type type, @Nullable String edsServiceName, - @Nullable String dnsHostName, @Nullable ServerInfo lrsServerInfo, - @Nullable Long maxConcurrentRequests, @Nullable UpstreamTlsContext tlsContext, - Map filterMetadata, @Nullable OutlierDetection outlierDetection, - @Nullable BackendMetricPropagation backendMetricPropagation) { - this.cluster = checkNotNull(cluster, "cluster"); - this.type = checkNotNull(type, "type"); - this.edsServiceName = edsServiceName; - this.dnsHostName = dnsHostName; - this.lrsServerInfo = lrsServerInfo; - this.maxConcurrentRequests = maxConcurrentRequests; - this.tlsContext = tlsContext; - this.filterMetadata = ImmutableMap.copyOf(checkNotNull(filterMetadata, "filterMetadata")); - this.outlierDetection = outlierDetection; - this.backendMetricPropagation = backendMetricPropagation; - } - - static DiscoveryMechanism forEds(String cluster, @Nullable String edsServiceName, - @Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests, - @Nullable UpstreamTlsContext tlsContext, Map filterMetadata, - OutlierDetection outlierDetection, - @Nullable BackendMetricPropagation backendMetricPropagation) { - return new DiscoveryMechanism(cluster, Type.EDS, edsServiceName, - null, lrsServerInfo, maxConcurrentRequests, tlsContext, - filterMetadata, outlierDetection, backendMetricPropagation); - } - - static DiscoveryMechanism forLogicalDns(String cluster, String dnsHostName, - @Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests, - @Nullable UpstreamTlsContext tlsContext, Map filterMetadata, - @Nullable BackendMetricPropagation backendMetricPropagation) { - return new DiscoveryMechanism(cluster, Type.LOGICAL_DNS, null, dnsHostName, - lrsServerInfo, maxConcurrentRequests, tlsContext, filterMetadata, null, - backendMetricPropagation); - } - - @Override - public int hashCode() { - return Objects.hash(cluster, type, lrsServerInfo, maxConcurrentRequests, tlsContext, - edsServiceName, dnsHostName, filterMetadata, - outlierDetection, backendMetricPropagation); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - DiscoveryMechanism that = (DiscoveryMechanism) o; - return cluster.equals(that.cluster) - && type == that.type - && Objects.equals(edsServiceName, that.edsServiceName) - && Objects.equals(dnsHostName, that.dnsHostName) - && Objects.equals(lrsServerInfo, that.lrsServerInfo) - && Objects.equals(maxConcurrentRequests, that.maxConcurrentRequests) - && Objects.equals(tlsContext, that.tlsContext) - && Objects.equals(filterMetadata, that.filterMetadata) - && Objects.equals(outlierDetection, that.outlierDetection); - } - - @Override - public String toString() { - MoreObjects.ToStringHelper toStringHelper = - MoreObjects.toStringHelper(this) - .add("cluster", cluster) - .add("type", type) - .add("edsServiceName", edsServiceName) - .add("dnsHostName", dnsHostName) - .add("lrsServerInfo", lrsServerInfo) - // Exclude tlsContext as its string representation is cumbersome. - .add("maxConcurrentRequests", maxConcurrentRequests) - .add("filterMetadata", filterMetadata) - // Exclude outlierDetection as its string representation is long. - ; - return toStringHelper.toString(); - } - } - } -} diff --git a/xds/src/main/java/io/grpc/xds/XdsLbPolicies.java b/xds/src/main/java/io/grpc/xds/XdsLbPolicies.java index dcca2fbfff3..ae5ac38b471 100644 --- a/xds/src/main/java/io/grpc/xds/XdsLbPolicies.java +++ b/xds/src/main/java/io/grpc/xds/XdsLbPolicies.java @@ -19,7 +19,6 @@ final class XdsLbPolicies { static final String CLUSTER_MANAGER_POLICY_NAME = "cluster_manager_experimental"; static final String CDS_POLICY_NAME = "cds_experimental"; - static final String CLUSTER_RESOLVER_POLICY_NAME = "cluster_resolver_experimental"; static final String PRIORITY_POLICY_NAME = "priority_experimental"; static final String CLUSTER_IMPL_POLICY_NAME = "cluster_impl_experimental"; static final String WEIGHTED_TARGET_POLICY_NAME = "weighted_target_experimental"; diff --git a/xds/src/main/resources/META-INF/services/io.grpc.LoadBalancerProvider b/xds/src/main/resources/META-INF/services/io.grpc.LoadBalancerProvider index e1c4d4aa427..04a2d9cf7a8 100644 --- a/xds/src/main/resources/META-INF/services/io.grpc.LoadBalancerProvider +++ b/xds/src/main/resources/META-INF/services/io.grpc.LoadBalancerProvider @@ -2,7 +2,6 @@ io.grpc.xds.CdsLoadBalancerProvider io.grpc.xds.PriorityLoadBalancerProvider io.grpc.xds.WeightedTargetLoadBalancerProvider io.grpc.xds.ClusterManagerLoadBalancerProvider -io.grpc.xds.ClusterResolverLoadBalancerProvider io.grpc.xds.ClusterImplLoadBalancerProvider io.grpc.xds.LeastRequestLoadBalancerProvider io.grpc.xds.RingHashLoadBalancerProvider diff --git a/xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java b/xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java index c1d3322faa2..928520aded7 100644 --- a/xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java +++ b/xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java @@ -17,7 +17,7 @@ package io.grpc.xds; import static com.google.common.truth.Truth.assertThat; -import static io.grpc.xds.XdsLbPolicies.CLUSTER_RESOLVER_POLICY_NAME; +import static io.grpc.xds.XdsLbPolicies.CLUSTER_IMPL_POLICY_NAME; import static io.grpc.xds.XdsLbPolicies.PRIORITY_POLICY_NAME; import static io.grpc.xds.XdsTestControlPlaneService.ADS_TYPE_URL_CDS; import static io.grpc.xds.XdsTestControlPlaneService.ADS_TYPE_URL_EDS; @@ -78,11 +78,7 @@ import io.grpc.testing.GrpcCleanupRule; import io.grpc.util.GracefulSwitchLoadBalancerAccessor; import io.grpc.xds.CdsLoadBalancerProvider.CdsConfig; -import io.grpc.xds.ClusterResolverLoadBalancerProvider.ClusterResolverConfig; -import io.grpc.xds.ClusterResolverLoadBalancerProvider.ClusterResolverConfig.DiscoveryMechanism; -import io.grpc.xds.EnvoyServerProtoData.FailurePercentageEjection; -import io.grpc.xds.EnvoyServerProtoData.SuccessRateEjection; -import io.grpc.xds.client.Bootstrapper.ServerInfo; +import io.grpc.xds.ClusterImplLoadBalancerProvider.ClusterImplConfig; import io.grpc.xds.client.XdsClient; import io.grpc.xds.internal.security.CommonTlsContextTestsUtil; import java.util.ArrayList; @@ -139,7 +135,6 @@ public class CdsLoadBalancer2Test { .directExecutor() .build()), fakeClock); - private final ServerInfo lrsServerInfo = xdsClient.getBootstrapInfo().servers().get(0); private XdsDependencyManager xdsDepManager; @Mock @@ -151,8 +146,10 @@ public class CdsLoadBalancer2Test { @Before public void setUp() throws Exception { - lbRegistry.register(new FakeLoadBalancerProvider(CLUSTER_RESOLVER_POLICY_NAME)); + lbRegistry.register(new FakeLoadBalancerProvider(PRIORITY_POLICY_NAME)); + lbRegistry.register(new FakeLoadBalancerProvider(CLUSTER_IMPL_POLICY_NAME)); lbRegistry.register(new FakeLoadBalancerProvider("round_robin")); + lbRegistry.register(new FakeLoadBalancerProvider("outlier_detection_experimental")); lbRegistry.register( new FakeLoadBalancerProvider("ring_hash_experimental", new RingHashLoadBalancerProvider())); lbRegistry.register(new FakeLoadBalancerProvider("least_request_experimental", @@ -222,7 +219,7 @@ private void shutdownLoadBalancer() { } @Test - public void discoverTopLevelEdsCluster() { + public void discoverTopLevelCluster() { Cluster cluster = Cluster.newBuilder() .setName(CLUSTER) .setType(Cluster.DiscoveryType.EDS) @@ -250,64 +247,7 @@ public void discoverTopLevelEdsCluster() { verify(helper, never()).updateBalancingState(eq(ConnectivityState.TRANSIENT_FAILURE), any()); assertThat(childBalancers).hasSize(1); FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); - assertThat(childBalancer.name).isEqualTo(CLUSTER_RESOLVER_POLICY_NAME); - ClusterResolverConfig childLbConfig = (ClusterResolverConfig) childBalancer.config; - assertThat(childLbConfig.discoveryMechanism).isEqualTo( - DiscoveryMechanism.forEds( - CLUSTER, EDS_SERVICE_NAME, lrsServerInfo, 100L, upstreamTlsContext, - Collections.emptyMap(), io.grpc.xds.EnvoyServerProtoData.OutlierDetection.create( - null, null, null, null, SuccessRateEjection.create(null, null, null, null), - FailurePercentageEjection.create(null, null, null, null)), null)); - assertThat( - GracefulSwitchLoadBalancerAccessor.getChildProvider(childLbConfig.lbConfig).getPolicyName()) - .isEqualTo("wrr_locality_experimental"); - } - - @Test - public void discoverTopLevelLogicalDnsCluster() { - Cluster cluster = Cluster.newBuilder() - .setName(CLUSTER) - .setType(Cluster.DiscoveryType.LOGICAL_DNS) - .setLoadAssignment(ClusterLoadAssignment.newBuilder() - .addEndpoints(LocalityLbEndpoints.newBuilder() - .addLbEndpoints(LbEndpoint.newBuilder() - .setEndpoint(Endpoint.newBuilder() - .setAddress(Address.newBuilder() - .setSocketAddress(SocketAddress.newBuilder() - .setAddress("dns.example.com") - .setPortValue(1111))))))) - .setEdsClusterConfig(Cluster.EdsClusterConfig.newBuilder() - .setServiceName(EDS_SERVICE_NAME) - .setEdsConfig(ConfigSource.newBuilder() - .setAds(AggregatedConfigSource.newBuilder()))) - .setLbPolicy(Cluster.LbPolicy.LEAST_REQUEST) - .setLrsServer(ConfigSource.newBuilder() - .setSelf(SelfConfigSource.getDefaultInstance())) - .setCircuitBreakers(CircuitBreakers.newBuilder() - .addThresholds(CircuitBreakers.Thresholds.newBuilder() - .setPriority(RoutingPriority.DEFAULT) - .setMaxRequests(UInt32Value.newBuilder().setValue(100)))) - .setTransportSocket(TransportSocket.newBuilder() - .setName("envoy.transport_sockets.tls") - .setTypedConfig(Any.pack(UpstreamTlsContext.newBuilder() - .setCommonTlsContext(upstreamTlsContext.getCommonTlsContext()) - .build()))) - .build(); - controlPlaneService.setXdsConfig(ADS_TYPE_URL_CDS, ImmutableMap.of(CLUSTER, cluster)); - startXdsDepManager(); - - verify(helper, never()).updateBalancingState(eq(ConnectivityState.TRANSIENT_FAILURE), any()); - assertThat(childBalancers).hasSize(1); - FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); - assertThat(childBalancer.name).isEqualTo(CLUSTER_RESOLVER_POLICY_NAME); - ClusterResolverConfig childLbConfig = (ClusterResolverConfig) childBalancer.config; - assertThat(childLbConfig.discoveryMechanism).isEqualTo( - DiscoveryMechanism.forLogicalDns( - CLUSTER, "dns.example.com:1111", lrsServerInfo, 100L, upstreamTlsContext, - Collections.emptyMap(), null)); - assertThat( - GracefulSwitchLoadBalancerAccessor.getChildProvider(childLbConfig.lbConfig).getPolicyName()) - .isEqualTo("wrr_locality_experimental"); + assertThat(childBalancer.name).isEqualTo(PRIORITY_POLICY_NAME); } @Test @@ -325,6 +265,7 @@ public void nonAggregateCluster_resourceNotExist_returnErrorPicker() { @Test public void nonAggregateCluster_resourceUpdate() { + lbRegistry.register(new PriorityLoadBalancerProvider()); Cluster cluster = EDS_CLUSTER.toBuilder() .setCircuitBreakers(CircuitBreakers.newBuilder() .addThresholds(CircuitBreakers.Thresholds.newBuilder() @@ -337,10 +278,9 @@ public void nonAggregateCluster_resourceUpdate() { verify(helper, never()).updateBalancingState(eq(ConnectivityState.TRANSIENT_FAILURE), any()); assertThat(childBalancers).hasSize(1); FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); - ClusterResolverConfig childLbConfig = (ClusterResolverConfig) childBalancer.config; - assertThat(childLbConfig.discoveryMechanism).isEqualTo( - DiscoveryMechanism.forEds( - CLUSTER, EDS_SERVICE_NAME, null, 100L, null, Collections.emptyMap(), null, null)); + ClusterImplConfig childLbConfig = (ClusterImplConfig) childBalancer.config; + assertThat(childLbConfig.cluster).isEqualTo(CLUSTER); + assertThat(childLbConfig.maxConcurrentRequests).isEqualTo(100L); cluster = EDS_CLUSTER.toBuilder() .setCircuitBreakers(CircuitBreakers.newBuilder() @@ -352,24 +292,21 @@ public void nonAggregateCluster_resourceUpdate() { verify(helper, never()).updateBalancingState(eq(ConnectivityState.TRANSIENT_FAILURE), any()); assertThat(childBalancers).hasSize(1); childBalancer = Iterables.getOnlyElement(childBalancers); - childLbConfig = (ClusterResolverConfig) childBalancer.config; - assertThat(childLbConfig.discoveryMechanism).isEqualTo( - DiscoveryMechanism.forEds( - CLUSTER, EDS_SERVICE_NAME, null, 200L, null, Collections.emptyMap(), null, null)); + childLbConfig = (ClusterImplConfig) childBalancer.config; + assertThat(childLbConfig.maxConcurrentRequests).isEqualTo(200L); } @Test public void nonAggregateCluster_resourceRevoked() { + lbRegistry.register(new PriorityLoadBalancerProvider()); controlPlaneService.setXdsConfig(ADS_TYPE_URL_CDS, ImmutableMap.of(CLUSTER, EDS_CLUSTER)); startXdsDepManager(); verify(helper, never()).updateBalancingState(eq(ConnectivityState.TRANSIENT_FAILURE), any()); assertThat(childBalancers).hasSize(1); FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); - ClusterResolverConfig childLbConfig = (ClusterResolverConfig) childBalancer.config; - assertThat(childLbConfig.discoveryMechanism).isEqualTo( - DiscoveryMechanism.forEds( - CLUSTER, EDS_SERVICE_NAME, null, null, null, Collections.emptyMap(), null, null)); + ClusterImplConfig childLbConfig = (ClusterImplConfig) childBalancer.config; + assertThat(childLbConfig.cluster).isEqualTo(CLUSTER); controlPlaneService.setXdsConfig(ADS_TYPE_URL_CDS, ImmutableMap.of()); @@ -398,10 +335,7 @@ public void dynamicCluster() { verify(helper, never()).updateBalancingState(eq(ConnectivityState.TRANSIENT_FAILURE), any()); assertThat(childBalancers).hasSize(1); FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); - ClusterResolverConfig childLbConfig = (ClusterResolverConfig) childBalancer.config; - assertThat(childLbConfig.discoveryMechanism).isEqualTo( - DiscoveryMechanism.forEds( - clusterName, EDS_SERVICE_NAME, null, null, null, Collections.emptyMap(), null, null)); + assertThat(childBalancer.name).isEqualTo(PRIORITY_POLICY_NAME); assertThat(this.lastXdsConfig.getClusters()).containsKey(clusterName); shutdownLoadBalancer(); @@ -410,7 +344,6 @@ public void dynamicCluster() { @Test public void discoverAggregateCluster_createsPriorityLbPolicy() { - lbRegistry.register(new FakeLoadBalancerProvider(PRIORITY_POLICY_NAME)); CdsLoadBalancerProvider cdsLoadBalancerProvider = new CdsLoadBalancerProvider(lbRegistry); lbRegistry.register(cdsLoadBalancerProvider); loadBalancer = (CdsLoadBalancer2) cdsLoadBalancerProvider.newLoadBalancer(helper); @@ -522,21 +455,17 @@ public void discoverAggregateCluster_testChildCdsLbPolicyParsing() { verify(helper, never()).updateBalancingState(eq(ConnectivityState.TRANSIENT_FAILURE), any()); assertThat(childBalancers).hasSize(2); - ClusterResolverConfig cluster1ResolverConfig = - (ClusterResolverConfig) childBalancers.get(0).config; - assertThat(cluster1ResolverConfig.discoveryMechanism.cluster) + ClusterImplConfig cluster1ImplConfig = + (ClusterImplConfig) childBalancers.get(0).config; + assertThat(cluster1ImplConfig.cluster) .isEqualTo("cluster-01.googleapis.com"); - assertThat(cluster1ResolverConfig.discoveryMechanism.type) - .isEqualTo(DiscoveryMechanism.Type.EDS); - assertThat(cluster1ResolverConfig.discoveryMechanism.edsServiceName) + assertThat(cluster1ImplConfig.edsServiceName) .isEqualTo("backend-service-1.googleapis.com"); - ClusterResolverConfig cluster2ResolverConfig = - (ClusterResolverConfig) childBalancers.get(1).config; - assertThat(cluster2ResolverConfig.discoveryMechanism.cluster) + ClusterImplConfig cluster2ImplConfig = + (ClusterImplConfig) childBalancers.get(1).config; + assertThat(cluster2ImplConfig.cluster) .isEqualTo("cluster-02.googleapis.com"); - assertThat(cluster2ResolverConfig.discoveryMechanism.type) - .isEqualTo(DiscoveryMechanism.Type.EDS); - assertThat(cluster2ResolverConfig.discoveryMechanism.edsServiceName) + assertThat(cluster2ImplConfig.edsServiceName) .isEqualTo("backend-service-1.googleapis.com"); } diff --git a/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerProviderTest.java b/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerProviderTest.java deleted file mode 100644 index a201ecfaa4b..00000000000 --- a/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerProviderTest.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2020 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import static com.google.common.truth.Truth.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import io.grpc.ChannelLogger; -import io.grpc.LoadBalancer; -import io.grpc.LoadBalancer.Helper; -import io.grpc.LoadBalancerProvider; -import io.grpc.LoadBalancerRegistry; -import io.grpc.NameResolver; -import io.grpc.NameResolver.ServiceConfigParser; -import io.grpc.NameResolverRegistry; -import io.grpc.SynchronizationContext; -import io.grpc.internal.FakeClock; -import io.grpc.internal.GrpcUtil; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Tests for {@link ClusterResolverLoadBalancerProvider}. */ -@RunWith(JUnit4.class) -public class ClusterResolverLoadBalancerProviderTest { - - @Test - public void provided() { - LoadBalancerProvider provider = - LoadBalancerRegistry.getDefaultRegistry().getProvider( - XdsLbPolicies.CLUSTER_RESOLVER_POLICY_NAME); - assertThat(provider).isInstanceOf(ClusterResolverLoadBalancerProvider.class); - } - - @Test - public void providesLoadBalancer() { - Helper helper = mock(Helper.class); - - SynchronizationContext syncContext = new SynchronizationContext( - new Thread.UncaughtExceptionHandler() { - @Override - public void uncaughtException(Thread t, Throwable e) { - throw new AssertionError(e); - } - }); - FakeClock fakeClock = new FakeClock(); - NameResolverRegistry nsRegistry = new NameResolverRegistry(); - NameResolver.Args args = NameResolver.Args.newBuilder() - .setDefaultPort(8080) - .setProxyDetector(GrpcUtil.NOOP_PROXY_DETECTOR) - .setSynchronizationContext(syncContext) - .setServiceConfigParser(mock(ServiceConfigParser.class)) - .setChannelLogger(mock(ChannelLogger.class)) - .build(); - when(helper.getNameResolverRegistry()).thenReturn(nsRegistry); - when(helper.getNameResolverArgs()).thenReturn(args); - when(helper.getSynchronizationContext()).thenReturn(syncContext); - when(helper.getScheduledExecutorService()).thenReturn(fakeClock.getScheduledExecutorService()); - when(helper.getAuthority()).thenReturn("api.google.com"); - LoadBalancerProvider provider = new ClusterResolverLoadBalancerProvider(); - LoadBalancer loadBalancer = provider.newLoadBalancer(helper); - assertThat(loadBalancer).isInstanceOf(ClusterResolverLoadBalancer.class); - } -} diff --git a/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java index 5c710699a96..5e524f79596 100644 --- a/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java @@ -36,7 +36,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; -import com.google.common.testing.EqualsTester; import com.google.protobuf.Any; import com.google.protobuf.Duration; import com.google.protobuf.UInt32Value; @@ -61,7 +60,6 @@ import io.grpc.ConnectivityState; import io.grpc.EquivalentAddressGroup; import io.grpc.HttpConnectProxiedSocketAddress; -import io.grpc.InsecureChannelCredentials; import io.grpc.LoadBalancer; import io.grpc.LoadBalancer.Helper; import io.grpc.LoadBalancer.PickResult; @@ -83,19 +81,13 @@ import io.grpc.internal.FakeClock; import io.grpc.internal.GrpcUtil; import io.grpc.testing.GrpcCleanupRule; -import io.grpc.util.GracefulSwitchLoadBalancer; import io.grpc.util.GracefulSwitchLoadBalancerAccessor; import io.grpc.util.OutlierDetectionLoadBalancer.OutlierDetectionLoadBalancerConfig; import io.grpc.util.OutlierDetectionLoadBalancerProvider; import io.grpc.xds.CdsLoadBalancerProvider.CdsConfig; import io.grpc.xds.ClusterImplLoadBalancerProvider.ClusterImplConfig; -import io.grpc.xds.ClusterResolverLoadBalancerProvider.ClusterResolverConfig; -import io.grpc.xds.ClusterResolverLoadBalancerProvider.ClusterResolverConfig.DiscoveryMechanism; import io.grpc.xds.Endpoints.DropOverload; -import io.grpc.xds.EnvoyServerProtoData.FailurePercentageEjection; -import io.grpc.xds.EnvoyServerProtoData.SuccessRateEjection; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; -import io.grpc.xds.LeastRequestLoadBalancer.LeastRequestConfig; import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig; import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig.PriorityChildConfig; import io.grpc.xds.RingHashLoadBalancer.RingHashConfig; @@ -105,7 +97,6 @@ import io.grpc.xds.client.LoadStatsManager2; import io.grpc.xds.client.XdsClient; import io.grpc.xds.internal.XdsInternalAttributes; -import io.grpc.xds.internal.security.CommonTlsContextTestsUtil; import java.net.InetSocketAddress; import java.net.URI; import java.util.ArrayList; @@ -202,7 +193,6 @@ public void uncaughtException(Thread t, Throwable e) { @Before public void setUp() throws Exception { - lbRegistry.register(new ClusterResolverLoadBalancerProvider(lbRegistry)); lbRegistry.register(new RingHashLoadBalancerProvider()); lbRegistry.register(new WrrLocalityLoadBalancerProvider()); lbRegistry.register(new FakeLoadBalancerProvider(PRIORITY_POLICY_NAME)); @@ -750,37 +740,34 @@ public void cdsRevoked_handledDirectly() { } @Test - public void edsMissing_handledByChildPolicy() { + public void edsMissing_failsRpcs() { controlPlaneService.setXdsConfig(ADS_TYPE_URL_EDS, ImmutableMap.of()); startXdsDepManager(); - assertThat(childBalancers).hasSize(1); // child LB policy created - FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); - assertThat(childBalancer.upstreamError).isNotNull(); - assertThat(childBalancer.upstreamError.getCode()).isEqualTo(Status.Code.UNAVAILABLE); + assertThat(childBalancers).hasSize(0); // Graceful switch handles it, so no child policies yet + verify(helper).updateBalancingState( + eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture()); String expectedDescription = "Error retrieving EDS resource " + EDS_SERVICE_NAME + ": NOT_FOUND. Details: Timed out waiting for resource " + EDS_SERVICE_NAME + " from xDS server nodeID: node-id"; - assertThat(childBalancer.upstreamError.getDescription()).isEqualTo(expectedDescription); - assertThat(childBalancer.shutdown).isFalse(); + Status expectedError = Status.UNAVAILABLE.withDescription(expectedDescription); + assertPicker(pickerCaptor.getValue(), expectedError, null); } @Test - public void logicalDnsLookupFailed_handledByChildPolicy() { + public void logicalDnsLookupFailed_failsRpcs() { controlPlaneService.setXdsConfig(ADS_TYPE_URL_CDS, ImmutableMap.of( CLUSTER, LOGICAL_DNS_CLUSTER)); startXdsDepManager(new CdsConfig(CLUSTER), /* forwardTime= */ false); FakeNameResolver resolver = assertResolverCreated("/" + DNS_HOST_NAME + ":9000"); assertThat(childBalancers).isEmpty(); - resolver.deliverError(Status.UNAVAILABLE.withDescription("OH NO! Who would have guessed?")); + Status status = Status.UNAVAILABLE.withDescription("OH NO! Who would have guessed?"); + resolver.deliverError(status); - assertThat(childBalancers).hasSize(1); // child LB policy created - FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); - assertThat(childBalancer.upstreamError).isNotNull(); - assertThat(childBalancer.upstreamError.getCode()).isEqualTo(Status.Code.UNAVAILABLE); - assertThat(childBalancer.upstreamError.getDescription()) - .isEqualTo("OH NO! Who would have guessed?"); - assertThat(childBalancer.shutdown).isFalse(); + assertThat(childBalancers).hasSize(0); // Graceful switch handles it, so no child policies yet + verify(helper).updateBalancingState( + eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture()); + assertPicker(pickerCaptor.getValue(), status, null); } @Test @@ -873,13 +860,12 @@ public void handleEdsResource_noHealthyEndpoint() { EDS_SERVICE_NAME, clusterLoadAssignment)); startXdsDepManager(); - verify(helper, never()).updateBalancingState(eq(ConnectivityState.TRANSIENT_FAILURE), any()); - assertThat(childBalancers).hasSize(1); - FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); - assertThat(childBalancer.upstreamError).isNotNull(); - assertThat(childBalancer.upstreamError.getCode()).isEqualTo(Status.Code.UNAVAILABLE); - assertThat(childBalancer.upstreamError.getDescription()) - .isEqualTo("No usable endpoint from cluster: " + CLUSTER); + assertThat(childBalancers).hasSize(0); // Graceful switch handles it, so no child policies yet + verify(helper).updateBalancingState( + eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture()); + Status expectedStatus = Status.UNAVAILABLE + .withDescription("No usable endpoint from cluster: " + CLUSTER); + assertPicker(pickerCaptor.getValue(), expectedStatus, null); } @Test @@ -1016,50 +1002,6 @@ public void outlierDetection_fullConfig() { "wrr_locality_experimental"); } - @Test - public void config_equalsTester() { - ServerInfo lrsServerInfo = - ServerInfo.create("lrs.googleapis.com", InsecureChannelCredentials.create()); - UpstreamTlsContext tlsContext = - CommonTlsContextTestsUtil.buildUpstreamTlsContext( - "google_cloud_private_spiffe", true); - DiscoveryMechanism edsDiscoveryMechanism1 = - DiscoveryMechanism.forEds(CLUSTER, EDS_SERVICE_NAME, lrsServerInfo, 100L, tlsContext, - Collections.emptyMap(), null, null); - io.grpc.xds.EnvoyServerProtoData.OutlierDetection outlierDetection = - io.grpc.xds.EnvoyServerProtoData.OutlierDetection.create( - 100L, 100L, 100L, 100, SuccessRateEjection.create(100, 100, 100, 100), - FailurePercentageEjection.create(100, 100, 100, 100)); - DiscoveryMechanism edsDiscoveryMechanismWithOutlierDetection = - DiscoveryMechanism.forEds(CLUSTER, EDS_SERVICE_NAME, lrsServerInfo, 100L, tlsContext, - Collections.emptyMap(), outlierDetection, null); - Object roundRobin = GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig( - new FakeLoadBalancerProvider("wrr_locality_experimental"), new WrrLocalityConfig( - GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig( - new FakeLoadBalancerProvider("round_robin"), null))); - Object leastRequest = GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig( - new FakeLoadBalancerProvider("wrr_locality_experimental"), new WrrLocalityConfig( - GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig( - new FakeLoadBalancerProvider("least_request_experimental"), - new LeastRequestConfig(3)))); - - new EqualsTester() - .addEqualityGroup( - new ClusterResolverConfig( - edsDiscoveryMechanism1, leastRequest, false), - new ClusterResolverConfig( - edsDiscoveryMechanism1, leastRequest, false)) - .addEqualityGroup(new ClusterResolverConfig( - edsDiscoveryMechanism1, roundRobin, false)) - .addEqualityGroup(new ClusterResolverConfig( - edsDiscoveryMechanism1, leastRequest, true)) - .addEqualityGroup(new ClusterResolverConfig( - edsDiscoveryMechanismWithOutlierDetection, - leastRequest, - false)) - .testEquals(); - } - private void startXdsDepManager() { startXdsDepManager(new CdsConfig(CLUSTER)); } @@ -1262,7 +1204,6 @@ private final class FakeLoadBalancer extends LoadBalancer { private final Helper helper; private List addresses; private Object config; - private Status upstreamError; private boolean shutdown; FakeLoadBalancer(String name, Helper helper) { @@ -1279,7 +1220,6 @@ public Status acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) { @Override public void handleNameResolutionError(Status error) { - upstreamError = error; } @Override