diff --git a/polaris-common/polaris-client/src/main/java/com/tencent/polaris/client/flow/BaseFlow.java b/polaris-common/polaris-client/src/main/java/com/tencent/polaris/client/flow/BaseFlow.java index 8295f0328..5b6a8875e 100644 --- a/polaris-common/polaris-client/src/main/java/com/tencent/polaris/client/flow/BaseFlow.java +++ b/polaris-common/polaris-client/src/main/java/com/tencent/polaris/client/flow/BaseFlow.java @@ -96,6 +96,7 @@ public static Instance commonGetOneInstance(Extensions extensions, ServiceKey se ServiceConfig serviceConfig = extensions.getConfiguration().getProvider().getService(); RouteInfo routeInfo = new RouteInfo( null, null, dstSvcInfo, null, "", serviceConfig); + routeInfo.putRouterMetadata("metadataRoute", metadata); ResourcesResponse resourcesResponse = BaseFlow .syncGetResources(extensions, false, provider, flowControlParam); LOG.debug("[ConnectionManager]success to discover service {}", svcEventKey); diff --git a/polaris-common/polaris-client/src/main/java/com/tencent/polaris/client/remote/ServiceAddressRepository.java b/polaris-common/polaris-client/src/main/java/com/tencent/polaris/client/remote/ServiceAddressRepository.java index 3dd4e5a97..eea419ae3 100644 --- a/polaris-common/polaris-client/src/main/java/com/tencent/polaris/client/remote/ServiceAddressRepository.java +++ b/polaris-common/polaris-client/src/main/java/com/tencent/polaris/client/remote/ServiceAddressRepository.java @@ -24,11 +24,7 @@ import com.tencent.polaris.api.plugin.common.PluginTypes; import com.tencent.polaris.api.plugin.compose.Extensions; import com.tencent.polaris.api.plugin.loadbalance.LoadBalancer; -import com.tencent.polaris.api.pojo.DefaultInstance; -import com.tencent.polaris.api.pojo.DefaultServiceInstances; -import com.tencent.polaris.api.pojo.Instance; -import com.tencent.polaris.api.pojo.ServiceInstances; -import com.tencent.polaris.api.pojo.ServiceKey; +import com.tencent.polaris.api.pojo.*; import com.tencent.polaris.api.rpc.Criteria; import com.tencent.polaris.api.utils.CollectionUtils; import com.tencent.polaris.api.utils.IPAddressUtils; @@ -190,11 +186,8 @@ public int nodeListSize() { } private Instance getDiscoverInstance() throws PolarisException { - Instance instance = BaseFlow.commonGetOneInstance(extensions, remoteCluster, routers, lbPolicy, protocol, + return BaseFlow.commonGetOneInstance(extensions, remoteCluster, routers, lbPolicy, protocol, clientId); - LOG.info("success to get instance for service {}, instance is {}:{}", remoteCluster, instance.getHost(), - instance.getPort()); - return instance; } @JustForTest diff --git a/polaris-plugins/polaris-plugin-api/src/main/java/com/tencent/polaris/api/plugin/route/RouteInfo.java b/polaris-plugins/polaris-plugin-api/src/main/java/com/tencent/polaris/api/plugin/route/RouteInfo.java index 8fcf8017d..8416e5dc9 100644 --- a/polaris-plugins/polaris-plugin-api/src/main/java/com/tencent/polaris/api/plugin/route/RouteInfo.java +++ b/polaris-plugins/polaris-plugin-api/src/main/java/com/tencent/polaris/api/plugin/route/RouteInfo.java @@ -253,6 +253,15 @@ public Map getRouterMetadata(String routerType) { return Collections.unmodifiableMap(metadata); } + public void putRouterMetadata(String routerType, Map metadata) { + Map tempMetadata = routerMetadata.get(routerType); + if (tempMetadata == null || tempMetadata.isEmpty()) { + tempMetadata = new HashMap<>(); + routerMetadata.put(routerType, tempMetadata); + } + tempMetadata.putAll(metadata); + } + public void setRouterArguments(Map> routerArguments) { Map> routerMetadata = this.routerMetadata; diff --git a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/AsyncRateLimitConnector.java b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/AsyncRateLimitConnector.java index ac65da47c..c205b996b 100644 --- a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/AsyncRateLimitConnector.java +++ b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/AsyncRateLimitConnector.java @@ -84,7 +84,13 @@ public StreamCounterSet getStreamCounterSet(Extensions extensions, ServiceKey re } if (null != streamCounterSet) { //切换了节点,去掉初始化记录 - streamCounterSet.deleteInitRecord(serviceIdentifier); + InitializeRecord removedRecord = streamCounterSet.deleteInitRecord(serviceIdentifier); + if (removedRecord != null) { + RateLimitWindow rateLimitWindow = removedRecord.getRateLimitWindow(); + uniqueKey = rateLimitWindow != null ? rateLimitWindow.getUniqueKey() : null; + LOG.info("[getStreamCounterSet] host switched, and initRecord removed serviceIdentifier: {}, window " + + "{} {}", serviceIdentifier, rateLimitWindow, uniqueKey); + } //切换了节点,老的不再使用 if (streamCounterSet.decreaseReference()) { nodeToStream.remove(node); diff --git a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/QuotaFlow.java b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/QuotaFlow.java index 634ed9845..9c11bdb53 100644 --- a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/QuotaFlow.java +++ b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/QuotaFlow.java @@ -92,6 +92,15 @@ public void init(Extensions extensions) throws PolarisException { FlowCache flowCache = extensions.getFlowCache(); return flowCache.loadPluginCacheObject(API_ID, key, path -> TrieUtil.buildSimpleApiTrieNode((String) path)); }; + rateLimitExtension.submitExpireJob(() -> { + try { + for (Map.Entry entry : svcToWindowSet.entrySet()) { + entry.getValue().cleanupContainers(); + } + } catch (Throwable e) { + LOG.error("Failed to cleanup expired rate limit window", e); + } + }); // init tsf rate limit master utils if need Map metadata = rateLimitConfig.getMetadata(); diff --git a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/RateLimitExtension.java b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/RateLimitExtension.java index f4db90fed..14c2d9332 100644 --- a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/RateLimitExtension.java +++ b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/RateLimitExtension.java @@ -25,20 +25,22 @@ import com.tencent.polaris.api.utils.StringUtils; import com.tencent.polaris.api.utils.ThreadPoolUtils; import com.tencent.polaris.client.util.NamedThreadFactory; +import com.tencent.polaris.logging.LoggerFactory; import com.tencent.polaris.ratelimit.client.sync.RemoteSyncTask; -import com.tencent.polaris.ratelimit.client.utils.RateLimitConstants; import com.tencent.polaris.specification.api.v1.traffic.manage.RateLimitProto; +import org.slf4j.Logger; import java.util.Collection; import java.util.HashMap; import java.util.Map; -import java.util.Random; import java.util.concurrent.*; import static com.tencent.polaris.api.plugin.ratelimiter.ServiceRateLimiter.*; public class RateLimitExtension extends Destroyable { + private static final Logger LOG = LoggerFactory.getLogger(RateLimitExtension.class); + private final Extensions extensions; private final Map rateLimiters = new HashMap<>(); @@ -113,9 +115,17 @@ private String getRateLimiterName(RateLimitProto.Rule.Resource resource, String * @param task 任务 */ public void submitSyncTask(RemoteSyncTask task, long initialDelay, long delay) { + if (scheduledTasks.containsKey(task.getWindow().getUniqueKey())) { + LOG.warn("task has exist, ignore, task {}, window {}, uniqueKey {} ", task, task.getWindow(), + task.getWindow().getUniqueKey()); + task.getWindow().setStatus(RateLimitWindow.WindowStatus.CREATED.ordinal()); + return; + } ScheduledFuture scheduledFuture = syncExecutor - .scheduleWithFixedDelay(task, 0, delay, TimeUnit.MILLISECONDS); + .scheduleWithFixedDelay(task, initialDelay, delay, TimeUnit.MILLISECONDS); scheduledTasks.put(task.getWindow().getUniqueKey(), scheduledFuture); + LOG.info("submit sync task success, task {}, future {}, window {}, uniqueKey {} ", task, scheduledFuture, + task.getWindow(), task.getWindow().getUniqueKey()); } private static final int EXPIRE_INTERVAL_SECOND = 5; @@ -130,8 +140,33 @@ public void submitExpireJob(Runnable task) { .scheduleWithFixedDelay(task, EXPIRE_INTERVAL_SECOND, EXPIRE_INTERVAL_SECOND, TimeUnit.SECONDS); } - public void stopSyncTask(String uniqueKey) { + /** + * 停止同步任务 + * + * @param uniqueKey 窗口唯一标识 + * @param window 限流窗口 + */ + public void stopSyncTask(String uniqueKey, RateLimitWindow window) { + // 从connector初始化列表清理 + Runnable cleanTask = () -> { + try { + AsyncRateLimitConnector connector = window.getWindowSet().getAsyncRateLimitConnector(); + ServiceIdentifier identifier = new ServiceIdentifier(window.getSvcKey().getService(), + window.getSvcKey().getNamespace(), window.getLabels()); + StreamCounterSet streamCounterSet = connector.getStreamCounterSet( + window.getWindowSet().getRateLimitExtension().getExtensions(), + window.getRemoteCluster(), window.getServiceAddressRepository(), window.getUniqueKey(), identifier); + if (streamCounterSet != null) { + streamCounterSet.deleteInitRecord(identifier, window); + } + LOG.info("clean task run success, window {}", window); + } catch (Throwable e) { + LOG.error("clean task run failed, window {}", window.getUniqueKey(), e); + } + }; + syncExecutor.schedule(cleanTask, 10, TimeUnit.MILLISECONDS); ScheduledFuture future = scheduledTasks.remove(uniqueKey); + LOG.info("scheduledTasks remove uniqueKey {}, future {}", uniqueKey, future); if (null != future) { future.cancel(true); } diff --git a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/RateLimitWindow.java b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/RateLimitWindow.java index d4b9e6f25..b678037c2 100644 --- a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/RateLimitWindow.java +++ b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/RateLimitWindow.java @@ -18,13 +18,14 @@ package com.tencent.polaris.ratelimit.client.flow; import com.tencent.polaris.api.config.consumer.LoadBalanceConfig; +import com.tencent.polaris.api.config.consumer.ServiceRouterConfig; import com.tencent.polaris.api.config.provider.RateLimitConfig; import com.tencent.polaris.api.plugin.compose.Extensions; import com.tencent.polaris.api.plugin.ratelimiter.InitCriteria; import com.tencent.polaris.api.plugin.ratelimiter.QuotaBucket; import com.tencent.polaris.api.plugin.ratelimiter.QuotaResult; import com.tencent.polaris.api.plugin.ratelimiter.ServiceRateLimiter; -import com.tencent.polaris.api.pojo.*; +import com.tencent.polaris.api.pojo.ServiceKey; import com.tencent.polaris.api.utils.StringUtils; import com.tencent.polaris.client.flow.FlowControlParam; import com.tencent.polaris.client.remote.ServiceAddressRepository; @@ -39,11 +40,12 @@ import com.tencent.polaris.specification.api.v1.traffic.manage.RateLimitProto.Amount; import com.tencent.polaris.specification.api.v1.traffic.manage.RateLimitProto.RateLimitCluster; import com.tencent.polaris.specification.api.v1.traffic.manage.RateLimitProto.Rule; -import java.util.Random; import org.slf4j.Logger; +import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -97,6 +99,8 @@ public enum WindowStatus { private final AtomicLong lastInitTimeMs = new AtomicLong(); + private final AtomicLong lastSyncTimeMs = new AtomicLong(); + // 执行正式分配的令牌桶 private final QuotaBucket allocatingBucket; @@ -144,7 +148,7 @@ public RateLimitWindow(RateLimitWindowSet windowSet, CommonQuotaRequest quotaReq this.syncParam = quotaRequest.getFlowControlParam(); remoteCluster = getLimiterClusterService(rule.getCluster(), rateLimitConfig); serviceAddressRepository = buildServiceAddressRepository(rateLimitConfig.getLimiterAddresses(), - uniqueKey, windowSet.getExtensions(), remoteCluster, null, LoadBalanceConfig.LOAD_BALANCE_RING_HASH, "grpc"); + uniqueKey, windowSet.getExtensions(), remoteCluster); allocatingBucket = getQuotaBucket(initCriteria, windowSet.getRateLimitExtension()); lastAccessTimeMs.set(System.currentTimeMillis()); this.rateLimitConfig = rateLimitConfig; @@ -152,8 +156,10 @@ public RateLimitWindow(RateLimitWindowSet windowSet, CommonQuotaRequest quotaReq } private ServiceAddressRepository buildServiceAddressRepository(List addresses, String hash, Extensions extensions, - ServiceKey remoteCluster, List routers, String lbPolicy, String protocol) { - return new ServiceAddressRepository(addresses, hash, extensions, remoteCluster, routers, lbPolicy, protocol); + ServiceKey remoteCluster) { + List routers = new ArrayList<>(); + routers.add(ServiceRouterConfig.DEFAULT_ROUTER_METADATA); + return new ServiceAddressRepository(addresses, hash, extensions, remoteCluster, routers, LoadBalanceConfig.LOAD_BALANCE_RING_HASH, "grpc"); } @@ -249,10 +255,12 @@ public void init() { } if (configMode == RateLimitConstants.CONFIG_QUOTA_LOCAL_MODE && !isTsfCluster) { //本地限流,则直接可用 + LOG.info("[RateLimitWindow] local window {} initiated", this); status.set(WindowStatus.INITIALIZED.ordinal()); return; } //加入轮询队列,走异步调度 + LOG.info("[RateLimitWindow] remote window {} first init", this); if (rule.getMetadataMap().containsKey("limiter") && StringUtils.equalsIgnoreCase("tsf", rule.getMetadataMap().get("limiter"))) { windowSet.getRateLimitExtension().submitSyncTask(new TsfRemoteSyncTask(this), 0L, 1000L); @@ -270,8 +278,13 @@ public void unInit() { return; } status.set(WindowStatus.DELETED.ordinal()); + LOG.info("[RateLimitWindow] window {} {} is set to DELETED", uniqueKey, this); //从轮询队列中剔除 - windowSet.getRateLimitExtension().stopSyncTask(uniqueKey); + if (configMode == RateLimitConstants.CONFIG_QUOTA_LOCAL_MODE) { + return; + } + LOG.info("[RateLimitWindow] stopSyncTask( uniqueKey {}, window {} ) ", uniqueKey, this); + windowSet.getRateLimitExtension().stopSyncTask(uniqueKey, this); } } @@ -301,16 +314,21 @@ public void returnQuota(CommonQuotaRequest request) { /** * 窗口已经过期 + * TSF 设置为不过期 * * @return boolean */ public boolean isExpired() { - long curTimeMs = System.currentTimeMillis(); - boolean expired = curTimeMs - lastAccessTimeMs.get() > expireDurationMs; - if (expired) { - LOG.info("[RateLimit]window has expired, expireDurationMs {}, uniqueKey {}", expireDurationMs, uniqueKey); + if (!isTsfCluster) { + long curTimeMs = System.currentTimeMillis(); + boolean expired = curTimeMs - lastAccessTimeMs.get() > expireDurationMs; + if (expired) { + LOG.info("[RateLimit] window has expired, expireDurationMs {}, uniqueKey {}, window {}", expireDurationMs, + uniqueKey, this); + } + return expired; } - return expired; + return false; } public long getLastInitTimeMs() { @@ -321,6 +339,14 @@ public void setLastInitTimeMs(long lastInitTimeMs) { this.lastInitTimeMs.set(lastInitTimeMs); } + public long getLastSyncTimeMs() { + return lastSyncTimeMs.get(); + } + + public void setLastSyncTimeMs(long lastSyncTimeMs) { + this.lastSyncTimeMs.set(lastSyncTimeMs); + } + /** * 获取当前窗口的状态 * diff --git a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/RateLimitWindowSet.java b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/RateLimitWindowSet.java index 94a466087..5de4bc5dd 100644 --- a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/RateLimitWindowSet.java +++ b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/RateLimitWindowSet.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; public class RateLimitWindowSet { @@ -133,6 +134,26 @@ public void deleteRules(Set rules) { } } + /** + * 过期清理单个rule下所有WindowContainer + */ + public void cleanupContainers() { + AtomicInteger rulesExpired = new AtomicInteger(0); + windowByRule.entrySet().removeIf(entry -> { + boolean expired = entry.getValue().checkAndCleanExpiredWindows(); + if (expired) { + rulesExpired.incrementAndGet(); + LOG.info("[RateLimitWindowSet] rule {} for service {} has been expired, window container {}", + entry.getKey(), serviceKey, entry.getValue()); + } + return expired; + }); + if (rulesExpired.get() > 0) { + LOG.info("[RateLimitWindowSet] {} rules have been cleaned up due to expiration, service {}", + rulesExpired, serviceKey); + } + } + public AsyncRateLimitConnector getAsyncRateLimitConnector() { return asyncRateLimitConnector; } diff --git a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/StreamCounterSet.java b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/StreamCounterSet.java index 65ad435a5..8253e07ff 100644 --- a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/StreamCounterSet.java +++ b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/StreamCounterSet.java @@ -19,12 +19,11 @@ import com.tencent.polaris.client.pojo.Node; import com.tencent.polaris.logging.LoggerFactory; +import org.slf4j.Logger; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import org.slf4j.Logger; - /** * 计数器对象 */ @@ -97,11 +96,20 @@ public boolean decreaseReference() { return false; } - public void deleteInitRecord(ServiceIdentifier serviceIdentifier) { + public InitializeRecord deleteInitRecord(ServiceIdentifier serviceIdentifier) { + StreamResource streamResource = currentStreamResource.get(); + if (null != streamResource) { + return streamResource.deleteInitRecord(serviceIdentifier); + } + return null; + } + + public InitializeRecord deleteInitRecord(ServiceIdentifier serviceIdentifier, RateLimitWindow window) { StreamResource streamResource = currentStreamResource.get(); if (null != streamResource) { - streamResource.deleteInitRecord(serviceIdentifier); + return streamResource.deleteInitRecord(serviceIdentifier, window); } + return null; } diff --git a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/StreamResource.java b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/StreamResource.java index 9488323f8..b83fd984e 100644 --- a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/StreamResource.java +++ b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/StreamResource.java @@ -122,6 +122,7 @@ public StreamResource(Node node) { private ManagedChannel createConnection(Node node) { ManagedChannelBuilder builder = ManagedChannelBuilder.forAddress(node.getHost(), node.getPort()) .usePlaintext(); + LOG.info("[ServerConnector]connection {} start to connect", node); return builder.build(); } @@ -181,17 +182,35 @@ public InitializeRecord getInitRecord(ServiceIdentifier serviceIdentifier, RateL if (record == null) { LOG.info("[RateLimit] add init record for {}, stream is {}", serviceIdentifier, this.hostNode); initRecord.putIfAbsent(serviceIdentifier, new InitializeRecord(rateLimitWindow)); + LOG.info("[RateLimit] record is null, write init record for task window is {} {} {}", rateLimitWindow, + rateLimitWindow.getUniqueKey(), rateLimitWindow.getStatus()); } else if (record.getRateLimitWindow() != rateLimitWindow) { // 存在旧窗口映射关系,说明已经淘汰 initRecord.put(serviceIdentifier, new InitializeRecord(rateLimitWindow)); RateLimitWindow oldWindow = record.getRateLimitWindow(); - LOG.info("remove init record for window {} {}", oldWindow.getUniqueKey(), oldWindow.getStatus()); + LOG.warn("[RateLimit] remove init record for window {} {} {}, task window is {} {} {}", oldWindow, + oldWindow.getUniqueKey(), oldWindow.getStatus(), rateLimitWindow, + rateLimitWindow.getUniqueKey(), rateLimitWindow.getStatus()); } return initRecord.get(serviceIdentifier); } - public void deleteInitRecord(ServiceIdentifier serviceIdentifier) { + public InitializeRecord deleteInitRecord(ServiceIdentifier serviceIdentifier) { LOG.info("[RateLimit] delete init record for {}, stream is {}", serviceIdentifier, this.hostNode); - initRecord.remove(serviceIdentifier); + return initRecord.remove(serviceIdentifier); + } + + // 淘汰时删除窗口初始化映射信息 + public InitializeRecord deleteInitRecord(ServiceIdentifier serviceIdentifier, RateLimitWindow rateLimitWindow) { + InitializeRecord record = initRecord.get(serviceIdentifier); + if (record != null && record.getRateLimitWindow() == rateLimitWindow) { + record.getDurationRecord().forEach((duration, counterKey) -> counters.remove(counterKey)); + initRecord.remove(serviceIdentifier); + LOG.info("[RateLimit] delete init record for {}, window {}", serviceIdentifier, rateLimitWindow.getUniqueKey()); + } else if (record != null && record.getRateLimitWindow() != rateLimitWindow) { + String recordWindow = record.getRateLimitWindow() != null ? record.getRateLimitWindow().getUniqueKey() : null; + LOG.warn("[RateLimit] delete init record for {}, window {} failed with {}", serviceIdentifier, rateLimitWindow.getUniqueKey(), recordWindow); + } + return record; } /** @@ -224,21 +243,35 @@ private void handleRateLimitInitResponse(RateLimitInitResponse rateLimitInitResp return; } //重新初始化后,之前的记录就不要了 + RateLimitWindow rateLimitWindow = initializeRecord.getRateLimitWindow(); + initializeRecord.getDurationRecord().forEach((duration, counterKey) -> counters.remove(counterKey)); initializeRecord.getDurationRecord().clear(); long remoteQuotaTimeMilli = rateLimitInitResponse.getTimestamp(); long localQuotaTimeMilli = getLocalTimeMilli(remoteQuotaTimeMilli); - RateLimitWindow rateLimitWindow = initializeRecord.getRateLimitWindow(); + long currentTimeMs = System.currentTimeMillis(); + rateLimitWindow.setLastSyncTimeMs(currentTimeMs); rateLimitWindow.setLastInitTimeMs(0); // 重置上次初始化时间,从而在metric变更或上报失败时可再次立刻再初始化 countersList.forEach(counter -> { initializeRecord.getDurationRecord().putIfAbsent(counter.getDuration(), counter.getCounterKey()); - counters.putIfAbsent(counter.getCounterKey(), - new DurationBaseCallback(counter.getDuration(), rateLimitWindow)); + DurationBaseCallback callback = new DurationBaseCallback(counter.getDuration(), rateLimitWindow); + DurationBaseCallback pre = counters.putIfAbsent(counter.getCounterKey(), callback); + if (pre != null && pre.getRateLimitWindow() != rateLimitWindow) { + counters.put(counter.getCounterKey(), callback); + LOG.warn("[handleRateLimitInitResponse] remove counter for window {}, new window {} {}", + pre.getRateLimitWindow().getUniqueKey(), rateLimitWindow.getUniqueKey(), + counter.getCounterKey()); + } RemoteQuotaInfo remoteQuotaInfo = new RemoteQuotaInfo(counter.getLeft(), counter.getClientCount(), - localQuotaTimeMilli, counter.getDuration() * 1000); + localQuotaTimeMilli, counter.getDuration() * 1000L); rateLimitWindow.getAllocatingBucket().onRemoteUpdate(remoteQuotaInfo); }); - LOG.info("[RateLimit] window {} has turn to initialized", rateLimitWindow.getUniqueKey()); - rateLimitWindow.setStatus(WindowStatus.INITIALIZED.ordinal()); + if (rateLimitWindow.getStatus() == WindowStatus.INITIALIZING) { + LOG.info("[handleRateLimitInitResponse] window {} has turn to initialized", rateLimitWindow.getUniqueKey()); + rateLimitWindow.setStatus(WindowStatus.INITIALIZED.ordinal()); + } else { + LOG.warn("[handleRateLimitInitResponse] failed to set window to INITIALIZED. window {} {}, status {} ", + rateLimitWindow, rateLimitWindow.getUniqueKey(), rateLimitWindow.getStatus()); + } } /** @@ -264,6 +297,7 @@ boolean handleRateLimitReportResponse(RateLimitReportResponse rateLimitReportRes RemoteQuotaInfo remoteQuotaInfo = new RemoteQuotaInfo(quotaLeft.getLeft(), quotaLeft.getClientCount(), localQuotaTimeMilli, callback.getDuration() * 1000); callback.getRateLimitWindow().getAllocatingBucket().onRemoteUpdate(remoteQuotaInfo); + callback.getRateLimitWindow().setLastSyncTimeMs(System.currentTimeMillis()); }); return true; } @@ -339,8 +373,30 @@ public boolean sendRateLimitRequest(RateLimitRequest rateLimitRequest) { } } - public boolean hasInit(ServiceIdentifier serviceIdentifier) { - return initRecord.containsKey(serviceIdentifier); + public boolean hasInit(ServiceIdentifier serviceIdentifier, RateLimitWindow rateLimitWindow) { + InitializeRecord record = initRecord.get(serviceIdentifier); + if (record == null || record.getDurationRecord().isEmpty()) { + return false; + } + if (record.getRateLimitWindow() != rateLimitWindow) { + record.getDurationRecord().forEach((duration, counterKey) -> counters.remove(counterKey)); + initRecord.remove(serviceIdentifier); // 清理索引触发重新初始化 + LOG.warn("[hasInit] init record {} is removed for switched. record window {} {}, param window {} {}", + initRecord, record.getRateLimitWindow(), record.getRateLimitWindow().getUniqueKey(), rateLimitWindow, + rateLimitWindow.getUniqueKey()); + return false; + } + if (System.currentTimeMillis() - rateLimitWindow.getLastSyncTimeMs() + > RateLimitConstants.WINDOW_INDEX_EXPIRE_TIME) { + record.getDurationRecord().forEach((duration, counterKey) -> counters.remove(counterKey)); + initRecord.remove(serviceIdentifier); // 清理索引触发重新初始化 + LOG.warn("[hasInit] init record is removed for expired. last sync time {}. " + + "record window {} {}, param window {} {}. ", rateLimitWindow.getLastSyncTimeMs(), + record.getRateLimitWindow(), record.getRateLimitWindow().getUniqueKey(), rateLimitWindow, + rateLimitWindow.getUniqueKey()); + return false; + } + return true; } public Integer getCounterKey(ServiceIdentifier serviceIdentifier, Integer duration) { diff --git a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/WindowContainer.java b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/WindowContainer.java index dcda1f4db..ee3c02e45 100644 --- a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/WindowContainer.java +++ b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/flow/WindowContainer.java @@ -19,10 +19,12 @@ import com.tencent.polaris.api.pojo.ServiceKey; import com.tencent.polaris.logging.LoggerFactory; +import org.slf4j.Logger; + +import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; -import org.slf4j.Logger; public class WindowContainer { @@ -71,4 +73,36 @@ public RateLimitWindow getMainWindow() { return mainWindow; } + /** + * 检查并淘汰窗口 + * + * @return 是否淘汰 + */ + public boolean checkAndCleanExpiredWindows() { + if (null != mainWindow) { + if (mainWindow.isExpired()) { + LOG.info("[RateLimit] mainWindow have been cleaned up due to expiration, service {}", serviceKey); + mainWindow.unInit(); + } + return mainWindow.isExpired(); + } + int expiredLabels = 0; + Iterator> iterator = windowByLabel.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + String labelKey = entry.getKey(); + RateLimitWindow window = entry.getValue(); + if (window.isExpired()) { + expiredLabels++; + iterator.remove(); // 使用迭代器的 remove 方法删除当前元素 + LOG.info("[WindowContainer] windowByLabel remove label key {} , window {}", labelKey, window); + window.unInit(); + } + } + if (expiredLabels > 0) { + LOG.info("[RateLimit] {} labels have been cleaned up due to expiration, service {}", expiredLabels, + serviceKey); + } + return windowByLabel.isEmpty(); + } } diff --git a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/sync/PolarisRemoteSyncTask.java b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/sync/PolarisRemoteSyncTask.java index b9587dbc8..f33c89c4b 100644 --- a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/sync/PolarisRemoteSyncTask.java +++ b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/sync/PolarisRemoteSyncTask.java @@ -182,7 +182,7 @@ private void doRemoteAcquire() { } StreamResource streamResource = streamCounterSet.checkAndCreateResource(serviceIdentifier, window); - if (!streamResource.hasInit(serviceIdentifier)) { + if (!streamResource.hasInit(serviceIdentifier, window)) { doRemoteInit(true); return; } diff --git a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/utils/RateLimitConstants.java b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/utils/RateLimitConstants.java index 44414016c..e0fb54e3f 100644 --- a/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/utils/RateLimitConstants.java +++ b/polaris-ratelimit/polaris-ratelimit-client/src/main/java/com/tencent/polaris/ratelimit/client/utils/RateLimitConstants.java @@ -70,7 +70,12 @@ public interface RateLimitConstants { /** * 等待服务端返回结果的时间 1000ms */ - int INIT_WAIT_RESPONSE_TIME = 1 * 1000; + int INIT_WAIT_RESPONSE_TIME = 1000; + + /** + * 窗口索引有效期时间 2000ms,服务器2000ms未收到请求则淘汰窗口 + */ + int WINDOW_INDEX_EXPIRE_TIME = 2000; /** * 服务端的返回code diff --git a/polaris-ratelimit/polaris-ratelimit-factory/src/test/java/com/tencent/polaris/ratelimit/test/core/SubmitPolarisRemoteSyncTaskTest.java b/polaris-ratelimit/polaris-ratelimit-factory/src/test/java/com/tencent/polaris/ratelimit/test/core/SubmitPolarisRemoteSyncTaskTest.java new file mode 100644 index 000000000..842bb31d7 --- /dev/null +++ b/polaris-ratelimit/polaris-ratelimit-factory/src/test/java/com/tencent/polaris/ratelimit/test/core/SubmitPolarisRemoteSyncTaskTest.java @@ -0,0 +1,165 @@ +/* + * Tencent is pleased to support the open source community by making polaris-java available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * Licensed under the BSD 3-Clause License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/BSD-3-Clause + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package com.tencent.polaris.ratelimit.test.core; + +import com.tencent.polaris.api.plugin.compose.Extensions; +import com.tencent.polaris.client.api.SDKContext; +import com.tencent.polaris.factory.config.ConfigurationImpl; +import com.tencent.polaris.ratelimit.client.flow.QuotaFlow; +import com.tencent.polaris.ratelimit.client.flow.RateLimitExtension; +import com.tencent.polaris.ratelimit.client.flow.RateLimitWindow; +import com.tencent.polaris.ratelimit.client.sync.RemoteSyncTask; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import java.lang.reflect.Field; +import java.util.Map; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.*; + +public class SubmitPolarisRemoteSyncTaskTest { + + @Mock + private Extensions extensions; + + @Mock + private ScheduledExecutorService syncExecutor; + + @Mock + private ScheduledFuture mockFuture; + + @Mock + private RemoteSyncTask remoteSyncTask; + + @Mock + private RateLimitWindow rateLimitWindow; + + private SDKContext context; + + private RateLimitExtension rateLimitExtension; + + private Map> scheduledTasks; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + QuotaFlow quotaFlow = new QuotaFlow(); + ConfigurationImpl config = new ConfigurationImpl(); + config.setDefault(); + config.getGlobal().getAPI().setReportEnable(false); + config.getGlobal().getStatReporter().setEnable(false); + context = SDKContext.initContextByConfig(config); + context.init(); + Extensions extensions = context.getExtensions(); + rateLimitExtension = new RateLimitExtension(extensions); + + // 使用反射替换 syncExecutor,避免真实线程池执行 + Field syncExecutorField = RateLimitExtension.class.getDeclaredField("syncExecutor"); + syncExecutorField.setAccessible(true); + syncExecutorField.set(rateLimitExtension, syncExecutor); + + // 使用反射获取 scheduledTasks + Field scheduledTasksField = RateLimitExtension.class.getDeclaredField("scheduledTasks"); + scheduledTasksField.setAccessible(true); + scheduledTasks = (Map>) scheduledTasksField.get(rateLimitExtension); + + // Mock 限流窗口 + when(remoteSyncTask.getWindow()).thenReturn(rateLimitWindow); + when(rateLimitWindow.getUniqueKey()).thenReturn("test-unique-key"); + } + + @After + public void tearDown() throws Exception { + context.destroy(); + } + + /* + case_name: 提交同步任务测试 + case_path: 访问限流/同步任务 + case_description: 测试在任务存在时,不重新提交任务 + */ + @Test + public void testSubmitPolarisRemoteSyncTask_TaskAlreadyExists() { + // 先放入一个任务 + scheduledTasks.put("test-unique-key", mockFuture); + + // 执行 submitSyncTask + rateLimitExtension.submitSyncTask(remoteSyncTask, 0, 30); + + // 确保任务没有被重新提交 + verify(syncExecutor, never()).scheduleWithFixedDelay(any(), anyLong(), anyLong(), any()); + + // 确保任务状态被设置为 CREATED + verify(rateLimitWindow, times(1)).setStatus(RateLimitWindow.WindowStatus.CREATED.ordinal()); + + // 确保 scheduledTasks 仍然是原来的任务 + assertEquals(mockFuture, scheduledTasks.get("test-unique-key")); + } + + /* + case_name: 提交同步任务测试 + case_path: 访问限流/同步任务 + case_description: 测试在任务不存在时,可以提交任务 + */ + @Test + public void testSubmitPolarisRemoteSyncTask_NewTask() { + // Mock 任务调度 + Mockito.>when( + syncExecutor.scheduleWithFixedDelay(any(Runnable.class), anyLong(), anyLong(), any(TimeUnit.class)) + ).thenReturn(mockFuture); + + // 执行 submitSyncTask + rateLimitExtension.submitSyncTask(remoteSyncTask, 0, 30); + + // 确保任务被提交 + verify(syncExecutor, times(1)).scheduleWithFixedDelay(eq(remoteSyncTask), eq(0L), anyLong(), + eq(TimeUnit.MILLISECONDS)); + + // 确保任务被添加到 scheduledTasks + assertEquals(mockFuture, scheduledTasks.get("test-unique-key")); + } + + /* + case_name: 提交同步任务测试 + case_path: 访问限流/同步任务 + case_description: 测试删除任务时,停止任务 + */ + @Test + public void testStopPolarisRemoteSyncTask() { + // 先放入一个任务 + scheduledTasks.put("test-unique-key", mockFuture); + + // 执行 stopSyncTask + rateLimitExtension.stopSyncTask("test-unique-key", rateLimitWindow); + + // 确保任务被移除 + assertNull(scheduledTasks.get("test-unique-key")); + + // 确保 future.cancel(true) 被调用 + verify(mockFuture, times(1)).cancel(true); + } +} \ No newline at end of file