[Directory] Strengthen single activation guarantees even further.

src/Orleans.Runtime/Configuration/Options/GrainDirectoryOptions.cs

@@ -1,99 +1,117 @@
-
-using System;
 using Orleans.Runtime.GrainDirectory;
 
-namespace Orleans.Configuration
+namespace Orleans.Configuration;
+
+public class GrainDirectoryOptions
 {
-    public class GrainDirectoryOptions
+    /// <summary>
+    /// Configuration type that controls the type of the grain directory caching algorithm that silo use.
+    /// </summary>
+    public enum CachingStrategyType
     {
-        /// <summary>
-        /// Configuration type that controls the type of the grain directory caching algorithm that silo use.
-        /// </summary>
-        public enum CachingStrategyType
-        {
-            /// <summary>Don't cache.</summary>
-            None,
-            /// <summary>Standard fixed-size LRU.</summary>
-            LRU,
-            /// <summary>Adaptive caching with fixed maximum size and refresh. This option should be used in production.</summary>
-            [Obsolete("Adaptive caching is deprecated in favor of LRU and will be removed in a future version. This value is now an alias for LRU.")]
-            Adaptive,
-            /// <summary>Custom cache implementation, configured by registering an <see cref="IGrainDirectoryCache"/> implementation in the dependency injection container.</summary>
-            Custom
-        }
+        /// <summary>Don't cache.</summary>
+        None,
+        /// <summary>Standard fixed-size LRU.</summary>
+        LRU,
+        /// <summary>Adaptive caching with fixed maximum size and refresh. This option should be used in production.</summary>
+        [Obsolete("Adaptive caching is deprecated in favor of LRU and will be removed in a future version. This value is now an alias for LRU.")]
+        Adaptive,
+        /// <summary>Custom cache implementation, configured by registering an <see cref="IGrainDirectoryCache"/> implementation in the dependency injection container.</summary>
+        Custom
+    }
 
-        /// <summary>
-        /// Gets or sets the caching strategy to use.
-        /// The options are None, which means don't cache directory entries locally;
-        /// LRU, which indicates that a standard fixed-size least recently used strategy should be used; and
-        /// Adaptive, which indicates that an adaptive strategy with a fixed maximum size should be used.
-        /// The LRU strategy is used by default.
-        /// </summary>
-        public CachingStrategyType CachingStrategy { get; set; } = DEFAULT_CACHING_STRATEGY;
+    /// <summary>
+    /// Gets or sets the caching strategy to use.
+    /// The options are None, which means don't cache directory entries locally;
+    /// LRU, which indicates that a standard fixed-size least recently used strategy should be used; and
+    /// Adaptive, which indicates that an adaptive strategy with a fixed maximum size should be used.
+    /// The LRU strategy is used by default.
+    /// </summary>
+    public CachingStrategyType CachingStrategy { get; set; } = DEFAULT_CACHING_STRATEGY;
 
-        /// <summary>
-        /// The default value for <see cref="CachingStrategy"/>.
-        /// </summary>
-        public const CachingStrategyType DEFAULT_CACHING_STRATEGY = CachingStrategyType.LRU;
+    /// <summary>
+    /// The default value for <see cref="CachingStrategy"/>.
+    /// </summary>
+    public const CachingStrategyType DEFAULT_CACHING_STRATEGY = CachingStrategyType.LRU;
 
-        /// <summary>
-        /// Gets or sets the maximum number of grains to cache directory information for.
-        /// </summary>
-        public int CacheSize { get; set; } = DEFAULT_CACHE_SIZE;
+    /// <summary>
+    /// Gets or sets the maximum number of grains to cache directory information for.
+    /// </summary>
+    public int CacheSize { get; set; } = DEFAULT_CACHE_SIZE;
 
-        /// <summary>
-        /// The default value for <see cref="CacheSize"/>.
-        /// </summary>
-        public const int DEFAULT_CACHE_SIZE = 1_000_000;
+    /// <summary>
+    /// The default value for <see cref="CacheSize"/>.
+    /// </summary>
+    public const int DEFAULT_CACHE_SIZE = 1_000_000;
 
-        /// <summary>
-        /// Gets or sets the initial (minimum) time, in seconds, to keep a cache entry before revalidating.
-        /// </summary>
-        [Obsolete("InitialCacheTTL is deprecated and will be removed in a future version.")]
-        public TimeSpan InitialCacheTTL { get; set; } = DEFAULT_INITIAL_CACHE_TTL;
+    /// <summary>
+    /// Gets or sets the initial (minimum) time, in seconds, to keep a cache entry before revalidating.
+    /// </summary>
+    [Obsolete("InitialCacheTTL is deprecated and will be removed in a future version.")]
+    public TimeSpan InitialCacheTTL { get; set; } = DEFAULT_INITIAL_CACHE_TTL;
 
-        /// <summary>
-        /// The default value for <see cref="InitialCacheTTL"/>.
-        /// </summary>
-        [Obsolete("DEFAULT_INITIAL_CACHE_TTL is deprecated and will be removed in a future version.")]
-        public static readonly TimeSpan DEFAULT_INITIAL_CACHE_TTL = TimeSpan.FromSeconds(30);
+    /// <summary>
+    /// The default value for <see cref="InitialCacheTTL"/>.
+    /// </summary>
+    [Obsolete("DEFAULT_INITIAL_CACHE_TTL is deprecated and will be removed in a future version.")]
+    public static readonly TimeSpan DEFAULT_INITIAL_CACHE_TTL = TimeSpan.FromSeconds(30);
 
-        /// <summary>
-        /// Gets or sets the maximum time, in seconds, to keep a cache entry before revalidating.
-        /// </summary>
-        [Obsolete("MaximumCacheTTL is deprecated and will be removed in a future version.")]
-        public TimeSpan MaximumCacheTTL { get; set; } = DEFAULT_MAXIMUM_CACHE_TTL;
+    /// <summary>
+    /// Gets or sets the maximum time, in seconds, to keep a cache entry before revalidating.
+    /// </summary>
+    [Obsolete("MaximumCacheTTL is deprecated and will be removed in a future version.")]
+    public TimeSpan MaximumCacheTTL { get; set; } = DEFAULT_MAXIMUM_CACHE_TTL;
 
-        /// <summary>
-        /// The default value for <see cref="MaximumCacheTTL"/>.
-        /// </summary>
-        [Obsolete("DEFAULT_MAXIMUM_CACHE_TTL is deprecated and will be removed in a future version.")]
-        public static readonly TimeSpan DEFAULT_MAXIMUM_CACHE_TTL = TimeSpan.FromSeconds(240);
+    /// <summary>
+    /// The default value for <see cref="MaximumCacheTTL"/>.
+    /// </summary>
+    [Obsolete("DEFAULT_MAXIMUM_CACHE_TTL is deprecated and will be removed in a future version.")]
+    public static readonly TimeSpan DEFAULT_MAXIMUM_CACHE_TTL = TimeSpan.FromSeconds(240);
 
-        /// <summary>
-        /// Gets or sets the factor by which cache entry TTLs should be extended when they are found to be stable.
-        /// </summary>
-        [Obsolete("CacheTTLExtensionFactor is deprecated and will be removed in a future version.")]
-        public double CacheTTLExtensionFactor { get; set; } = DEFAULT_TTL_EXTENSION_FACTOR;
+    /// <summary>
+    /// Gets or sets the factor by which cache entry TTLs should be extended when they are found to be stable.
+    /// </summary>
+    [Obsolete("CacheTTLExtensionFactor is deprecated and will be removed in a future version.")]
+    public double CacheTTLExtensionFactor { get; set; } = DEFAULT_TTL_EXTENSION_FACTOR;
 
-        /// <summary>
-        /// The default value for <see cref="CacheTTLExtensionFactor"/>.
-        /// </summary>
-        [Obsolete("DEFAULT_TTL_EXTENSION_FACTOR is deprecated and will be removed in a future version.")]
-        public const double DEFAULT_TTL_EXTENSION_FACTOR = 2.0;
+    /// <summary>
+    /// The default value for <see cref="CacheTTLExtensionFactor"/>.
+    /// </summary>
+    [Obsolete("DEFAULT_TTL_EXTENSION_FACTOR is deprecated and will be removed in a future version.")]
+    public const double DEFAULT_TTL_EXTENSION_FACTOR = 2.0;
 
-        /// <summary>
-        /// Gets or sets the time span between when we have added an entry for an activation to the grain directory and when we are allowed
-        /// to conditionally remove that entry. 
-        /// Conditional deregistration is used for lazy clean-up of activations whose prompt deregistration failed for some reason (e.g., message failure).
-        /// This should always be at least one minute, since we compare the times on the directory partition, so message delays and clcks skues have
-        /// to be allowed.
-        /// </summary>
-        public TimeSpan LazyDeregistrationDelay { get; set; } = DEFAULT_UNREGISTER_RACE_DELAY;
+    /// <summary>
+    /// Gets or sets the time span between when we have added an entry for an activation to the grain directory and when we are allowed
+    /// to conditionally remove that entry. 
+    /// Conditional deregistration is used for lazy clean-up of activations whose prompt deregistration failed for some reason (e.g., message failure).
+    /// This should always be at least one minute, since we compare the times on the directory partition, so message delays and clcks skues have
+    /// to be allowed.
+    /// </summary>
+    public TimeSpan LazyDeregistrationDelay { get; set; } = DEFAULT_UNREGISTER_RACE_DELAY;
 
-        /// <summary>
-        /// The default value for <see cref="LazyDeregistrationDelay"/>.
-        /// </summary>
-        public static readonly TimeSpan DEFAULT_UNREGISTER_RACE_DELAY = TimeSpan.FromMinutes(1);
-    }
+    /// <summary>
+    /// The default value for <see cref="LazyDeregistrationDelay"/>.
+    /// </summary>
+    public static readonly TimeSpan DEFAULT_UNREGISTER_RACE_DELAY = TimeSpan.FromMinutes(1);
+
+    /// <summary>
+    /// Gets or sets the duration for the safety lease hold applied after an ungraceful silo failure.
+    /// This duration applies in two scenarios:
+    /// <list type="bullet">
+    /// <item>When a specific silo crashes ungracefully, grain lease holds prevent individual re-registration of its grains for this duration.</item>
+    /// <item>When a directory partition can not acquire a snapshot from a previous owner, range lease holds prevent new registrations in that whole range for this duration.</item>
+    /// </list>
+    /// </summary>
+    /// <remarks>
+    /// Depending on the value of this, the duration is understood as:
+    /// <list type="bullet">
+    /// <item><c>SafetyLeaseHoldDuration > TimeSpan.Zero</c> - The lease duration is explicitly controlled by the user.</item>
+    /// <item><c>SafetyLeaseHoldDuration = TimeSpan.Zero</c>. No leases are placed at all, effectively nullifying this safety option.</item>
+    /// <item><c>SafetyLeaseHoldDuration = null</c> - The system computes a lease duration as:
+    /// <c>2 × <see cref="ClusterMembershipOptions.ProbeTimeout"/> × <see cref="ClusterMembershipOptions.NumMissedProbesLimit"/></c>.
+    /// This is the default value, and is designed to be long enough to allow for failure detection and cluster stabilization.
+    /// </item>
+    /// </list>
+    /// </remarks>
+    public TimeSpan? SafetyLeaseHoldDuration { get; set; }
 }

src/Orleans.Runtime/GrainDirectory/DirectoryResult.cs

@@ -4,22 +4,40 @@ namespace Orleans.Runtime;
 
 internal static class DirectoryResult
 {
-    public static DirectoryResult<T> FromResult<T>(T result, MembershipVersion version) => new DirectoryResult<T>(result, version);
-    public static DirectoryResult<T> RefreshRequired<T>(MembershipVersion version) => new DirectoryResult<T>(default, version);
+    public static DirectoryResult<T> FromResult<T>(T result, MembershipVersion version) => new(result, version);
+    public static DirectoryResult<T> RefreshRequired<T>(MembershipVersion version) => new(default, version);
+    public static DirectoryResult<T> RetryAfter<T>(TimeSpan retryAfter) => new(retryAfter);
 }
 
 [GenerateSerializer, Alias("DirectoryResult`1"), Immutable]
-internal readonly struct DirectoryResult<T>(T? result, MembershipVersion version)
+internal readonly struct DirectoryResult<T>
 {
     [Id(0)]
-    private readonly T? _result = result;
+    private readonly T? _result;
 
     [Id(1)]
-    public readonly MembershipVersion Version = version;
+    public readonly MembershipVersion Version;
+
+    /// <summary>
+    /// When greater than <see cref="TimeSpan.Zero"/>, indicates that the caller should retry after the specified delay.
+    /// </summary>
+    [Id(2)]
+    public readonly TimeSpan RetryAfterDelay;
+
+    public DirectoryResult(T? result, MembershipVersion version)
+    {
+        _result = result;
+        Version = version;
+    }
+
+    public DirectoryResult(TimeSpan retryAfter)
+    {
+        RetryAfterDelay = retryAfter;
+    }
 
     public bool TryGetResult(MembershipVersion version, [NotNullWhen(true)] out T? result)
     {
-        if (Version == version)
+        if (RetryAfterDelay <= TimeSpan.Zero && Version == version)
         {
             result = _result!;
             return true;