diff --git a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj index d2ce2ed2..3392fea8 100644 --- a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj +++ b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj @@ -21,7 +21,7 @@ - 8.0.0 + 8.1.0 diff --git a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj index 4420db74..d9659c5f 100644 --- a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj +++ b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj @@ -24,7 +24,7 @@ - 8.0.0 + 8.1.0 diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationExtensions.cs index dc022bac..8b4bf8c8 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationExtensions.cs @@ -33,7 +33,9 @@ private static bool IsProviderDisabled() /// /// The configuration builder to add key-values to. /// The connection string used to connect to the configuration store. - /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// will always be thrown when the caller gives an invalid input configuration (connection strings, endpoints, key/label filters...etc). + /// /// The provided configuration builder. public static IConfigurationBuilder AddAzureAppConfiguration( this IConfigurationBuilder configurationBuilder, @@ -48,7 +50,9 @@ public static IConfigurationBuilder AddAzureAppConfiguration( /// /// The configuration builder to add key-values to. /// The list of connection strings used to connect to the configuration store and its replicas. - /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// will always be thrown when the caller gives an invalid input configuration (connection strings, endpoints, key/label filters...etc). + /// /// The provided configuration builder. public static IConfigurationBuilder AddAzureAppConfiguration( this IConfigurationBuilder configurationBuilder, @@ -63,7 +67,9 @@ public static IConfigurationBuilder AddAzureAppConfiguration( /// /// The configuration builder to add key-values to. /// A callback used to configure Azure App Configuration options. - /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// Determines the behavior of the App Configuration provider when an exception occurs while loading data from server. If false, the exception is thrown. If true, the exception is suppressed and no settings are populated from Azure App Configuration. + /// will always be thrown when the caller gives an invalid input configuration (connection strings, endpoints, key/label filters...etc). + /// /// The provided configuration builder. public static IConfigurationBuilder AddAzureAppConfiguration( this IConfigurationBuilder configurationBuilder, diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 7d9a9cad..9391f21b 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -23,12 +23,13 @@ public class AzureAppConfigurationOptions private const int MaxRetries = 2; private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromMinutes(1); - private List _changeWatchers = new List(); - private List _multiKeyWatchers = new List(); + private List _individualKvWatchers = new List(); + private List _ffWatchers = new List(); private List _adapters; private List>> _mappers = new List>>(); - private List _kvSelectors = new List(); + private List _selectors; private IConfigurationRefresher _refresher = new AzureAppConfigurationRefresher(); + private bool _selectCalled = false; // The following set is sorted in descending order. // Since multiple prefixes could start with the same characters, we need to trim the longest prefix first. @@ -62,19 +63,29 @@ public class AzureAppConfigurationOptions internal TokenCredential Credential { get; private set; } /// - /// A collection of . + /// A collection of specified by user. /// - internal IEnumerable KeyValueSelectors => _kvSelectors; + internal IEnumerable Selectors => _selectors; + + /// + /// Indicates if was called. + /// + internal bool RegisterAllEnabled { get; private set; } + + /// + /// Refresh interval for selected key-value collections when is called. + /// + internal TimeSpan KvCollectionRefreshInterval { get; private set; } /// /// A collection of . /// - internal IEnumerable ChangeWatchers => _changeWatchers; + internal IEnumerable IndividualKvWatchers => _individualKvWatchers; /// /// A collection of . /// - internal IEnumerable MultiKeyWatchers => _multiKeyWatchers; + internal IEnumerable FeatureFlagWatchers => _ffWatchers; /// /// A collection of . @@ -96,11 +107,15 @@ internal IEnumerable Adapters internal IEnumerable KeyPrefixes => _keyPrefixes; /// - /// An optional configuration client manager that can be used to provide clients to communicate with Azure App Configuration. + /// For use in tests only. An optional configuration client manager that can be used to provide clients to communicate with Azure App Configuration. /// - /// This property is used only for unit testing. internal IConfigurationClientManager ClientManager { get; set; } + /// + /// For use in tests only. An optional class used to process pageable results from Azure App Configuration. + /// + internal IConfigurationSettingPageIterator ConfigurationSettingPageIterator { get; set; } + /// /// An optional timespan value to set the minimum backoff duration to a value other than the default. /// @@ -142,6 +157,9 @@ public AzureAppConfigurationOptions() new JsonKeyValueAdapter(), new FeatureManagementKeyValueAdapter(FeatureFlagTracing) }; + + // Adds the default query to App Configuration if and are never called. + _selectors = new List { new KeyValueSelector { KeyFilter = KeyFilter.Any, LabelFilter = LabelFilter.Null } }; } /// @@ -170,22 +188,30 @@ public AzureAppConfigurationOptions Select(string keyFilter, string labelFilter throw new ArgumentNullException(nameof(keyFilter)); } + // Do not support * and , for label filter for now. + if (labelFilter != null && (labelFilter.Contains('*') || labelFilter.Contains(','))) + { + throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); + } + if (string.IsNullOrWhiteSpace(labelFilter)) { labelFilter = LabelFilter.Null; } - // Do not support * and , for label filter for now. - if (labelFilter.Contains('*') || labelFilter.Contains(',')) + if (!_selectCalled) { - throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); + _selectors.Clear(); + + _selectCalled = true; } - _kvSelectors.AppendUnique(new KeyValueSelector + _selectors.AppendUnique(new KeyValueSelector { KeyFilter = keyFilter, LabelFilter = labelFilter }); + return this; } @@ -201,7 +227,14 @@ public AzureAppConfigurationOptions SelectSnapshot(string name) throw new ArgumentNullException(nameof(name)); } - _kvSelectors.AppendUnique(new KeyValueSelector + if (!_selectCalled) + { + _selectors.Clear(); + + _selectCalled = true; + } + + _selectors.AppendUnique(new KeyValueSelector { SnapshotName = name }); @@ -212,7 +245,7 @@ public AzureAppConfigurationOptions SelectSnapshot(string name) /// /// Configures options for Azure App Configuration feature flags that will be parsed and transformed into feature management configuration. /// If no filtering is specified via the then all feature flags with no label are loaded. - /// All loaded feature flags will be automatically registered for refresh on an individual flag level. + /// All loaded feature flags will be automatically registered for refresh as a collection. /// /// A callback used to configure feature flag options. public AzureAppConfigurationOptions UseFeatureFlags(Action configure = null) @@ -237,25 +270,22 @@ public AzureAppConfigurationOptions UseFeatureFlags(Action c options.FeatureFlagSelectors.Add(new KeyValueSelector { KeyFilter = FeatureManagementConstants.FeatureFlagMarker + "*", - LabelFilter = options.Label == null ? LabelFilter.Null : options.Label + LabelFilter = string.IsNullOrWhiteSpace(options.Label) ? LabelFilter.Null : options.Label, + IsFeatureFlagSelector = true }); } - foreach (var featureFlagSelector in options.FeatureFlagSelectors) + foreach (KeyValueSelector featureFlagSelector in options.FeatureFlagSelectors) { - var featureFlagFilter = featureFlagSelector.KeyFilter; - var labelFilter = featureFlagSelector.LabelFilter; + _selectors.AppendUnique(featureFlagSelector); - Select(featureFlagFilter, labelFilter); - - _multiKeyWatchers.AppendUnique(new KeyValueWatcher + _ffWatchers.AppendUnique(new KeyValueWatcher { - Key = featureFlagFilter, - Label = labelFilter, + Key = featureFlagSelector.KeyFilter, + Label = featureFlagSelector.LabelFilter, // If UseFeatureFlags is called multiple times for the same key and label filters, last refresh interval wins RefreshInterval = options.RefreshInterval }); - } return this; @@ -376,18 +406,41 @@ public AzureAppConfigurationOptions ConfigureClientOptions(ActionA callback used to configure Azure App Configuration refresh options. public AzureAppConfigurationOptions ConfigureRefresh(Action configure) { + if (RegisterAllEnabled) + { + throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() cannot be invoked multiple times when {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} has been invoked."); + } + var refreshOptions = new AzureAppConfigurationRefreshOptions(); configure?.Invoke(refreshOptions); - if (!refreshOptions.RefreshRegistrations.Any()) + bool isRegisterCalled = refreshOptions.RefreshRegistrations.Any(); + RegisterAllEnabled = refreshOptions.RegisterAllEnabled; + + if (!isRegisterCalled && !RegisterAllEnabled) + { + throw new InvalidOperationException($"{nameof(ConfigureRefresh)}() must call either {nameof(AzureAppConfigurationRefreshOptions.Register)}()" + + $" or {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)}()"); + } + + // Check if both register methods are called at any point + if (RegisterAllEnabled && (_individualKvWatchers.Any() || isRegisterCalled)) { - throw new ArgumentException($"{nameof(ConfigureRefresh)}() must have at least one key-value registered for refresh."); + throw new InvalidOperationException($"Cannot call both {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} and " + + $"{nameof(AzureAppConfigurationRefreshOptions.Register)}."); } - foreach (var item in refreshOptions.RefreshRegistrations) + if (RegisterAllEnabled) + { + KvCollectionRefreshInterval = refreshOptions.RefreshInterval; + } + else { - item.RefreshInterval = refreshOptions.RefreshInterval; - _changeWatchers.Add(item); + foreach (KeyValueWatcher item in refreshOptions.RefreshRegistrations) + { + item.RefreshInterval = refreshOptions.RefreshInterval; + _individualKvWatchers.Add(item); + } } return this; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index b5dd42f1..b3756309 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -4,7 +4,6 @@ using Azure; using Azure.Data.AppConfiguration; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; using Microsoft.Extensions.Logging; using System; @@ -32,9 +31,13 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private Uri _lastSuccessfulEndpoint; private AzureAppConfigurationOptions _options; private Dictionary _mappedData; - private Dictionary _watchedSettings = new Dictionary(); + private Dictionary _watchedIndividualKvs = new Dictionary(); + private HashSet _ffKeys = new HashSet(); + private Dictionary> _kvEtags = new Dictionary>(); + private Dictionary> _ffEtags = new Dictionary>(); private RequestTracingOptions _requestTracingOptions; private Dictionary _configClientBackoffs = new Dictionary(); + private DateTimeOffset _nextCollectionRefreshTime; private readonly TimeSpan MinRefreshInterval; @@ -108,11 +111,18 @@ public AzureAppConfigurationProvider(IConfigurationClientManager configClientMan _options = options ?? throw new ArgumentNullException(nameof(options)); _optional = optional; - IEnumerable watchers = options.ChangeWatchers.Union(options.MultiKeyWatchers); + IEnumerable watchers = options.IndividualKvWatchers.Union(options.FeatureFlagWatchers); - if (watchers.Any()) + bool hasWatchers = watchers.Any(); + TimeSpan minWatcherRefreshInterval = hasWatchers ? watchers.Min(w => w.RefreshInterval) : TimeSpan.MaxValue; + + if (options.RegisterAllEnabled) + { + MinRefreshInterval = TimeSpan.FromTicks(Math.Min(minWatcherRefreshInterval.Ticks, options.KvCollectionRefreshInterval.Ticks)); + } + else if (hasWatchers) { - MinRefreshInterval = watchers.Min(w => w.RefreshInterval); + MinRefreshInterval = minWatcherRefreshInterval; } else { @@ -194,13 +204,15 @@ public async Task RefreshAsync(CancellationToken cancellationToken) EnsureAssemblyInspected(); var utcNow = DateTimeOffset.UtcNow; - IEnumerable refreshableWatchers = _options.ChangeWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); - IEnumerable refreshableMultiKeyWatchers = _options.MultiKeyWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); + IEnumerable refreshableIndividualKvWatchers = _options.IndividualKvWatchers.Where(kvWatcher => utcNow >= kvWatcher.NextRefreshTime); + IEnumerable refreshableFfWatchers = _options.FeatureFlagWatchers.Where(ffWatcher => utcNow >= ffWatcher.NextRefreshTime); + bool isRefreshDue = utcNow >= _nextCollectionRefreshTime; // Skip refresh if mappedData is loaded, but none of the watchers or adapters are refreshable. if (_mappedData != null && - !refreshableWatchers.Any() && - !refreshableMultiKeyWatchers.Any() && + !refreshableIndividualKvWatchers.Any() && + !refreshableFfWatchers.Any() && + !isRefreshDue && !_options.Adapters.Any(adapter => adapter.NeedsRefresh())) { return; @@ -208,6 +220,11 @@ public async Task RefreshAsync(CancellationToken cancellationToken) IEnumerable clients = _configClientManager.GetClients(); + if (_requestTracingOptions != null) + { + _requestTracingOptions.ReplicaCount = clients.Count() - 1; + } + // // Filter clients based on their backoff status clients = clients.Where(client => @@ -249,179 +266,166 @@ public async Task RefreshAsync(CancellationToken cancellationToken) // // Avoid instance state modification - Dictionary watchedSettings = null; + Dictionary> kvEtags = null; + Dictionary> ffEtags = null; + HashSet ffKeys = null; + Dictionary watchedIndividualKvs = null; List keyValueChanges = null; - List changedKeyValuesCollection = null; Dictionary data = null; + Dictionary ffCollectionData = null; + bool ffCollectionUpdated = false; bool refreshAll = false; StringBuilder logInfoBuilder = new StringBuilder(); StringBuilder logDebugBuilder = new StringBuilder(); await ExecuteWithFailOverPolicyAsync(clients, async (client) => - { - data = null; - watchedSettings = null; - keyValueChanges = new List(); - changedKeyValuesCollection = null; - refreshAll = false; - Uri endpoint = _configClientManager.GetEndpointForClient(client); - logDebugBuilder.Clear(); - logInfoBuilder.Clear(); - - foreach (KeyValueWatcher changeWatcher in refreshableWatchers) - { - string watchedKey = changeWatcher.Key; - string watchedLabel = changeWatcher.Label; - - KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); - - KeyValueChange change = default; - - // - // Find if there is a change associated with watcher - if (_watchedSettings.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) - { - await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, - async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); - } - else - { - // Load the key-value in case the previous load attempts had failed - - try - { - await CallWithRequestTracing( - async () => watchedKv = await client.GetConfigurationSettingAsync(watchedKey, watchedLabel, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); - } - catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound) - { - watchedKv = null; - } - - if (watchedKv != null) - { - change = new KeyValueChange() - { - Key = watchedKv.Key, - Label = watchedKv.Label.NormalizeNull(), - Current = watchedKv, - ChangeType = KeyValueChangeType.Modified - }; - } - } - - // Check if a change has been detected in the key-value registered for refresh - if (change.ChangeType != KeyValueChangeType.None) - { - logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildKeyValueSettingUpdatedMessage(change.Key)); - keyValueChanges.Add(change); - - if (changeWatcher.RefreshAll) - { - refreshAll = true; - break; - } - } - else - { - logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); - } - } + { + kvEtags = null; + ffEtags = null; + ffKeys = null; + watchedIndividualKvs = null; + keyValueChanges = new List(); + data = null; + ffCollectionData = null; + ffCollectionUpdated = false; + refreshAll = false; + logDebugBuilder.Clear(); + logInfoBuilder.Clear(); + Uri endpoint = _configClientManager.GetEndpointForClient(client); - if (refreshAll) + if (_options.RegisterAllEnabled) + { + // Get key value collection changes if RegisterAll was called + if (isRefreshDue) { - // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true - data = await LoadSelectedKeyValues(client, cancellationToken).ConfigureAwait(false); - watchedSettings = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); - watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings, data); - logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); - return; + refreshAll = await HaveCollectionsChanged( + _options.Selectors.Where(selector => !selector.IsFeatureFlagSelector), + _kvEtags, + client, + cancellationToken).ConfigureAwait(false); } + } + else + { + refreshAll = await RefreshIndividualKvWatchers( + client, + keyValueChanges, + refreshableIndividualKvWatchers, + endpoint, + logDebugBuilder, + logInfoBuilder, + cancellationToken).ConfigureAwait(false); + } - changedKeyValuesCollection = await GetRefreshedKeyValueCollections(refreshableMultiKeyWatchers, client, logDebugBuilder, logInfoBuilder, endpoint, cancellationToken).ConfigureAwait(false); + if (refreshAll) + { + // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true, + // or if any key-value collection change was detected. + kvEtags = new Dictionary>(); + ffEtags = new Dictionary>(); + ffKeys = new HashSet(); + + data = await LoadSelected(client, kvEtags, ffEtags, _options.Selectors, ffKeys, cancellationToken).ConfigureAwait(false); + watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); + logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); + return; + } - if (!changedKeyValuesCollection.Any()) + // Get feature flag changes + ffCollectionUpdated = await HaveCollectionsChanged( + refreshableFfWatchers.Select(watcher => new KeyValueSelector { - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); - } - }, - cancellationToken) - .ConfigureAwait(false); + KeyFilter = watcher.Key, + LabelFilter = watcher.Label, + IsFeatureFlagSelector = true + }), + _ffEtags, + client, + cancellationToken).ConfigureAwait(false); + + if (ffCollectionUpdated) + { + ffEtags = new Dictionary>(); + ffKeys = new HashSet(); + + ffCollectionData = await LoadSelected( + client, + new Dictionary>(), + ffEtags, + _options.Selectors.Where(selector => selector.IsFeatureFlagSelector), + ffKeys, + cancellationToken).ConfigureAwait(false); + + logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); + } + else + { + logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); + } + }, + cancellationToken) + .ConfigureAwait(false); - if (!refreshAll) + if (refreshAll) { - watchedSettings = new Dictionary(_watchedSettings); + _mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); - foreach (KeyValueWatcher changeWatcher in refreshableWatchers.Concat(refreshableMultiKeyWatchers)) + // Invalidate all the cached KeyVault secrets + foreach (IKeyValueAdapter adapter in _options.Adapters) { - UpdateNextRefreshTime(changeWatcher); + adapter.OnChangeDetected(); } - foreach (KeyValueChange change in keyValueChanges.Concat(changedKeyValuesCollection)) + // Update the next refresh time for all refresh registered settings and feature flags + foreach (KeyValueWatcher changeWatcher in _options.IndividualKvWatchers.Concat(_options.FeatureFlagWatchers)) { - KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); - if (change.ChangeType == KeyValueChangeType.Modified) - { - ConfigurationSetting setting = change.Current; - ConfigurationSetting settingCopy = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); - watchedSettings[changeIdentifier] = settingCopy; + UpdateNextRefreshTime(changeWatcher); + } + } + else + { + watchedIndividualKvs = new Dictionary(_watchedIndividualKvs); - foreach (Func> func in _options.Mappers) - { - setting = await func(setting).ConfigureAwait(false); - } + await ProcessKeyValueChangesAsync(keyValueChanges, _mappedData, watchedIndividualKvs).ConfigureAwait(false); - if (setting == null) - { - _mappedData.Remove(change.Key); - } - else - { - _mappedData[change.Key] = setting; - } - } - else if (change.ChangeType == KeyValueChangeType.Deleted) + if (ffCollectionUpdated) + { + // Remove all feature flag keys that are not present in the latest loading of feature flags, but were loaded previously + foreach (string key in _ffKeys.Except(ffKeys)) { - _mappedData.Remove(change.Key); - watchedSettings.Remove(changeIdentifier); + _mappedData.Remove(key); } - // Invalidate the cached Key Vault secret (if any) for this ConfigurationSetting - foreach (IKeyValueAdapter adapter in _options.Adapters) + Dictionary mappedFfData = await MapConfigurationSettings(ffCollectionData).ConfigureAwait(false); + + foreach (KeyValuePair kvp in mappedFfData) { - // If the current setting is null, try to pass the previous setting instead - if (change.Current != null) - { - adapter.OnChangeDetected(change.Current); - } - else if (change.Previous != null) - { - adapter.OnChangeDetected(change.Previous); - } + _mappedData[kvp.Key] = kvp.Value; } } - } - else - { - _mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); - - // Invalidate all the cached KeyVault secrets - foreach (IKeyValueAdapter adapter in _options.Adapters) - { - adapter.OnChangeDetected(); - } - // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.MultiKeyWatchers)) + // + // update the next refresh time for all refresh registered settings and feature flags + foreach (KeyValueWatcher changeWatcher in refreshableIndividualKvWatchers.Concat(refreshableFfWatchers)) { UpdateNextRefreshTime(changeWatcher); } } - if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || changedKeyValuesCollection?.Any() == true || keyValueChanges.Any()) + if (isRefreshDue) { - _watchedSettings = watchedSettings; + _nextCollectionRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); + } + + if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any() || refreshAll || ffCollectionUpdated) + { + _watchedIndividualKvs = watchedIndividualKvs ?? _watchedIndividualKvs; + + _ffEtags = ffEtags ?? _ffEtags; + + _kvEtags = kvEtags ?? _kvEtags; + + _ffKeys = ffKeys ?? _ffKeys; if (logDebugBuilder.Length > 0) { @@ -432,6 +436,7 @@ await CallWithRequestTracing( { _logger.LogInformation(logInfoBuilder.ToString().Trim()); } + // PrepareData makes calls to KeyVault and may throw exceptions. But, we still update watchers before // SetData because repeating appconfig calls (by not updating watchers) won't help anything for keyvault calls. // As long as adapter.NeedsRefresh is true, we will attempt to update keyvault again the next time RefreshAsync is called. @@ -555,14 +560,21 @@ private void SetDirty(TimeSpan? maxDelay) { DateTimeOffset nextRefreshTime = AddRandomDelay(DateTimeOffset.UtcNow, maxDelay ?? DefaultMaxSetDirtyDelay); - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers) + if (_options.RegisterAllEnabled) { - changeWatcher.NextRefreshTime = nextRefreshTime; + _nextCollectionRefreshTime = nextRefreshTime; + } + else + { + foreach (KeyValueWatcher kvWatcher in _options.IndividualKvWatchers) + { + kvWatcher.NextRefreshTime = nextRefreshTime; + } } - foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) + foreach (KeyValueWatcher featureFlagWatcher in _options.FeatureFlagWatchers) { - changeWatcher.NextRefreshTime = nextRefreshTime; + featureFlagWatcher.NextRefreshTime = nextRefreshTime; } } @@ -612,6 +624,11 @@ private async Task LoadAsync(bool ignoreFailures, CancellationToken cancellation { IEnumerable clients = _configClientManager.GetClients(); + if (_requestTracingOptions != null) + { + _requestTracingOptions.ReplicaCount = clients.Count() - 1; + } + if (await TryInitializeAsync(clients, startupExceptions, cancellationToken).ConfigureAwait(false)) { break; @@ -707,34 +724,44 @@ private async Task TryInitializeAsync(IEnumerable cli private async Task InitializeAsync(IEnumerable clients, CancellationToken cancellationToken = default) { Dictionary data = null; - Dictionary watchedSettings = null; + Dictionary> kvEtags = new Dictionary>(); + Dictionary> ffEtags = new Dictionary>(); + Dictionary watchedIndividualKvs = null; + HashSet ffKeys = new HashSet(); await ExecuteWithFailOverPolicyAsync( clients, async (client) => { - data = await LoadSelectedKeyValues( + data = await LoadSelected( client, + kvEtags, + ffEtags, + _options.Selectors, + ffKeys, cancellationToken) .ConfigureAwait(false); - watchedSettings = await LoadKeyValuesRegisteredForRefresh( + watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh( client, data, cancellationToken) .ConfigureAwait(false); - - watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings, data); }, cancellationToken) .ConfigureAwait(false); // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.MultiKeyWatchers)) + foreach (KeyValueWatcher changeWatcher in _options.IndividualKvWatchers.Concat(_options.FeatureFlagWatchers)) { UpdateNextRefreshTime(changeWatcher); } + if (_options.RegisterAllEnabled) + { + _nextCollectionRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); + } + if (data != null) { // Invalidate all the cached KeyVault secrets @@ -744,51 +771,71 @@ await ExecuteWithFailOverPolicyAsync( } Dictionary mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); + SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); - _watchedSettings = watchedSettings; + _mappedData = mappedData; + _kvEtags = kvEtags; + _ffEtags = ffEtags; + _watchedIndividualKvs = watchedIndividualKvs; + _ffKeys = ffKeys; } } - private async Task> LoadSelectedKeyValues(ConfigurationClient client, CancellationToken cancellationToken) + private async Task> LoadSelected( + ConfigurationClient client, + Dictionary> kvEtags, + Dictionary> ffEtags, + IEnumerable selectors, + HashSet ffKeys, + CancellationToken cancellationToken) { - var serverData = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary data = new Dictionary(); - // Use default query if there are no key-values specified for use other than the feature flags - bool useDefaultQuery = !_options.KeyValueSelectors.Any(selector => selector.KeyFilter == null || - !selector.KeyFilter.StartsWith(FeatureManagementConstants.FeatureFlagMarker)); - - if (useDefaultQuery) + foreach (KeyValueSelector loadOption in selectors) { - // Load all key-values with the null label. - var selector = new SettingSelector - { - KeyFilter = KeyFilter.Any, - LabelFilter = LabelFilter.Null - }; - - await CallWithRequestTracing(async () => + if (string.IsNullOrEmpty(loadOption.SnapshotName)) { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) + var selector = new SettingSelector() { - serverData[setting.Key] = setting; - } - }).ConfigureAwait(false); - } + KeyFilter = loadOption.KeyFilter, + LabelFilter = loadOption.LabelFilter + }; - foreach (KeyValueSelector loadOption in _options.KeyValueSelectors) - { - IAsyncEnumerable settingsEnumerable; + var matchConditions = new List(); - if (string.IsNullOrEmpty(loadOption.SnapshotName)) - { - settingsEnumerable = client.GetConfigurationSettingsAsync( - new SettingSelector + await CallWithRequestTracing(async () => + { + AsyncPageable pageableSettings = client.GetConfigurationSettingsAsync(selector, cancellationToken); + + await foreach (Page page in pageableSettings.AsPages(_options.ConfigurationSettingPageIterator).ConfigureAwait(false)) { - KeyFilter = loadOption.KeyFilter, - LabelFilter = loadOption.LabelFilter - }, - cancellationToken); + using Response response = page.GetRawResponse(); + + ETag serverEtag = (ETag)response.Headers.ETag; + + foreach (ConfigurationSetting setting in page.Values) + { + data[setting.Key] = setting; + + if (loadOption.IsFeatureFlagSelector) + { + ffKeys.Add(setting.Key); + } + } + + matchConditions.Add(new MatchConditions { IfNoneMatch = serverEtag }); + } + }).ConfigureAwait(false); + + if (loadOption.IsFeatureFlagSelector) + { + ffEtags[loadOption] = matchConditions; + } + else + { + kvEtags[loadOption] = matchConditions; + } } else { @@ -808,38 +855,42 @@ await CallWithRequestTracing(async () => throw new InvalidOperationException($"{nameof(snapshot.SnapshotComposition)} for the selected snapshot with name '{snapshot.Name}' must be 'key', found '{snapshot.SnapshotComposition}'."); } - settingsEnumerable = client.GetConfigurationSettingsForSnapshotAsync( + IAsyncEnumerable settingsEnumerable = client.GetConfigurationSettingsForSnapshotAsync( loadOption.SnapshotName, cancellationToken); - } - await CallWithRequestTracing(async () => - { - await foreach (ConfigurationSetting setting in settingsEnumerable.ConfigureAwait(false)) + await CallWithRequestTracing(async () => { - serverData[setting.Key] = setting; - } - }).ConfigureAwait(false); + await foreach (ConfigurationSetting setting in settingsEnumerable.ConfigureAwait(false)) + { + data[setting.Key] = setting; + } + }).ConfigureAwait(false); + } } - return serverData; + return data; } - private async Task> LoadKeyValuesRegisteredForRefresh(ConfigurationClient client, IDictionary existingSettings, CancellationToken cancellationToken) + private async Task> LoadKeyValuesRegisteredForRefresh( + ConfigurationClient client, + IDictionary existingSettings, + CancellationToken cancellationToken) { - Dictionary watchedSettings = new Dictionary(); + var watchedIndividualKvs = new Dictionary(); - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers) + foreach (KeyValueWatcher kvWatcher in _options.IndividualKvWatchers) { - string watchedKey = changeWatcher.Key; - string watchedLabel = changeWatcher.Label; + string watchedKey = kvWatcher.Key; + string watchedLabel = kvWatcher.Label; + KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); // Skip the loading for the key-value in case it has already been loaded if (existingSettings.TryGetValue(watchedKey, out ConfigurationSetting loadedKv) && watchedKeyLabel.Equals(new KeyValueIdentifier(loadedKv.Key, loadedKv.Label))) { - watchedSettings[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); + watchedIndividualKvs[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); continue; } @@ -857,61 +908,84 @@ private async Task> LoadKey // If the key-value was found, store it for updating the settings if (watchedKv != null) { - watchedSettings[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); + watchedIndividualKvs[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); existingSettings[watchedKey] = watchedKv; } } - return watchedSettings; + return watchedIndividualKvs; } - private Dictionary UpdateWatchedKeyValueCollections(Dictionary watchedSettings, IDictionary existingSettings) + private async Task RefreshIndividualKvWatchers( + ConfigurationClient client, + List keyValueChanges, + IEnumerable refreshableIndividualKvWatchers, + Uri endpoint, + StringBuilder logDebugBuilder, + StringBuilder logInfoBuilder, + CancellationToken cancellationToken) { - foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) + foreach (KeyValueWatcher kvWatcher in refreshableIndividualKvWatchers) { - IEnumerable currentKeyValues = GetCurrentKeyValueCollection(changeWatcher.Key, changeWatcher.Label, existingSettings.Values); + string watchedKey = kvWatcher.Key; + string watchedLabel = kvWatcher.Label; - foreach (ConfigurationSetting setting in currentKeyValues) + KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); + + KeyValueChange change = default; + + // + // Find if there is a change associated with watcher + if (_watchedIndividualKvs.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) { - watchedSettings[new KeyValueIdentifier(setting.Key, setting.Label)] = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, + async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); } - } + else + { + // Load the key-value in case the previous load attempts had failed - return watchedSettings; - } + try + { + await CallWithRequestTracing( + async () => watchedKv = await client.GetConfigurationSettingAsync(watchedKey, watchedLabel, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + } + catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound) + { + watchedKv = null; + } - private async Task> GetRefreshedKeyValueCollections( - IEnumerable multiKeyWatchers, - ConfigurationClient client, - StringBuilder logDebugBuilder, - StringBuilder logInfoBuilder, - Uri endpoint, - CancellationToken cancellationToken) - { - var keyValueChanges = new List(); + if (watchedKv != null) + { + change = new KeyValueChange() + { + Key = watchedKv.Key, + Label = watchedKv.Label.NormalizeNull(), + Current = watchedKv, + ChangeType = KeyValueChangeType.Modified + }; + } + } - foreach (KeyValueWatcher changeWatcher in multiKeyWatchers) - { - IEnumerable currentKeyValues = GetCurrentKeyValueCollection(changeWatcher.Key, changeWatcher.Label, _watchedSettings.Values); + // Check if a change has been detected in the key-value registered for refresh + if (change.ChangeType != KeyValueChangeType.None) + { + logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); + logInfoBuilder.AppendLine(LogHelper.BuildKeyValueSettingUpdatedMessage(change.Key)); + keyValueChanges.Add(change); - keyValueChanges.AddRange( - await client.GetKeyValueChangeCollection( - currentKeyValues, - new GetKeyValueChangeCollectionOptions - { - KeyFilter = changeWatcher.Key, - Label = changeWatcher.Label.NormalizeNull(), - RequestTracingEnabled = _requestTracingEnabled, - RequestTracingOptions = _requestTracingOptions - }, - logDebugBuilder, - logInfoBuilder, - endpoint, - cancellationToken) - .ConfigureAwait(false)); + if (kvWatcher.RefreshAll) + { + return true; + } + } + else + { + logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); + } } - return keyValueChanges; + return false; } private void SetData(IDictionary data) @@ -966,7 +1040,6 @@ private void SetRequestTracingOptions() IsDevEnvironment = TracingUtils.IsDevEnvironment(), IsKeyVaultConfigured = _options.IsKeyVaultConfigured, IsKeyVaultRefreshConfigured = _options.IsKeyVaultRefreshConfigured, - ReplicaCount = _options.Endpoints?.Count() - 1 ?? _options.ConnectionStrings?.Count() - 1 ?? 0, FeatureFlagTracing = _options.FeatureFlagTracing, IsLoadBalancingEnabled = _options.LoadBalancingEnabled }; @@ -1179,30 +1252,6 @@ private async Task> MapConfigurationSet return mappedData; } - private IEnumerable GetCurrentKeyValueCollection(string key, string label, IEnumerable existingSettings) - { - IEnumerable currentKeyValues; - - if (key.EndsWith("*")) - { - // Get current application settings starting with changeWatcher.Key, excluding the last * character - string keyPrefix = key.Substring(0, key.Length - 1); - currentKeyValues = existingSettings.Where(kv => - { - return kv.Key.StartsWith(keyPrefix) && kv.Label == label.NormalizeNull(); - }); - } - else - { - currentKeyValues = existingSettings.Where(kv => - { - return kv.Key.Equals(key) && kv.Label == label.NormalizeNull(); - }); - } - - return currentKeyValues; - } - private void EnsureAssemblyInspected() { if (!_isAssemblyInspected) @@ -1248,6 +1297,88 @@ private void UpdateClientBackoffStatus(Uri endpoint, bool successful) _configClientBackoffs[endpoint] = clientBackoffStatus; } + private async Task HaveCollectionsChanged( + IEnumerable selectors, + Dictionary> pageEtags, + ConfigurationClient client, + CancellationToken cancellationToken) + { + bool haveCollectionsChanged = false; + + foreach (KeyValueSelector selector in selectors) + { + if (pageEtags.TryGetValue(selector, out IEnumerable matchConditions)) + { + await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, + async () => haveCollectionsChanged = await client.HaveCollectionsChanged( + selector, + matchConditions, + _options.ConfigurationSettingPageIterator, + cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + } + + if (haveCollectionsChanged) + { + return true; + } + } + + return haveCollectionsChanged; + } + + private async Task ProcessKeyValueChangesAsync( + IEnumerable keyValueChanges, + Dictionary mappedData, + Dictionary watchedIndividualKvs) + { + foreach (KeyValueChange change in keyValueChanges) + { + KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); + + if (change.ChangeType == KeyValueChangeType.Modified) + { + ConfigurationSetting setting = change.Current; + ConfigurationSetting settingCopy = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + watchedIndividualKvs[changeIdentifier] = settingCopy; + + foreach (Func> func in _options.Mappers) + { + setting = await func(setting).ConfigureAwait(false); + } + + if (setting == null) + { + mappedData.Remove(change.Key); + } + else + { + mappedData[change.Key] = setting; + } + } + else if (change.ChangeType == KeyValueChangeType.Deleted) + { + mappedData.Remove(change.Key); + + watchedIndividualKvs.Remove(changeIdentifier); + } + + // Invalidate the cached Key Vault secret (if any) for this ConfigurationSetting + foreach (IKeyValueAdapter adapter in _options.Adapters) + { + // If the current setting is null, try to pass the previous setting instead + if (change.Current != null) + { + adapter.OnChangeDetected(change.Current); + } + else if (change.Previous != null) + { + adapter.OnChangeDetected(change.Previous); + } + } + } + } + public void Dispose() { (_configClientManager as ConfigurationClientManager)?.Dispose(); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs index f3fb6c4a..cf2847a8 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs @@ -14,6 +14,7 @@ public class AzureAppConfigurationRefreshOptions { internal TimeSpan RefreshInterval { get; private set; } = RefreshConstants.DefaultRefreshInterval; internal ISet RefreshRegistrations = new HashSet(); + internal bool RegisterAllEnabled { get; private set; } /// /// Register the specified individual key-value to be refreshed when the configuration provider's triggers a refresh. @@ -50,6 +51,17 @@ public AzureAppConfigurationRefreshOptions Register(string key, string label = L return this; } + /// + /// Register all key-values loaded outside of to be refreshed when the configuration provider's triggers a refresh. + /// The instance can be obtained by calling . + /// + public AzureAppConfigurationRefreshOptions RegisterAll() + { + RegisterAllEnabled = true; + + return this; + } + /// /// Sets the cache expiration time for the key-values registered for refresh. Default value is 30 seconds. Must be greater than 1 second. /// Any refresh operation triggered using will not update the value for a key until the cached value for that key has expired. diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs new file mode 100644 index 00000000..aba7684b --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs @@ -0,0 +1,33 @@ +using Azure.Data.AppConfiguration; +using Azure; +using System.Collections.Generic; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + static class ConfigurationSettingPageExtensions + { + public static IAsyncEnumerable> AsPages(this AsyncPageable pageable, IConfigurationSettingPageIterator pageIterator) + { + // + // Allow custom iteration + if (pageIterator != null) + { + return pageIterator.IteratePages(pageable); + } + + return pageable.AsPages(); + } + + public static IAsyncEnumerable> AsPages(this AsyncPageable pageable, IConfigurationSettingPageIterator pageIterator, IEnumerable matchConditions) + { + // + // Allow custom iteration + if (pageIterator != null) + { + return pageIterator.IteratePages(pageable, matchConditions); + } + + return pageable.AsPages(matchConditions); + } + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSourceExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSourceExtensions.cs new file mode 100644 index 00000000..af7f1f03 --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSourceExtensions.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + /// + /// Provides extension methods for configuration sources. + /// + public static class ConfigurationSourceExtensions + { + /// + /// Determines whether the specified configuration source is an Azure App Configuration source. + /// + /// The configuration source to check. + /// true if the specified source is an Azure App Configuration source; otherwise, false. + public static bool IsAzureAppConfigurationSource(this IConfigurationSource source) + { + return source is AzureAppConfigurationSource; + } + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs index 86576a48..3bdcaecd 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs @@ -20,14 +20,15 @@ internal class LoggingConstants // Successful update, debug log level public const string RefreshKeyValueRead = "Key-value read from App Configuration."; public const string RefreshKeyVaultSecretRead = "Secret read from Key Vault for key-value."; - public const string RefreshFeatureFlagRead = "Feature flag read from App Configuration."; public const string RefreshFeatureFlagsUnchanged = "Feature flags read from App Configuration. Change:'None'"; + public const string RefreshSelectedKeyValueCollectionsUnchanged = "Selected key-value collections read from App Configuration. Change:'None'"; // Successful update, information log level public const string RefreshConfigurationUpdatedSuccess = "Configuration reloaded."; public const string RefreshKeyValueSettingUpdated = "Setting updated."; public const string RefreshKeyVaultSettingUpdated = "Setting updated from Key Vault."; - public const string RefreshFeatureFlagUpdated = "Feature flag updated."; + public const string RefreshFeatureFlagsUpdated = "Feature flags reloaded."; + public const string RefreshSelectedKeyValuesAndFeatureFlagsUpdated = "Selected key-value collections and feature flags reloaded."; // Other public const string RefreshSkippedNoClientAvailable = "Refresh skipped because no endpoint is accessible."; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs index 15e862b6..1084c274 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/RequestTracingConstants.cs @@ -43,7 +43,6 @@ internal class RequestTracingConstants public const string FeatureFlagUsesTelemetryTag = "Telemetry"; public const string FeatureFlagUsesSeedTag = "Seed"; public const string FeatureFlagMaxVariantsKey = "MaxVariants"; - public const string FeatureFlagUsesVariantConfigurationReferenceTag = "ConfigRef"; public const string DiagnosticHeaderActivityName = "Azure.CustomDiagnosticHeaders"; public const string CorrelationContextHeader = "Correlation-Context"; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index d479ad6b..a50b4d45 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -3,12 +3,10 @@ // using Azure; using Azure.Data.AppConfiguration; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; using System; using System.Collections.Generic; -using System.Linq; using System.Net; -using System.Text; using System.Threading; using System.Threading.Tasks; @@ -31,7 +29,8 @@ public static async Task GetKeyValueChange(this ConfigurationCli try { Response response = await client.GetConfigurationSettingAsync(setting, onlyIfChanged: true, cancellationToken).ConfigureAwait(false); - if (response.GetRawResponse().Status == (int)HttpStatusCode.OK) + if (response.GetRawResponse().Status == (int)HttpStatusCode.OK && + !response.Value.ETag.Equals(setting.ETag)) { return new KeyValueChange { @@ -65,131 +64,50 @@ public static async Task GetKeyValueChange(this ConfigurationCli }; } - public static async Task> GetKeyValueChangeCollection( - this ConfigurationClient client, - IEnumerable keyValues, - GetKeyValueChangeCollectionOptions options, - StringBuilder logDebugBuilder, - StringBuilder logInfoBuilder, - Uri endpoint, - CancellationToken cancellationToken) + public static async Task HaveCollectionsChanged(this ConfigurationClient client, KeyValueSelector keyValueSelector, IEnumerable matchConditions, IConfigurationSettingPageIterator pageIterator, CancellationToken cancellationToken) { - if (options == null) + if (matchConditions == null) { - throw new ArgumentNullException(nameof(options)); + throw new ArgumentNullException(nameof(matchConditions)); } - if (keyValues == null) + if (keyValueSelector == null) { - keyValues = Enumerable.Empty(); + throw new ArgumentNullException(nameof(keyValueSelector)); } - if (options.KeyFilter == null) + if (keyValueSelector.SnapshotName != null) { - options.KeyFilter = string.Empty; + throw new ArgumentException("Cannot check snapshot for changes.", $"{nameof(keyValueSelector)}.{nameof(keyValueSelector.SnapshotName)}"); } - if (keyValues.Any(k => string.IsNullOrEmpty(k.Key))) + SettingSelector selector = new SettingSelector { - throw new ArgumentNullException($"{nameof(keyValues)}[].{nameof(ConfigurationSetting.Key)}"); - } - - if (keyValues.Any(k => !string.Equals(k.Label.NormalizeNull(), options.Label.NormalizeNull()))) - { - throw new ArgumentException("All key-values registered for refresh must use the same label.", $"{nameof(keyValues)}[].{nameof(ConfigurationSetting.Label)}"); - } - - if (keyValues.Any(k => k.Label != null && k.Label.Contains("*"))) - { - throw new ArgumentException("The label filter cannot contain '*'", $"{nameof(options)}.{nameof(options.Label)}"); - } - - var hasKeyValueCollectionChanged = false; - var selector = new SettingSelector - { - KeyFilter = options.KeyFilter, - LabelFilter = string.IsNullOrEmpty(options.Label) ? LabelFilter.Null : options.Label, - Fields = SettingFields.ETag | SettingFields.Key + KeyFilter = keyValueSelector.KeyFilter, + LabelFilter = keyValueSelector.LabelFilter }; - // Dictionary of eTags that we write to and use for comparison - var eTagMap = keyValues.ToDictionary(kv => kv.Key, kv => kv.ETag); + AsyncPageable pageable = client.GetConfigurationSettingsAsync(selector, cancellationToken); - // Fetch e-tags for prefixed key-values that can be used to detect changes - await TracingUtils.CallWithRequestTracing(options.RequestTracingEnabled, RequestType.Watch, options.RequestTracingOptions, - async () => - { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) - { - if (!eTagMap.TryGetValue(setting.Key, out ETag etag) || !etag.Equals(setting.ETag)) - { - hasKeyValueCollectionChanged = true; - break; - } + using IEnumerator existingMatchConditionsEnumerator = matchConditions.GetEnumerator(); - eTagMap.Remove(setting.Key); - } - }).ConfigureAwait(false); - - // Check for any deletions - if (eTagMap.Any()) - { - hasKeyValueCollectionChanged = true; - } - - var changes = new List(); - - // If changes have been observed, refresh prefixed key-values - if (hasKeyValueCollectionChanged) + await foreach (Page page in pageable.AsPages(pageIterator, matchConditions).ConfigureAwait(false)) { - selector = new SettingSelector - { - KeyFilter = options.KeyFilter, - LabelFilter = string.IsNullOrEmpty(options.Label) ? LabelFilter.Null : options.Label - }; + using Response response = page.GetRawResponse(); - eTagMap = keyValues.ToDictionary(kv => kv.Key, kv => kv.ETag); - await TracingUtils.CallWithRequestTracing(options.RequestTracingEnabled, RequestType.Watch, options.RequestTracingOptions, - async () => - { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) - { - if (!eTagMap.TryGetValue(setting.Key, out ETag etag) || !etag.Equals(setting.ETag)) - { - changes.Add(new KeyValueChange - { - ChangeType = KeyValueChangeType.Modified, - Key = setting.Key, - Label = options.Label.NormalizeNull(), - Previous = null, - Current = setting - }); - string key = setting.Key.Substring(FeatureManagementConstants.FeatureFlagMarker.Length); - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagReadMessage(key, options.Label.NormalizeNull(), endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildFeatureFlagUpdatedMessage(key)); - } + ETag serverEtag = (ETag)response.Headers.ETag; - eTagMap.Remove(setting.Key); - } - }).ConfigureAwait(false); - - foreach (var kvp in eTagMap) + // Return true if the lists of etags are different + if ((!existingMatchConditionsEnumerator.MoveNext() || + !existingMatchConditionsEnumerator.Current.IfNoneMatch.Equals(serverEtag)) && + response.Status == (int)HttpStatusCode.OK) { - changes.Add(new KeyValueChange - { - ChangeType = KeyValueChangeType.Deleted, - Key = kvp.Key, - Label = options.Label.NormalizeNull(), - Previous = null, - Current = null - }); - string key = kvp.Key.Substring(FeatureManagementConstants.FeatureFlagMarker.Length); - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagReadMessage(key, options.Label.NormalizeNull(), endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildFeatureFlagUpdatedMessage(key)); + return true; } } - return changes; + // Need to check if pages were deleted and no change was found within the new shorter list of match conditions + return existingMatchConditionsEnumerator.MoveNext(); } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs index 7bcf7212..e97dfa2f 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs @@ -3,18 +3,11 @@ // namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions { - internal static class LabelFilters - { - public static readonly string Null = "\0"; - - public static readonly string Any = "*"; - } - internal static class StringExtensions { public static string NormalizeNull(this string s) { - return s == LabelFilters.Null ? null : s; + return s == LabelFilter.Null ? null : s; } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs index 1e8beae6..26390762 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs @@ -102,7 +102,8 @@ public FeatureFlagOptions Select(string featureFlagFilter, string labelFilter = FeatureFlagSelectors.AppendUnique(new KeyValueSelector { KeyFilter = featureFlagPrefix, - LabelFilter = labelFilter + LabelFilter = labelFilter, + IsFeatureFlagSelector = true }); return this; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagTracing.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagTracing.cs index f48b5220..8c696e49 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagTracing.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagTracing.cs @@ -25,7 +25,6 @@ internal class FeatureFlagTracing public bool UsesTargetingFilter { get; set; } = false; public bool UsesSeed { get; set; } = false; public bool UsesTelemetry { get; set; } = false; - public bool UsesVariantConfigurationReference { get; set; } = false; public int MaxVariants { get; set; } public bool UsesAnyFeatureFilter() @@ -35,7 +34,7 @@ public bool UsesAnyFeatureFilter() public bool UsesAnyTracingFeature() { - return UsesSeed || UsesTelemetry || UsesVariantConfigurationReference; + return UsesSeed || UsesTelemetry; } public void ResetFeatureFlagTracing() @@ -46,7 +45,6 @@ public void ResetFeatureFlagTracing() UsesTargetingFilter = false; UsesSeed = false; UsesTelemetry = false; - UsesVariantConfigurationReference = false; MaxVariants = 0; } @@ -147,16 +145,6 @@ public string CreateFeaturesString() sb.Append(RequestTracingConstants.FeatureFlagUsesSeedTag); } - if (UsesVariantConfigurationReference) - { - if (sb.Length > 0) - { - sb.Append(RequestTracingConstants.Delimiter); - } - - sb.Append(RequestTracingConstants.FeatureFlagUsesVariantConfigurationReferenceTag); - } - if (UsesTelemetry) { if (sb.Length > 0) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs index c6d86d84..6d385797 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs @@ -30,7 +30,6 @@ internal class FeatureManagementConstants public const string Parameters = "parameters"; public const string Variant = "variant"; public const string ConfigurationValue = "configuration_value"; - public const string ConfigurationReference = "configuration_reference"; public const string StatusOverride = "status_override"; public const string DefaultWhenDisabled = "default_when_disabled"; public const string DefaultWhenEnabled = "default_when_enabled"; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs index 2b0e0833..b6d137f3 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs @@ -187,13 +187,6 @@ private List> ProcessMicrosoftSchemaFeatureFlag(Fea (string.IsNullOrEmpty(kvp.Key) ? "" : $":{kvp.Key}"), kvp.Value)); } - if (featureVariant.ConfigurationReference != null) - { - _featureFlagTracing.UsesVariantConfigurationReference = true; - - keyValues.Add(new KeyValuePair($"{variantsPath}:{FeatureManagementConstants.ConfigurationReference}", featureVariant.ConfigurationReference)); - } - if (featureVariant.StatusOverride != null) { keyValues.Add(new KeyValuePair($"{variantsPath}:{FeatureManagementConstants.StatusOverride}", featureVariant.StatusOverride)); @@ -1157,24 +1150,6 @@ private FeatureVariant ParseFeatureVariant(ref Utf8JsonReader reader, string set break; } - case FeatureManagementConstants.ConfigurationReference: - { - if (reader.Read() && reader.TokenType == JsonTokenType.String) - { - featureVariant.ConfigurationReference = reader.GetString(); - } - else if (reader.TokenType != JsonTokenType.Null) - { - throw CreateFeatureFlagFormatException( - FeatureManagementConstants.ConfigurationReference, - settingKey, - reader.TokenType.ToString(), - JsonTokenType.String.ToString()); - } - - break; - } - case FeatureManagementConstants.ConfigurationValue: { if (reader.Read()) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureVariant.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureVariant.cs index 87c5c0b1..7bfd9d6f 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureVariant.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureVariant.cs @@ -11,8 +11,6 @@ internal class FeatureVariant public JsonElement ConfigurationValue { get; set; } - public string ConfigurationReference { get; set; } - public string StatusOverride { get; set; } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs deleted file mode 100644 index 5cb9b83d..00000000 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. -// -namespace Microsoft.Extensions.Configuration.AzureAppConfiguration -{ - internal class GetKeyValueChangeCollectionOptions - { - public string KeyFilter { get; set; } - public string Label { get; set; } - public bool RequestTracingEnabled { get; set; } - public RequestTracingOptions RequestTracingOptions { get; set; } - } -} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs new file mode 100644 index 00000000..08c95751 --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs @@ -0,0 +1,13 @@ +using Azure.Data.AppConfiguration; +using Azure; +using System.Collections.Generic; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + internal interface IConfigurationSettingPageIterator + { + IAsyncEnumerable> IteratePages(AsyncPageable pageable); + + IAsyncEnumerable> IteratePages(AsyncPageable pageable, IEnumerable matchConditions); + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs index 11442dcb..4f999406 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs @@ -25,14 +25,19 @@ public static string BuildFeatureFlagsUnchangedMessage(string endpoint) return $"{LoggingConstants.RefreshFeatureFlagsUnchanged} Endpoint:'{endpoint?.TrimEnd('/')}'"; } - public static string BuildFeatureFlagReadMessage(string key, string label, string endpoint) - { - return $"{LoggingConstants.RefreshFeatureFlagRead} Key:'{key}' Label:'{label}' Endpoint:'{endpoint?.TrimEnd('/')}'"; + public static string BuildFeatureFlagsUpdatedMessage() + { + return LoggingConstants.RefreshFeatureFlagsUpdated; + } + + public static string BuildSelectedKeyValueCollectionsUnchangedMessage(string endpoint) + { + return $"{LoggingConstants.RefreshSelectedKeyValueCollectionsUnchanged} Endpoint:'{endpoint?.TrimEnd('/')}'"; } - public static string BuildFeatureFlagUpdatedMessage(string key) + public static string BuildSelectedKeyValuesAndFeatureFlagsUpdatedMessage() { - return $"{LoggingConstants.RefreshFeatureFlagUpdated} Key:'{key}'"; + return LoggingConstants.RefreshSelectedKeyValuesAndFeatureFlagsUpdated; } public static string BuildKeyVaultSecretReadMessage(string key, string label) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index 3997c788..9cac9155 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -35,7 +35,7 @@ - 8.0.0 + 8.1.0 diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs index 5491d04d..54bda1a4 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs @@ -24,6 +24,11 @@ public class KeyValueSelector /// public string SnapshotName { get; set; } + /// + /// A boolean that signifies whether this selector is intended to select feature flags. + /// + public bool IsFeatureFlagSelector { get; set; } + /// /// Determines whether the specified object is equal to the current object. /// diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs index 7b06535b..3838ee95 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/RequestTracingOptions.cs @@ -66,7 +66,7 @@ internal class RequestTracingOptions /// /// Checks whether any tracing feature is used. /// - /// True if any tracing feature is used, otherwise false. + /// true if any tracing feature is used, otherwise false. public bool UsesAnyTracingFeature() { return IsLoadBalancingEnabled || IsSignalRUsed; diff --git a/test.ps1 b/test.ps1 index 19c80dff..9acc5054 100644 --- a/test.ps1 +++ b/test.ps1 @@ -3,7 +3,17 @@ $ErrorActionPreference = "Stop" $dotnet = & "$PSScriptRoot/build/resolve-dotnet.ps1" & $dotnet test "$PSScriptRoot\tests\Tests.AzureAppConfiguration\Tests.AzureAppConfiguration.csproj" --logger trx + +if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE +} + & $dotnet test "$PSScriptRoot\tests\Tests.AzureAppConfiguration.AspNetCore\Tests.AzureAppConfiguration.AspNetCore.csproj" --logger trx + +if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE +} + & $dotnet test "$PSScriptRoot\tests\Tests.AzureAppConfiguration.Functions.Worker\Tests.AzureAppConfiguration.Functions.Worker.csproj" --logger trx exit $LASTEXITCODE \ No newline at end of file diff --git a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs index 886d0a77..aaee0b9c 100644 --- a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs +++ b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs @@ -17,6 +17,8 @@ public MockResponse(int status, string reasonPhrase = null) { Status = status; ReasonPhrase = reasonPhrase; + + AddHeader(new HttpHeader(HttpHeader.Names.ETag, "\"" + Guid.NewGuid().ToString() + "\"")); } public override int Status { get; } diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 44c20b4d..c374e2c3 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -435,7 +435,6 @@ public class FeatureManagementTests }, { ""name"": ""Small"", - ""configuration_reference"": ""ShoppingCart:Small"", ""status_override"": ""Disabled"" } ], @@ -660,19 +659,24 @@ public void UsesFeatureFlags() [Fact] public async Task WatchesFeatureFlags() { + var mockResponse = new MockResponse(200); + var featureFlags = new List { _kv }; - var mockResponse = new Mock(); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + var mockClient = new Mock(MockBehavior.Strict); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); @@ -712,7 +716,7 @@ public async Task WatchesFeatureFlags() ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); featureFlags.Add(_kv2); @@ -731,11 +735,12 @@ public async Task WatchesFeatureFlagsUsingCacheExpirationInterval() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); var cacheExpirationInterval = TimeSpan.FromSeconds(1); @@ -744,6 +749,7 @@ public async Task WatchesFeatureFlagsUsingCacheExpirationInterval() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.CacheExpirationInterval = cacheExpirationInterval); refresher = options.GetRefresher(); @@ -802,17 +808,20 @@ public async Task SkipRefreshIfRefreshIntervalHasNotElapsed() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(TimeSpan.FromSeconds(10))); refresher = options.GetRefresher(); @@ -869,17 +878,20 @@ public async Task SkipRefreshIfCacheNotExpired() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.CacheExpirationInterval = TimeSpan.FromSeconds(10)); refresher = options.GetRefresher(); @@ -988,17 +1000,22 @@ public void QueriesFeatureFlags() } [Fact] - public async Task UsesEtagForFeatureFlagRefresh() + public async Task DoesNotUseEtagForFeatureFlagRefresh() { + var mockAsyncPageable = new MockAsyncPageable(new List { _kv }); + var mockClient = new Mock(MockBehavior.Strict); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(new List { _kv })); + .Callback(() => mockAsyncPageable.UpdateCollection(new List { _kv })) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); @@ -1029,6 +1046,7 @@ public void SelectFeatureFlags() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(testClient); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(RefreshInterval); @@ -1378,18 +1396,19 @@ public async Task DifferentCacheExpirationsForMultipleFeatureFlagRegistrations() IConfigurationRefresher refresher = null; var featureFlagCollection = new List(_featureFlagCollection); + var mockAsyncPageable = new MockAsyncPageable(featureFlagCollection); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(() => - { - return new MockAsyncPageable(featureFlagCollection.Where(s => + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlagCollection.Where(s => (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1) || - (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix2) && s.Label == label2 && s.Key != FeatureManagementConstants.FeatureFlagMarker + "App2_Feature3")).ToList()); - }); + (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix2) && s.Label == label2 && s.Key != FeatureManagementConstants.FeatureFlagMarker + "App2_Feature3")).ToList())) + .Returns(mockAsyncPageable); var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(refreshInterval1); @@ -1548,18 +1567,18 @@ public async Task SelectAndRefreshSingleFeatureFlag() var label1 = "App1_Label"; IConfigurationRefresher refresher = null; var featureFlagCollection = new List(_featureFlagCollection); + var mockAsyncPageable = new MockAsyncPageable(featureFlagCollection); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(() => - { - return new MockAsyncPageable(featureFlagCollection.Where(s => - s.Key.Equals(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1).ToList()); - }); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlagCollection.Where(s => + s.Key.Equals(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1).ToList())) + .Returns(mockAsyncPageable); var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(RefreshInterval); @@ -1612,8 +1631,17 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); + + mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((Func>)GetIfChanged); + + mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((Func>)GetTestKey); string informationalInvocation = ""; string verboseInvocation = ""; @@ -1637,6 +1665,7 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre .AddAzureAppConfiguration(options => { options.ClientManager = mockClientManager; + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); }) @@ -1645,10 +1674,10 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre Assert.Equal("SuperUsers", config["FeatureManagement:MyFeature2:EnabledFor:0:Name"]); featureFlags[0] = ConfigurationModelFactory.ConfigurationSetting( - key: FeatureManagementConstants.FeatureFlagMarker + "myFeature1", + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature2", value: @" { - ""id"": ""MyFeature"", + ""id"": ""MyFeature2"", ""description"": ""The new beta version of our web site."", ""display_name"": ""Beta Feature"", ""enabled"": true, @@ -1663,21 +1692,19 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); - Assert.Equal("AllUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); - Assert.Contains(LogHelper.BuildFeatureFlagReadMessage("myFeature1", null, TestHelpers.PrimaryConfigStoreEndpoint.ToString().TrimEnd('/')), verboseInvocation); - Assert.Contains(LogHelper.BuildFeatureFlagUpdatedMessage("myFeature1"), informationalInvocation); + Assert.Equal("AllUsers", config["FeatureManagement:MyFeature2:EnabledFor:0:Name"]); + Assert.Contains(LogHelper.BuildFeatureFlagsUpdatedMessage(), informationalInvocation); featureFlags.RemoveAt(0); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); Assert.Null(config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); - Assert.Contains(LogHelper.BuildFeatureFlagReadMessage("myFeature1", null, TestHelpers.PrimaryConfigStoreEndpoint.ToString().TrimEnd('/')), verboseInvocation); - Assert.Contains(LogHelper.BuildFeatureFlagUpdatedMessage("myFeature1"), informationalInvocation); + Assert.Contains(LogHelper.BuildFeatureFlagsUpdatedMessage(), informationalInvocation); } [Fact] @@ -1688,8 +1715,11 @@ public async Task ValidateFeatureFlagsUnchangedLogged() var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((Func>)GetIfChanged); @@ -1713,6 +1743,7 @@ public async Task ValidateFeatureFlagsUnchangedLogged() .AddAzureAppConfiguration(options => { options.ClientManager = mockClientManager; + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); options.ConfigureRefresh(refreshOptions => { @@ -1724,7 +1755,7 @@ public async Task ValidateFeatureFlagsUnchangedLogged() .Build(); Assert.Equal("SuperUsers", config["FeatureManagement:MyFeature2:EnabledFor:0:Name"]); - FirstKeyValue.Value = "newValue1"; + FirstKeyValue = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -1762,9 +1793,11 @@ public async Task MapTransformFeatureFlagWithRefresh() IConfigurationRefresher refresher = null; var featureFlags = new List { _kv }; var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) + .Returns(mockAsyncPageable); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((Func>)GetIfChanged); @@ -1817,7 +1850,7 @@ public async Task MapTransformFeatureFlagWithRefresh() Assert.Equal("TestValue1", config["TestKey1"]); Assert.Equal("NoUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); - FirstKeyValue.Value = "newValue1"; + FirstKeyValue = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); featureFlags[0] = ConfigurationModelFactory.ConfigurationSetting( key: FeatureManagementConstants.FeatureFlagMarker + "myFeature", value: @" @@ -1837,7 +1870,7 @@ public async Task MapTransformFeatureFlagWithRefresh() ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -1868,7 +1901,6 @@ public void WithVariants() Assert.Equal("Big", config["feature_management:feature_flags:0:variants:0:name"]); Assert.Equal("600px", config["feature_management:feature_flags:0:variants:0:configuration_value"]); Assert.Equal("Small", config["feature_management:feature_flags:0:variants:1:name"]); - Assert.Equal("ShoppingCart:Small", config["feature_management:feature_flags:0:variants:1:configuration_reference"]); Assert.Equal("Disabled", config["feature_management:feature_flags:0:variants:1:status_override"]); Assert.Equal("Small", config["feature_management:feature_flags:0:allocation:default_when_disabled"]); Assert.Equal("Small", config["feature_management:feature_flags:0:allocation:default_when_enabled"]); @@ -2023,7 +2055,7 @@ public void ThrowsOnIncorrectJsonTypes() var settings = new List() { CreateFeatureFlag("Feature1", variantsJsonString: @"[{""name"": 1}]"), - CreateFeatureFlag("Feature2", variantsJsonString: @"[{""configuration_reference"": true}]"), + CreateFeatureFlag("Feature2", requirementType: "2"), CreateFeatureFlag("Feature3", variantsJsonString: @"[{""status_override"": []}]"), CreateFeatureFlag("Feature4", seed: "{}"), CreateFeatureFlag("Feature5", defaultWhenDisabled: "5"), @@ -2038,8 +2070,7 @@ public void ThrowsOnIncorrectJsonTypes() CreateFeatureFlag("Feature14", telemetryEnabled: "14"), CreateFeatureFlag("Feature15", telemetryMetadataJsonString: @"{""key"": 15}"), CreateFeatureFlag("Feature16", clientFiltersJsonString: @"[{""name"": 16}]"), - CreateFeatureFlag("Feature17", clientFiltersJsonString: @"{""key"": [{""name"": ""name"", ""parameters"": 17}]}"), - CreateFeatureFlag("Feature18", requirementType: "18") + CreateFeatureFlag("Feature17", clientFiltersJsonString: @"{""key"": [{""name"": ""name"", ""parameters"": 17}]}") }; var mockResponse = new Mock(); diff --git a/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs b/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs index 06c88040..274bfe7f 100644 --- a/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs +++ b/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs @@ -505,9 +505,9 @@ public void DoesNotThrowKeyVaultExceptionWhenProviderIsOptional() .Returns(new MockAsyncPageable(new List { _kv })); var mockKeyValueAdapter = new Mock(MockBehavior.Strict); - mockKeyValueAdapter.Setup(adapter => adapter.CanProcess(_kv)) + mockKeyValueAdapter.Setup(adapter => adapter.CanProcess(It.IsAny())) .Returns(true); - mockKeyValueAdapter.Setup(adapter => adapter.ProcessKeyValue(_kv, It.IsAny(), It.IsAny(), It.IsAny())) + mockKeyValueAdapter.Setup(adapter => adapter.ProcessKeyValue(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Throws(new KeyVaultReferenceException("Key vault error", null)); mockKeyValueAdapter.Setup(adapter => adapter.OnChangeDetected(null)); mockKeyValueAdapter.Setup(adapter => adapter.OnConfigUpdated()); @@ -743,7 +743,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal(_secretValue, config[_kv.Key]); // Update sentinel key-value - sentinelKv.Value = "Value2"; + sentinelKv = TestHelpers.ChangeValue(sentinelKv, "Value2"); Thread.Sleep(refreshInterval); await refresher.RefreshAsync(); @@ -815,7 +815,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal(_secretValue, config[_kv.Key]); // Update sentinel key-value to trigger refresh operation - sentinelKv.Value = "Value2"; + sentinelKv = TestHelpers.ChangeValue(sentinelKv, "Value2"); Thread.Sleep(refreshInterval); await refresher.RefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/LoggingTests.cs b/tests/Tests.AzureAppConfiguration/LoggingTests.cs index 547c65bd..6fcec29f 100644 --- a/tests/Tests.AzureAppConfiguration/LoggingTests.cs +++ b/tests/Tests.AzureAppConfiguration/LoggingTests.cs @@ -239,7 +239,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal("SentinelValue", config["SentinelKey"]); // Update sentinel key-value to trigger refreshAll operation - sentinelKv.Value = "UpdatedSentinelValue"; + sentinelKv = TestHelpers.ChangeValue(sentinelKv, "UpdatedSentinelValue"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -377,7 +377,7 @@ public async Task ValidateFailoverToDifferentEndpointMessageLoggedAfterFailover( .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -389,7 +389,7 @@ public async Task ValidateFailoverToDifferentEndpointMessageLoggedAfterFailover( .Throws(new RequestFailedException(HttpStatusCodes.TooManyRequests, "Too many requests")); mockClient2.Setup(c => c.ToString()).Returns("client"); - FirstKeyValue.Value = "TestValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "TestValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -431,7 +431,7 @@ public async Task ValidateConfigurationUpdatedSuccessLoggedDuringRefresh() .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -480,7 +480,7 @@ public async Task ValidateCorrectEndpointLoggedOnConfigurationUpdate() }) .Build(); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -528,7 +528,7 @@ public async Task ValidateCorrectKeyValueLoggedDuringRefresh() .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/MapTests.cs b/tests/Tests.AzureAppConfiguration/MapTests.cs index 623ae477..cdf15f4e 100644 --- a/tests/Tests.AzureAppConfiguration/MapTests.cs +++ b/tests/Tests.AzureAppConfiguration/MapTests.cs @@ -181,7 +181,7 @@ public async Task MapTransformWithRefresh() Assert.Equal("TestValue1 mapped first", config["TestKey1"]); Assert.Equal("TestValue2 second", config["TestKey2"]); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue1"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -232,8 +232,8 @@ public async Task MapTransformSettingKeyWithRefresh() Assert.Null(config["TestKey1"]); Assert.Equal("TestValue2", config["TestKey2"]); - FirstKeyValue.Value = "newValue1"; - _kvCollection.Last().Value = "newValue2"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue1"); + _kvCollection[1] = TestHelpers.ChangeValue(_kvCollection[1], "newValue2"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -282,8 +282,8 @@ public async Task MapTransformSettingLabelWithRefresh() Assert.Equal("TestValue1 changed", config["TestKey1"]); Assert.Equal("TestValue2 changed", config["TestKey2"]); - FirstKeyValue.Value = "newValue1"; - _kvCollection.Last().Value = "newValue2"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue1"); + _kvCollection[1] = TestHelpers.ChangeValue(_kvCollection[1], "newValue2"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); @@ -523,8 +523,8 @@ public async Task MapTransformSettingKeyWithLogAndRefresh() Assert.Null(config["TestKey1"]); Assert.Equal("TestValue2", config["TestKey2"]); - FirstKeyValue.Value = "newValue1"; - _kvCollection.Last().Value = "newValue2"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue1"); + _kvCollection[1] = TestHelpers.ChangeValue(_kvCollection[1], "newValue2"); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/PushRefreshTests.cs b/tests/Tests.AzureAppConfiguration/PushRefreshTests.cs index c4c7c38c..41e265d2 100644 --- a/tests/Tests.AzureAppConfiguration/PushRefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/PushRefreshTests.cs @@ -347,7 +347,7 @@ public async Task RefreshAsyncUpdatesConfig() .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue1"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue1"); refresher.ProcessPushNotification(_pushNotificationList.First(), TimeSpan.FromSeconds(0)); await refresher.RefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/RefreshTests.cs b/tests/Tests.AzureAppConfiguration/RefreshTests.cs index 6edc1a9a..e18e8453 100644 --- a/tests/Tests.AzureAppConfiguration/RefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/RefreshTests.cs @@ -7,6 +7,7 @@ using Azure.Identity; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration.AzureAppConfiguration; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; using Microsoft.Extensions.Logging.Abstractions; using Moq; using System; @@ -208,7 +209,7 @@ public async Task RefreshTests_RefreshIsNotSkippedIfCacheIsExpired() .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue"; + _kvCollection[0] = TestHelpers.ChangeValue(FirstKeyValue, "newValue"); // Wait for the cache to expire Thread.Sleep(1500); @@ -221,7 +222,6 @@ public async Task RefreshTests_RefreshIsNotSkippedIfCacheIsExpired() [Fact] public async Task RefreshTests_RefreshAllFalseDoesNotUpdateEntireConfiguration() { - var keyValueCollection = new List(_kvCollection); IConfigurationRefresher refresher = null; var mockClient = GetMockConfigurationClient(); @@ -244,7 +244,7 @@ public async Task RefreshTests_RefreshAllFalseDoesNotUpdateEntireConfiguration() Assert.Equal("TestValue2", config["TestKey2"]); Assert.Equal("TestValue3", config["TestKey3"]); - keyValueCollection.ForEach(kv => kv.Value = "newValue"); + _kvCollection = _kvCollection.Select(kv => TestHelpers.ChangeValue(kv, "newValue")).ToList(); // Wait for the cache to expire Thread.Sleep(1500); @@ -259,7 +259,6 @@ public async Task RefreshTests_RefreshAllFalseDoesNotUpdateEntireConfiguration() [Fact] public async Task RefreshTests_RefreshAllTrueUpdatesEntireConfiguration() { - var keyValueCollection = new List(_kvCollection); IConfigurationRefresher refresher = null; var mockClient = GetMockConfigurationClient(); @@ -282,7 +281,7 @@ public async Task RefreshTests_RefreshAllTrueUpdatesEntireConfiguration() Assert.Equal("TestValue2", config["TestKey2"]); Assert.Equal("TestValue3", config["TestKey3"]); - keyValueCollection.ForEach(kv => kv.Value = "newValue"); + _kvCollection = _kvCollection.Select(kv => TestHelpers.ChangeValue(kv, "newValue")).ToList(); // Wait for the cache to expire Thread.Sleep(1500); @@ -353,7 +352,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal("TestValue2", config["TestKey2"]); Assert.Equal("TestValue3", config["TestKey3"]); - keyValueCollection.First().Value = "newValue"; + keyValueCollection[0] = TestHelpers.ChangeValue(keyValueCollection[0], "newValue"); keyValueCollection.Remove(keyValueCollection.FirstOrDefault(s => s.Key == "TestKey3" && s.Label == "label")); // Wait for the cache to expire @@ -426,8 +425,8 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal("TestValue2", config["TestKey2"]); Assert.Equal("TestValue3", config["TestKey3"]); - keyValueCollection.ElementAt(0).Value = "newValue1"; - keyValueCollection.ElementAt(1).Value = "newValue2"; + keyValueCollection[0] = TestHelpers.ChangeValue(keyValueCollection[0], "newValue1"); + keyValueCollection[1] = TestHelpers.ChangeValue(keyValueCollection[1], "newValue2"); keyValueCollection.Remove(keyValueCollection.FirstOrDefault(s => s.Key == "TestKey3" && s.Label == "label")); // Wait for the cache to expire @@ -499,7 +498,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal("TestValue1", config["TestKey1"]); Assert.Equal(1, requestCount); - keyValueCollection.First().Value = "newValue"; + keyValueCollection[0] = TestHelpers.ChangeValue(keyValueCollection[0], "newValue"); // Simulate simultaneous refresh calls with expired cache from multiple threads var task1 = Task.Run(() => WaitAndRefresh(refresher, 1500)); @@ -606,7 +605,7 @@ public async Task RefreshTests_TryRefreshAsyncUpdatesConfigurationAndReturnsTrue .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue"); // Wait for the cache to expire Thread.Sleep(1500); @@ -702,7 +701,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - FirstKeyValue.Value = "newValue"; + _kvCollection[0] = TestHelpers.ChangeValue(_kvCollection[0], "newValue"); // Wait for the cache to expire Thread.Sleep(1500); @@ -823,7 +822,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o Assert.Equal("TestValue2", config["TestKey2"]); Assert.Equal("TestValue3", config["TestKey3"]); - keyValueCollection.ForEach(kv => kv.Value = "newValue"); + keyValueCollection = keyValueCollection.Select(kv => TestHelpers.ChangeValue(kv, "newValue")).ToList(); // Wait for the cache to expire Thread.Sleep(1500); @@ -874,7 +873,7 @@ public async Task RefreshTests_RefreshAllTrueForOverwrittenSentinelUpdatesEntire Assert.Equal("TestValue3", config["TestKey3"]); Assert.Equal("TestValueForLabel2", config["TestKeyWithMultipleLabels"]); - keyValueCollection.ForEach(kv => kv.Value = "newValue"); + _kvCollection = _kvCollection.Select(kv => TestHelpers.ChangeValue(kv, "newValue")).ToList(); // Wait for the cache to expire Thread.Sleep(1500); @@ -890,8 +889,7 @@ public async Task RefreshTests_RefreshAllTrueForOverwrittenSentinelUpdatesEntire [Fact] public async Task RefreshTests_RefreshAllFalseForOverwrittenSentinelUpdatesConfig() { - var keyValueCollection = new List(_kvCollection); - ConfigurationSetting refreshRegisteredSetting = keyValueCollection.FirstOrDefault(s => s.Key == "TestKeyWithMultipleLabels" && s.Label == "label1"); + ConfigurationSetting refreshRegisteredSetting = _kvCollection.FirstOrDefault(s => s.Key == "TestKeyWithMultipleLabels" && s.Label == "label1"); var mockClient = GetMockConfigurationClient(); IConfigurationRefresher refresher = null; @@ -916,7 +914,7 @@ public async Task RefreshTests_RefreshAllFalseForOverwrittenSentinelUpdatesConfi Assert.Equal("TestValue3", config["TestKey3"]); Assert.Equal("TestValueForLabel2", config["TestKeyWithMultipleLabels"]); - refreshRegisteredSetting.Value = "UpdatedValueForLabel1"; + _kvCollection[_kvCollection.IndexOf(refreshRegisteredSetting)] = TestHelpers.ChangeValue(refreshRegisteredSetting, "UpdatedValueForLabel1"); // Wait for the cache to expire Thread.Sleep(1500); @@ -933,8 +931,7 @@ public async Task RefreshTests_RefreshAllFalseForOverwrittenSentinelUpdatesConfi [Fact] public async Task RefreshTests_RefreshRegisteredKvOverwritesSelectedKv() { - var keyValueCollection = new List(_kvCollection); - ConfigurationSetting refreshAllRegisteredSetting = keyValueCollection.FirstOrDefault(s => s.Key == "TestKeyWithMultipleLabels" && s.Label == "label1"); + ConfigurationSetting refreshAllRegisteredSetting = _kvCollection.FirstOrDefault(s => s.Key == "TestKeyWithMultipleLabels" && s.Label == "label1"); var mockClient = GetMockConfigurationClient(); IConfigurationRefresher refresher = null; @@ -959,7 +956,7 @@ public async Task RefreshTests_RefreshRegisteredKvOverwritesSelectedKv() Assert.Equal("TestValue3", config["TestKey3"]); Assert.Equal("TestValueForLabel1", config["TestKeyWithMultipleLabels"]); - refreshAllRegisteredSetting.Value = "UpdatedValueForLabel1"; + _kvCollection[_kvCollection.IndexOf(refreshAllRegisteredSetting)] = TestHelpers.ChangeValue(refreshAllRegisteredSetting, "UpdatedValueForLabel1"); // Wait for the cache to expire Thread.Sleep(1500); @@ -1056,6 +1053,170 @@ public void RefreshTests_RefreshIsCancelled() Assert.Equal("TestValue1", config["TestKey1"]); } + [Fact] + public async Task RefreshTests_SelectedKeysRefreshWithRegisterAll() + { + IConfigurationRefresher refresher = null; + var mockClient = GetMockConfigurationClient(); + + var mockAsyncPageable = new MockAsyncPageable(_kvCollection); + + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Callback(() => mockAsyncPageable.UpdateCollection(_kvCollection)) + .Returns(mockAsyncPageable); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.Select("TestKey*"); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); + options.ConfigureRefresh(refreshOptions => + { + refreshOptions.RegisterAll() + .SetRefreshInterval(TimeSpan.FromSeconds(1)); + }); + + refresher = options.GetRefresher(); + }) + .Build(); + + Assert.Equal("TestValue1", config["TestKey1"]); + Assert.Equal("TestValue3", config["TestKey3"]); + FirstKeyValue.Value = "newValue1"; + _kvCollection[2].Value = "newValue3"; + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Equal("newValue3", config["TestKey3"]); + + _kvCollection.RemoveAt(2); + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Null(config["TestKey3"]); + } + + [Fact] + public async Task RefreshTests_RegisterAllRefreshesFeatureFlags() + { + IConfigurationRefresher refresher = null; + var mockClient = GetMockConfigurationClient(); + + var featureFlags = new List { + ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature", + value: @" + { + ""id"": ""MyFeature"", + ""description"": ""The new beta version of our web site."", + ""display_name"": ""Beta Feature"", + ""enabled"": true, + ""conditions"": { + ""client_filters"": [ + { + ""name"": ""SuperUsers"" + } + ] + } + } + ", + label: default, + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")) + }; + + var mockAsyncPageableKv = new MockAsyncPageable(_kvCollection); + + var mockAsyncPageableFf = new MockAsyncPageable(featureFlags); + + MockAsyncPageable GetTestKeys(SettingSelector selector, CancellationToken ct) + { + if (selector.KeyFilter.StartsWith(FeatureManagementConstants.FeatureFlagMarker)) + { + mockAsyncPageableFf.UpdateCollection(featureFlags); + + return mockAsyncPageableFf; + } + + mockAsyncPageableKv.UpdateCollection(_kvCollection); + + return mockAsyncPageableKv; + } + + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Returns((Func)GetTestKeys); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.Select("TestKey*"); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); + options.ConfigureRefresh(refreshOptions => + { + refreshOptions.RegisterAll() + .SetRefreshInterval(TimeSpan.FromSeconds(1)); + }); + options.UseFeatureFlags(); + + refresher = options.GetRefresher(); + }) + .Build(); + + Assert.Equal("TestValue1", config["TestKey1"]); + Assert.Equal("SuperUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); + + FirstKeyValue.Value = "newValue1"; + featureFlags[0] = ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature", + value: @" + { + ""id"": ""MyFeature"", + ""description"": ""The new beta version of our web site."", + ""display_name"": ""Beta Feature"", + ""enabled"": true, + ""conditions"": { + ""client_filters"": [ + { + ""name"": ""AllUsers"" + } + ] + } + } + ", + label: default, + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Equal("AllUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); + + FirstKeyValue.Value = "newerValue1"; + featureFlags.RemoveAt(0); + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newerValue1", config["TestKey1"]); + Assert.Null(config["FeatureManagement:MyFeature"]); + } + #if NET8_0 [Fact] public void RefreshTests_ChainedConfigurationProviderUsedAsRootForRefresherProvider() diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index bc7989b2..ca929efc 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -3,6 +3,7 @@ // using Azure; using Azure.Core; +using Azure.Core.Testing; using Azure.Data.AppConfiguration; using Microsoft.Extensions.Configuration.AzureAppConfiguration; using Microsoft.Extensions.Logging; @@ -10,6 +11,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; using System.Text.Json; using System.Threading; @@ -115,6 +117,11 @@ public static ConfigurationSetting CloneSetting(ConfigurationSetting setting) return ConfigurationModelFactory.ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ContentType, setting.ETag, setting.LastModified); } + public static ConfigurationSetting ChangeValue(ConfigurationSetting setting, string value) + { + return ConfigurationModelFactory.ConfigurationSetting(setting.Key, value, setting.Label, setting.ContentType, new ETag(Guid.NewGuid().ToString()), setting.LastModified); + } + public static List LoadJsonSettingsFromFile(string path) { List _kvCollection = new List(); @@ -155,19 +162,69 @@ public static bool ValidateLog(Mock logger, string expectedMessage, Log class MockAsyncPageable : AsyncPageable { - private readonly List _collection; + private readonly List _collection = new List(); + private int _status; public MockAsyncPageable(List collection) { - _collection = collection; + foreach (ConfigurationSetting setting in collection) + { + var newSetting = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + newSetting.ContentType = setting.ContentType; + + _collection.Add(newSetting); + } + + _status = 200; + } + + public void UpdateCollection(List newCollection) + { + if (_collection.Count() == newCollection.Count() && + _collection.All(setting => newCollection.Any(newSetting => + setting.Key == newSetting.Key && + setting.Value == newSetting.Value && + setting.Label == newSetting.Label && + setting.ETag == newSetting.ETag))) + { + _status = 304; + } + else + { + _status = 200; + + _collection.Clear(); + + foreach (ConfigurationSetting setting in newCollection) + { + var newSetting = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + newSetting.ContentType = setting.ContentType; + + _collection.Add(newSetting); + } + } } #pragma warning disable 1998 public async override IAsyncEnumerable> AsPages(string continuationToken = null, int? pageSizeHint = null) #pragma warning restore 1998 { - yield return Page.FromValues(_collection, null, new Mock().Object); + yield return Page.FromValues(_collection, null, new MockResponse(_status)); + } + } + internal class MockConfigurationSettingPageIterator : IConfigurationSettingPageIterator + { + public IAsyncEnumerable> IteratePages(AsyncPageable pageable, IEnumerable matchConditions) + { + return pageable.AsPages(); + } + + public IAsyncEnumerable> IteratePages(AsyncPageable pageable) + { + return pageable.AsPages(); } } @@ -182,7 +239,7 @@ public MockPageable(List collection) public override IEnumerable> AsPages(string continuationToken = null, int? pageSizeHint = null) { - yield return Page.FromValues(_collection, null, new Mock().Object); + yield return Page.FromValues(_collection, null, new MockResponse(200)); } } }