Skip to content

Commit 623df91

Browse files
junaed-optimizelyjunaed-optimizely
committed
[FSSDK-11546] decision service holdout test addition
1 parent 2cbe374 commit 623df91

File tree

4 files changed

+251
-2
lines changed

4 files changed

+251
-2
lines changed

‎OptimizelySDK.Tests/BucketerHoldoutTest.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/*
1+
/*
22
* Copyright 2025, Optimizely
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");

‎OptimizelySDK.Tests/DecisionServiceHoldoutTest.cs

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
/*
2+
* Copyright 2025, Optimizely
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
using System;
18+
using System.Collections.Generic;
19+
using System.IO;
20+
using System.Linq;
21+
using Moq;
22+
using Newtonsoft.Json.Linq;
23+
using NUnit.Framework;
24+
using OptimizelySDK.Bucketing;
25+
using OptimizelySDK.Config;
26+
using OptimizelySDK.Entity;
27+
using OptimizelySDK.ErrorHandler;
28+
using OptimizelySDK.Logger;
29+
using OptimizelySDK.OptimizelyDecisions;
30+
31+
namespace OptimizelySDK.Tests
32+
{
33+
[TestFixture]
34+
public class DecisionServiceHoldoutTest
35+
{
36+
private Mock<ILogger> LoggerMock;
37+
private DecisionService DecisionService;
38+
private DatafileProjectConfig Config;
39+
private JObject TestData;
40+
private Optimizely OptimizelyInstance;
41+
42+
private const string TestUserId = "testUserId";
43+
private const string TestBucketingId = "testBucketingId";
44+
45+
[SetUp]
46+
public void Initialize()
47+
{
48+
LoggerMock = new Mock<ILogger>();
49+
50+
// Load test data
51+
var testDataPath = Path.Combine(TestContext.CurrentContext.TestDirectory,
52+
"TestData", "HoldoutTestData.json");
53+
var jsonContent = File.ReadAllText(testDataPath);
54+
TestData = JObject.Parse(jsonContent);
55+
56+
// Use datafile with holdouts for proper config setup
57+
var datafileWithHoldouts = TestData["datafileWithHoldouts"].ToString();
58+
Config = DatafileProjectConfig.Create(datafileWithHoldouts, LoggerMock.Object,
59+
new ErrorHandler.NoOpErrorHandler()) as DatafileProjectConfig;
60+
61+
// Use real Bucketer instead of mock
62+
var realBucketer = new Bucketer(LoggerMock.Object);
63+
DecisionService = new DecisionService(realBucketer,
64+
new ErrorHandler.NoOpErrorHandler(), null, LoggerMock.Object);
65+
66+
// Create an Optimizely instance for creating user contexts
67+
var eventDispatcher = new Event.Dispatcher.DefaultEventDispatcher(LoggerMock.Object);
68+
OptimizelyInstance = new Optimizely(datafileWithHoldouts, eventDispatcher, LoggerMock.Object);
69+
70+
// Verify that the config contains holdouts
71+
Assert.IsNotNull(Config.Holdouts, "Config should have holdouts");
72+
Assert.IsTrue(Config.Holdouts.Length > 0, "Config should contain holdouts");
73+
}
74+
75+
[Test]
76+
public void TestGetVariationsForFeatureList_HoldoutActiveVariationBucketed()
77+
{
78+
// Test GetVariationsForFeatureList with holdout that has an active variation
79+
var featureFlag = Config.FeatureKeyMap["test_flag_1"]; // Use actual feature flag from test data
80+
var holdout = Config.GetHoldout("holdout_included_1"); // This holdout includes flag_1
81+
Assert.IsNotNull(holdout, "Holdout should exist in config");
82+
83+
// Create user context
84+
var userContext = new OptimizelyUserContext(OptimizelyInstance, TestUserId, null,
85+
new ErrorHandler.NoOpErrorHandler(), LoggerMock.Object);
86+
87+
var result = DecisionService.GetVariationsForFeatureList(
88+
new List<FeatureFlag> { featureFlag }, userContext, Config,
89+
new UserAttributes(), new OptimizelyDecideOption[0]);
90+
91+
Assert.IsNotNull(result);
92+
Assert.IsTrue(result.Count > 0, "Should have at least one decision");
93+
94+
// Find the holdout decision
95+
var holdoutDecision = result.FirstOrDefault(r => r.ResultObject?.Source == FeatureDecision.DECISION_SOURCE_HOLDOUT);
96+
Assert.IsNotNull(holdoutDecision, "Should have a holdout decision");
97+
98+
// Verify that we got a valid variation (real bucketer should determine this based on traffic allocation)
99+
Assert.IsNotNull(holdoutDecision.ResultObject?.Variation, "Should have a variation");
100+
}
101+
102+
[Test]
103+
public void TestGetVariationsForFeatureList_HoldoutInactiveNoBucketing()
104+
{
105+
// Test that inactive holdouts don't bucket users
106+
var featureFlag = Config.FeatureKeyMap["test_flag_1"]; // Use actual feature flag from test data
107+
108+
// Get one of the holdouts that's actually processed for test_flag_1 (based on debug output)
109+
var holdout = Config.GetHoldout("holdout_global_1"); // global_holdout is one of the holdouts being processed
110+
Assert.IsNotNull(holdout, "Holdout should exist in config");
111+
112+
// Mock holdout as inactive
113+
holdout.Status = "Paused";
114+
115+
var userContext = new OptimizelyUserContext(OptimizelyInstance, TestUserId, null,
116+
new ErrorHandler.NoOpErrorHandler(), LoggerMock.Object);
117+
118+
var result = DecisionService.GetVariationsForFeatureList(
119+
new List<FeatureFlag> { featureFlag }, userContext, Config,
120+
new UserAttributes(), new OptimizelyDecideOption[0]);
121+
122+
// Verify appropriate log message for inactive holdout
123+
LoggerMock.Verify(l => l.Log(LogLevel.INFO,
124+
It.Is<string>(s => s.Contains("Holdout") && s.Contains("is not running"))),
125+
Times.AtLeastOnce);
126+
}
127+
128+
[Test]
129+
public void TestGetVariationsForFeatureList_HoldoutUserNotBucketed()
130+
{
131+
// Test when user is not bucketed into holdout (outside traffic allocation)
132+
var featureFlag = Config.FeatureKeyMap["test_flag_1"]; // Use actual feature flag from test data
133+
var holdout = Config.GetHoldout("holdout_included_1"); // This holdout includes flag_1
134+
Assert.IsNotNull(holdout, "Holdout should exist in config");
135+
136+
var userContext = new OptimizelyUserContext(OptimizelyInstance, TestUserId, null,
137+
new ErrorHandler.NoOpErrorHandler(), LoggerMock.Object);
138+
139+
var result = DecisionService.GetVariationsForFeatureList(
140+
new List<FeatureFlag> { featureFlag }, userContext, Config,
141+
new UserAttributes(), new OptimizelyDecideOption[0]);
142+
143+
// With real bucketer, we can't guarantee specific bucketing results
144+
// but we can verify the method executes successfully
145+
Assert.IsNotNull(result, "Result should not be null");
146+
}
147+
148+
[Test]
149+
public void TestGetVariationsForFeatureList_HoldoutWithUserAttributes()
150+
{
151+
// Test holdout evaluation with user attributes for audience targeting
152+
var featureFlag = Config.FeatureKeyMap["test_flag_1"]; // Use actual feature flag from test data
153+
var holdout = Config.GetHoldout("holdout_included_1"); // This holdout includes flag_1
154+
Assert.IsNotNull(holdout, "Holdout should exist in config");
155+
156+
var userAttributes = new UserAttributes
157+
{
158+
{"browser", "chrome"},
159+
{"location", "us"}
160+
};
161+
162+
var userContext = new OptimizelyUserContext(OptimizelyInstance, TestUserId, userAttributes,
163+
new ErrorHandler.NoOpErrorHandler(), LoggerMock.Object);
164+
165+
var result = DecisionService.GetVariationsForFeatureList(
166+
new List<FeatureFlag> { featureFlag }, userContext, Config,
167+
userAttributes, new OptimizelyDecideOption[0]);
168+
169+
Assert.IsNotNull(result, "Result should not be null");
170+
171+
// With real bucketer, we can't guarantee specific variations but can verify execution
172+
// Additional assertions would depend on the holdout configuration and user bucketing
173+
}
174+
175+
[Test]
176+
public void TestGetVariationsForFeatureList_MultipleHoldouts()
177+
{
178+
// Test multiple holdouts for a single feature flag
179+
var featureFlag = Config.FeatureKeyMap["test_flag_1"]; // Use actual feature flag from test data
180+
181+
var userContext = new OptimizelyUserContext(OptimizelyInstance, TestUserId, null,
182+
new ErrorHandler.NoOpErrorHandler(), LoggerMock.Object);
183+
184+
var result = DecisionService.GetVariationsForFeatureList(
185+
new List<FeatureFlag> { featureFlag }, userContext, Config,
186+
new UserAttributes(), new OptimizelyDecideOption[0]);
187+
188+
Assert.IsNotNull(result, "Result should not be null");
189+
190+
// With real bucketer, we can't guarantee specific bucketing results
191+
// but we can verify the method executes successfully
192+
}
193+
194+
[Test]
195+
public void TestGetVariationsForFeatureList_Holdout_EmptyUserId()
196+
{
197+
// Test GetVariationsForFeatureList with empty user ID
198+
var featureFlag = Config.FeatureKeyMap["test_flag_1"]; // Use actual feature flag from test data
199+
200+
var userContext = new OptimizelyUserContext(OptimizelyInstance, "", null,
201+
new ErrorHandler.NoOpErrorHandler(), LoggerMock.Object);
202+
203+
var result = DecisionService.GetVariationsForFeatureList(
204+
new List<FeatureFlag> { featureFlag }, userContext, Config,
205+
new UserAttributes(), new OptimizelyDecideOption[0]);
206+
207+
Assert.IsNotNull(result);
208+
209+
// Empty user ID should still allow holdout bucketing (matches Swift SDK behavior)
210+
// The Swift SDK's testBucketToVariation_EmptyBucketingId shows empty string is valid
211+
var holdoutDecisions = result.Where(r => r.ResultObject?.Source == FeatureDecision.DECISION_SOURCE_HOLDOUT).ToList();
212+
213+
// Should not log error about invalid user ID since empty string is valid for bucketing
214+
LoggerMock.Verify(l => l.Log(LogLevel.ERROR,
215+
It.Is<string>(s => s.Contains("User ID") && (s.Contains("null") || s.Contains("empty")))),
216+
Times.Never);
217+
}
218+
219+
[Test]
220+
public void TestGetVariationsForFeatureList_Holdout_DecisionReasons()
221+
{
222+
// Test that decision reasons are properly populated for holdouts
223+
var featureFlag = Config.FeatureKeyMap["test_flag_1"]; // Use actual feature flag from test data
224+
var holdout = Config.GetHoldout("holdout_included_1"); // This holdout includes flag_1
225+
Assert.IsNotNull(holdout, "Holdout should exist in config");
226+
227+
var userContext = new OptimizelyUserContext(OptimizelyInstance, TestUserId, null,
228+
new ErrorHandler.NoOpErrorHandler(), LoggerMock.Object);
229+
230+
var result = DecisionService.GetVariationsForFeatureList(
231+
new List<FeatureFlag> { featureFlag }, userContext, Config,
232+
new UserAttributes(), new OptimizelyDecideOption[0]);
233+
234+
Assert.IsNotNull(result, "Result should not be null");
235+
236+
// With real bucketer, we expect proper decision reasons to be generated
237+
// Find any decision with reasons
238+
var decisionWithReasons = result.FirstOrDefault(r => r.DecisionReasons != null && r.DecisionReasons.ToReport().Count > 0);
239+
240+
if (decisionWithReasons != null)
241+
{
242+
Assert.IsTrue(decisionWithReasons.DecisionReasons.ToReport().Count > 0, "Should have decision reasons");
243+
}
244+
}
245+
}
246+
}

‎OptimizelySDK.Tests/OptimizelySDK.Tests.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@
107107
<Compile Include="Properties\AssemblyInfo.cs"/>
108108
<Compile Include="TestBucketer.cs"/>
109109
<Compile Include="BucketerHoldoutTest.cs"/>
110+
<Compile Include="DecisionServiceHoldoutTest.cs"/>
110111
<Compile Include="BucketerTest.cs"/>
111112
<Compile Include="ProjectConfigTest.cs"/>
112113
<Compile Include="TestSetup.cs"/>

‎OptimizelySDK/Bucketing/DecisionService.cs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -883,7 +883,9 @@ ProjectConfig config
883883

884884
if (!holdout.IsActivated)
885885
{
886-
reasons.AddInfo($"Holdout \"{holdout.Key}\" is not running.");
886+
var infoMessage = $"Holdout \"{holdout.Key}\" is not running.";
887+
Logger.Log(LogLevel.INFO, infoMessage);
888+
reasons.AddInfo(infoMessage);
887889
return Result<FeatureDecision>.NewResult(
888890
new FeatureDecision(null, null, FeatureDecision.DECISION_SOURCE_HOLDOUT),
889891
reasons

0 commit comments

Comments
 (0)