Skip to content

Commit 5a1cca2

Browse files
muzahidul-optimuzahidul-opti
committed
feat: add method for handling decision notifications with holdouts
- Implement createOptimizelyWithHoldouts method to create Optimizely instance with holdouts - Add test for decisionNotification with holdout scenario - Update decide_for_keys_with_holdout test for holdout variations - Implement decide_all_with_holdout test for multiple flag keys with holdouts
1 parent 075fd4e commit 5a1cca2

File tree

1 file changed

+187
-0
lines changed

1 file changed

+187
-0
lines changed

‎core-api/src/test/java/com/optimizely/ab/OptimizelyUserContextTest.java

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,9 @@
5757
import static junit.framework.TestCase.assertEquals;
5858
import static junit.framework.TestCase.assertTrue;
5959
import static org.junit.Assert.*;
60+
import static org.mockito.Matchers.any;
61+
import static org.mockito.Matchers.anyObject;
62+
import static org.mockito.Matchers.eq;
6063
import static org.mockito.Mockito.*;
6164

6265
public class OptimizelyUserContextTest {
@@ -2074,4 +2077,188 @@ OptimizelyDecision callDecideWithIncludeReasons(String flagKey) {
20742077
return callDecideWithIncludeReasons(flagKey, Collections.emptyMap());
20752078
}
20762079

2080+
private Optimizely createOptimizelyWithHoldouts() throws Exception {
2081+
String holdoutDatafile = com.google.common.io.Resources.toString(
2082+
com.google.common.io.Resources.getResource("config/holdouts-project-config.json"),
2083+
com.google.common.base.Charsets.UTF_8
2084+
);
2085+
return new Optimizely.Builder().withDatafile(holdoutDatafile).withEventProcessor(new ForwardingEventProcessor(eventHandler, null)).build();
2086+
}
2087+
2088+
@Test
2089+
public void decisionNotification_with_holdout() throws Exception {
2090+
// Use holdouts datafile
2091+
Optimizely optWithHoldout = createOptimizelyWithHoldouts();
2092+
String flagKey = "boolean_feature";
2093+
String userId = "user123";
2094+
String ruleKey = "basic_holdout"; // holdout rule key
2095+
String variationKey = "ho_off_key"; // holdout (off) variation key
2096+
String experimentId = "10075323428"; // holdout experiment id in holdouts-project-config.json
2097+
String variationId = "$opt_dummy_variation_id";// dummy variation id used for holdout impressions
2098+
String expectedReason = "User (" + userId + ") is in variation (" + variationKey + ") of holdout (" + ruleKey + ").";
2099+
2100+
Map<String, Object> attrs = new HashMap<>();
2101+
attrs.put("$opt_bucketing_id", "ppid160000"); // deterministic bucketing into basic_holdout
2102+
attrs.put("nationality", "English"); // non-reserved attribute should appear in impression & notification
2103+
2104+
OptimizelyUserContext user = optWithHoldout.createUserContext(userId, attrs);
2105+
2106+
// Register notification handler similar to decisionNotification test
2107+
isListenerCalled = false;
2108+
optWithHoldout.addDecisionNotificationHandler(decisionNotification -> {
2109+
Assert.assertEquals(NotificationCenter.DecisionNotificationType.FLAG.toString(), decisionNotification.getType());
2110+
Assert.assertEquals(userId, decisionNotification.getUserId());
2111+
2112+
Assert.assertEquals(attrs, decisionNotification.getAttributes());
2113+
2114+
Map<String, ?> info = decisionNotification.getDecisionInfo();
2115+
Assert.assertEquals(flagKey, info.get(FLAG_KEY));
2116+
Assert.assertEquals(variationKey, info.get(VARIATION_KEY));
2117+
Assert.assertEquals(false, info.get(ENABLED));
2118+
Assert.assertEquals(ruleKey, info.get(RULE_KEY));
2119+
Assert.assertEquals(experimentId, info.get(EXPERIMENT_ID));
2120+
Assert.assertEquals(variationId, info.get(VARIATION_ID));
2121+
// Variables should be empty because feature is disabled by holdout
2122+
Assert.assertTrue(((Map<?, ?>) info.get(VARIABLES)).isEmpty());
2123+
// Event should be dispatched (no DISABLE_DECISION_EVENT option)
2124+
Assert.assertEquals(true, info.get(DECISION_EVENT_DISPATCHED));
2125+
2126+
@SuppressWarnings("unchecked")
2127+
List<String> reasons = (List<String>) info.get(REASONS);
2128+
Assert.assertTrue("Expected holdout reason present", reasons.contains(expectedReason));
2129+
isListenerCalled = true;
2130+
});
2131+
2132+
// Execute decision with INCLUDE_REASONS so holdout reason is present
2133+
OptimizelyDecision decision = user.decide(flagKey, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS));
2134+
assertTrue(isListenerCalled);
2135+
2136+
// Sanity checks on returned decision
2137+
assertEquals(variationKey, decision.getVariationKey());
2138+
assertFalse(decision.getEnabled());
2139+
assertTrue(decision.getReasons().contains(expectedReason));
2140+
2141+
// Impression expectation (nationality only)
2142+
DecisionMetadata metadata = new DecisionMetadata.Builder()
2143+
.setFlagKey(flagKey)
2144+
.setRuleKey(ruleKey)
2145+
.setRuleType("holdout")
2146+
.setVariationKey(variationKey)
2147+
.setEnabled(false)
2148+
.build();
2149+
eventHandler.expectImpression(experimentId, variationId, userId, Collections.singletonMap("nationality", "English"), metadata);
2150+
2151+
// Log expectation (reuse existing pattern)
2152+
logbackVerifier.expectMessage(Level.INFO, expectedReason);
2153+
}
2154+
2155+
@Test
2156+
public void decide_for_keys_with_holdout() throws Exception {
2157+
Optimizely optWithHoldout = createOptimizelyWithHoldouts();
2158+
String userId = "user123";
2159+
Map<String, Object> attrs = new HashMap<>();
2160+
attrs.put("$opt_bucketing_id", "ppid160000");
2161+
OptimizelyUserContext user = optWithHoldout.createUserContext(userId, attrs);
2162+
2163+
List<String> flagKeys = Arrays.asList(
2164+
"boolean_feature", // previously validated basic_holdout membership
2165+
"double_single_variable_feature", // also subject to global/basic holdout
2166+
"integer_single_variable_feature" // also subject to global/basic holdout
2167+
);
2168+
2169+
Map<String, OptimizelyDecision> decisions = user.decideForKeys(flagKeys, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS));
2170+
assertEquals(3, decisions.size());
2171+
2172+
String holdoutExperimentId = "10075323428"; // basic_holdout id
2173+
String variationId = "$opt_dummy_variation_id";
2174+
String variationKey = "ho_off_key";
2175+
String expectedReason = "User (" + userId + ") is in variation (" + variationKey + ") of holdout (basic_holdout).";
2176+
2177+
for (String flagKey : flagKeys) {
2178+
OptimizelyDecision d = decisions.get(flagKey);
2179+
assertNotNull(d);
2180+
assertEquals(flagKey, d.getFlagKey());
2181+
assertEquals(variationKey, d.getVariationKey());
2182+
assertFalse(d.getEnabled());
2183+
assertTrue("Expected holdout reason for flag " + flagKey, d.getReasons().contains(expectedReason));
2184+
DecisionMetadata metadata = new DecisionMetadata.Builder()
2185+
.setFlagKey(flagKey)
2186+
.setRuleKey("basic_holdout")
2187+
.setRuleType("holdout")
2188+
.setVariationKey(variationKey)
2189+
.setEnabled(false)
2190+
.build();
2191+
// attributes map expected empty (reserved $opt_ attribute filtered out)
2192+
eventHandler.expectImpression(holdoutExperimentId, variationId, userId, Collections.emptyMap(), metadata);
2193+
}
2194+
2195+
// At least one log message confirming holdout membership
2196+
logbackVerifier.expectMessage(Level.INFO, expectedReason);
2197+
}
2198+
2199+
@Test
2200+
public void decide_all_with_holdout() throws Exception {
2201+
2202+
Optimizely optWithHoldout = createOptimizelyWithHoldouts();
2203+
String userId = "user123";
2204+
Map<String, Object> attrs = new HashMap<>();
2205+
// ppid120000 buckets user into holdout_included_flags
2206+
attrs.put("$opt_bucketing_id", "ppid120000");
2207+
OptimizelyUserContext user = optWithHoldout.createUserContext(userId, attrs);
2208+
2209+
// All flag keys present in holdouts-project-config.json
2210+
List<String> allFlagKeys = Arrays.asList(
2211+
"boolean_feature",
2212+
"double_single_variable_feature",
2213+
"integer_single_variable_feature",
2214+
"boolean_single_variable_feature",
2215+
"string_single_variable_feature",
2216+
"multi_variate_feature",
2217+
"multi_variate_future_feature",
2218+
"mutex_group_feature"
2219+
);
2220+
2221+
// Flags INCLUDED in holdout_included_flags (only these should be holdout decisions)
2222+
List<String> includedInHoldout = Arrays.asList(
2223+
"boolean_feature",
2224+
"double_single_variable_feature",
2225+
"integer_single_variable_feature"
2226+
);
2227+
2228+
Map<String, OptimizelyDecision> decisions = user.decideAll(Arrays.asList(
2229+
OptimizelyDecideOption.INCLUDE_REASONS,
2230+
OptimizelyDecideOption.DISABLE_DECISION_EVENT
2231+
));
2232+
assertEquals(allFlagKeys.size(), decisions.size());
2233+
2234+
String holdoutExperimentId = "1007543323427"; // holdout_included_flags id
2235+
String variationId = "$opt_dummy_variation_id";
2236+
String variationKey = "ho_off_key";
2237+
String expectedReason = "User (" + userId + ") is in variation (" + variationKey + ") of holdout (holdout_included_flags).";
2238+
2239+
int holdoutCount = 0;
2240+
for (String flagKey : allFlagKeys) {
2241+
OptimizelyDecision d = decisions.get(flagKey);
2242+
assertNotNull("Missing decision for flag " + flagKey, d);
2243+
if (includedInHoldout.contains(flagKey)) {
2244+
// Should be holdout decision
2245+
assertEquals(variationKey, d.getVariationKey());
2246+
assertFalse(d.getEnabled());
2247+
assertTrue("Expected holdout reason for flag " + flagKey, d.getReasons().contains(expectedReason));
2248+
DecisionMetadata metadata = new DecisionMetadata.Builder()
2249+
.setFlagKey(flagKey)
2250+
.setRuleKey("holdout_included_flags")
2251+
.setRuleType("holdout")
2252+
.setVariationKey(variationKey)
2253+
.setEnabled(false)
2254+
.build();
2255+
holdoutCount++;
2256+
} else {
2257+
// Should NOT be a holdout decision
2258+
assertFalse("Non-included flag should not have holdout reason: " + flagKey, d.getReasons().contains(expectedReason));
2259+
}
2260+
}
2261+
assertEquals("Expected exactly the included flags to be in holdout", includedInHoldout.size(), holdoutCount);
2262+
logbackVerifier.expectMessage(Level.INFO, expectedReason);
2263+
}
20772264
}

0 commit comments

Comments
 (0)