feat: SDK Package test runner; allow servers to skip dynamic-typing, bandits (#95)

* Add spec for SDK details endpoint

* Implement SDK Details in dotnet relay

* sdk details supports bools instead of map WIP

* separate params

* exta _

* defaults and renames

Author: typotter

package-testing/dotnet-sdk-relay/EppoSDKRelay/Controllers/AssignmentController.cs

@@ -49,6 +49,4 @@ public ActionResult<string> Post([FromBody] AssignmentRequest data)
 
         return JsonError("Invalid Assignment Type " + data.AssignmentType);
     }
-
-
 }

package-testing/dotnet-sdk-relay/EppoSDKRelay/Controllers/JsonControllerBase.cs

@@ -1,21 +1,16 @@
-using Microsoft.AspNetCore.Mvc;
-
-using EppoSDKRelay.DTO;
 using System.Text.Json;
-using Newtonsoft.Json.Linq;
 using eppo_sdk.dto;
 using eppo_sdk.dto.bandit;
+using EppoSDKRelay.DTO;
+using Microsoft.AspNetCore.Mvc;
+using Newtonsoft.Json.Linq;
 
 namespace EppoSDKRelay.controllers;
 
 public class JsonControllerBase : ControllerBase
 {
-
-    protected static readonly JsonSerializerOptions SerializeOptions = new()
-    {
-        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
-        WriteIndented = true
-    };
+    protected static readonly JsonSerializerOptions SerializeOptions =
+        new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, WriteIndented = true };
 
     protected static ActionResult<string> JsonTestResponse(object result)
     {
@@ -30,18 +25,29 @@ protected static ActionResult<string> JsonTestResponse(object result)
         var response = new TestResponse
         {
             Result = result,
-            AssignmentLog = DequeueAllForResponse<AssignmentLogData>(AssignmentLogger.Instance.AssignmentLogs),
-            BanditLog =  DequeueAllForResponse<BanditLogEvent>(AssignmentLogger.Instance.BanditLogs),
+            AssignmentLog = DequeueAllForResponse<AssignmentLogData>(
+                AssignmentLogger.Instance.AssignmentLogs
+            ),
+            BanditLog = DequeueAllForResponse<BanditLogEvent>(AssignmentLogger.Instance.BanditLogs),
         };
         return JsonSerializer.Serialize(response, SerializeOptions);
     }
 
-    protected static ActionResult<string> JsonError(String error)
+    protected static ActionResult<string> JsonObjectResponse(object result)
     {
-        return JsonSerializer.Serialize(new TestResponse
+        // System.Text.Json does not play nicely with Newtonsoft types
+        // Since "Objects" implement IEnumerable, System.Text will try to encode
+        // the json object as an array. :(
+        if (result is JObject)
         {
-            Error = error
-        }, SerializeOptions);
+            result = ((JObject)result).ToObject<Dictionary<string, object>>();
+        }
+        return JsonSerializer.Serialize(result, SerializeOptions);
+    }
+
+    protected static ActionResult<string> JsonError(String error)
+    {
+        return JsonSerializer.Serialize(new TestResponse { Error = error }, SerializeOptions);
     }
 
     public static List<object> DequeueAllForResponse<T>(Queue<T> queue)

package-testing/dotnet-sdk-relay/EppoSDKRelay/Controllers/SDKContoller.cs

@@ -1,12 +1,14 @@
 using eppo_sdk;
+using eppo_sdk.helpers;
 using Microsoft.AspNetCore.Mvc;
 
+
 namespace EppoSDKRelay.controllers;
 
-[Route("/sdk/reset")]
-public class SDKController : ControllerBase
+public class SDKController : JsonControllerBase
 {
     // POST sdk/reset
+    [Route("/sdk/reset")]
     [HttpPost]
     public ActionResult<IEnumerable<string>> Reset()
     {
@@ -15,4 +17,23 @@ public ActionResult<IEnumerable<string>> Reset()
         return Ok();
     }
 
+    // GET sdk/details
+    [Route("/sdk/details")]
+    [HttpGet]
+    public ActionResult<String> Details()
+    {
+        // Sneak the SDK version and name from the AppDetails object
+        var apd = new AppDetails();
+        var sdkDetails = new Dictionary<string, object>
+        {
+            ["sdkVersion"] = apd.Version,
+            ["sdkName"] = apd.Name,
+            ["supportsBandits"] = true,
+            ["supportsDynamicTyping"] = true
+        };
+
+        return JsonObjectResponse(
+            sdkDetails
+        );
+    }
 }

package-testing/dotnet-sdk-relay/EppoSDKRelay/Startup.cs

@@ -5,36 +5,34 @@ namespace EppoSDKRelay;
 
 public class Startup
 {
-    static readonly String eppoBaseUrl = Environment.GetEnvironmentVariable("EPPO_BASE_URL") ?? "http://localhost:5000/api";
-    static readonly String apiToken = Environment.GetEnvironmentVariable("EPPO_API_TOKEN") ?? "NO_TOKEN";
-
+    static readonly String eppoBaseUrl =
+        Environment.GetEnvironmentVariable("EPPO_BASE_URL") ?? "http://localhost:5000/api";
+    static readonly String apiToken =
+        Environment.GetEnvironmentVariable("EPPO_API_TOKEN") ?? "NO_TOKEN";
 
     public static void InitEppoClient()
     {
         Console.WriteLine("Initializating SDK pointed at" + eppoBaseUrl);
 
         var eppoClientConfig = new EppoClientConfig(apiToken, AssignmentLogger.Instance)
         {
-            BaseUrl = eppoBaseUrl
+            BaseUrl = eppoBaseUrl,
         };
 
         EppoClient.Init(eppoClientConfig);
     }
 
     public void ConfigureServices(IServiceCollection services)
     {
-
         Startup.InitEppoClient();
 
         services.AddControllers();
         services.AddSwaggerGen(c =>
         {
-            c.SwaggerDoc("v1", new OpenApiInfo
-            { Title = "Eppo SDK Relay Server", Version = "v1" });
+            c.SwaggerDoc("v1", new OpenApiInfo { Title = "Eppo SDK Relay Server", Version = "v1" });
         });
     }
 
-
     // Startup.cs
     public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
     {
@@ -54,7 +52,6 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
         app.UseEndpoints(endpoints =>
         {
             endpoints.MapControllers();
-
         });
     }
 }

package-testing/sdk-test-runner/README.md

@@ -199,6 +199,25 @@ The test runner sends assignment and bandit action requests to the SDK Relay Ser
 
 Any non-empty response
 
+##### SDK Details
+
+`POST /sdk/details`
+
+If possible, the SDK relay server should respond with the `sdkName` and `sdkVersion` in use. This may not be directly possible with all SDKs.
+If the SDK does not support Bandits or dynamic typing, the test runner will skip the related test cases if the corresponding values are `false`.
+
+`GET /sdk/details`
+
+```ts
+// Expected response data:
+type SDKDetailsResponse = {
+  sdkName?: string;
+  sdkVersion?: string;
+  supportsBandits?: boolean;
+  supportsDynamicTyping?: boolean;
+};
+```
+
 ##### Reset SDK
 
 `POST /sdk/reset`
@@ -225,13 +244,13 @@ type Assignment = {
   subjectAttributes: Record<string, object>;
 };
 
-// Expect response data:
-export type TestResponse {
-    result?: Object,          // Relayed `EppoClient` response
-    assignmentLog?: Object[], // Assignment log events (not yet tested)
-    banditLog?: Object[],     // Bandit selection log events (not yet tested)
-    error?: string            // Error encountered (not yet tested; automatically fails test when present)
-}
+// Expected response data:
+type TestResponse = {
+    result?: Object;          // Relayed `EppoClient` response
+    assignmentLog?: Object[]; // Assignment log events (not yet tested)
+    banditLog?: Object[];     // Bandit selection log events (not yet tested)
+    error?: string;           // Error encountered (not yet tested; automatically fails test when present)
+};
 ```
 
 ##### Bandits
@@ -264,12 +283,12 @@ export type BanditActionRequest = {
   };
 
 // Expects response data:
-export type TestResponse {
-    result?: Object,          // Relayed `EppoClient` response, form of {variation: string, action: string}
-    assignmentLog?: Object[], // Assignment log events (not yet tested)
-    banditLog?: Object[],     // Bandit selection log events (not yet tested)
-    error?: string            // Error encountered (not yet tested; automatically fails test when present)
-}
+type TestResponse = {
+    result?: Object;          // Relayed `EppoClient` response, form of {variation: string, action: string}
+    assignmentLog?: Object[]; // Assignment log events (not yet tested)
+    banditLog?: Object[];     // Bandit selection log events (not yet tested)
+    error?: string;           // Error encountered (not yet tested; automatically fails test when present)
+};
 ```
 
 ### build-and-run.sh

package-testing/sdk-test-runner/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sdk-test-runner",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Test runner for SDK package testing",
   "main": "src/app.ts",
   "repository": "https://github.com/Eppo-exp/sdk-test-data",

package-testing/sdk-test-runner/src/app.ts

@@ -138,7 +138,7 @@ export default class App {
       suites: testSuites,
     };
     const junitXml = getJunitXml(testSuiteReport);
-    fs.writeFileSync(junitFile, junitXml);
+    fs.writeFileSync('./' + junitFile, junitXml);
 
     log(green(`Test results written to ${junitFile}`));
   }
@@ -201,6 +201,14 @@ export default class App {
     for (const child of testCases) {
       const filePath = path.join(testCaseDir, child);
 
+      // Skip dynamic typing files if not supported
+      if (!sdkRelay.getSDKDetails().supportsDynamicTyping && isDynamicTypingFile(filePath)) {
+        logIndent(1, yellow('skipped') + ` ${child} SDK does not support dynamic typing`);
+        const testCaseResult: TestCase = { name: child, classname: child, skipped: true };
+        testCaseResults.push(testCaseResult);
+        continue;
+      }
+
       // Skip directories.
       if (!fs.statSync(filePath).isFile()) {
         continue;
@@ -218,6 +226,14 @@ export default class App {
       // Flag testing!!
       const isFlagTest = testCaseObj['variationType'];
 
+      // Skip bandit tests if not supported
+      if (!isFlagTest && !sdkRelay.getSDKDetails().supportsBandits) {
+        logIndent(1, yellow('skipped') + ' SDK does not support Bandits');
+        const testCaseResult: TestCase = { name: child, classname: child, skipped: true };
+        testCaseResults.push(testCaseResult);
+        continue;
+      }
+
       if (testCaseObj['subjects'].length === 0) {
         testCaseResults.push({ name: testCase, errors: [{ message: 'No test subjects found' }] });
       }
@@ -277,7 +293,7 @@ export default class App {
           .catch((error) => {
             if (error instanceof FeatureNotSupportedError) {
               // Skip this test
-              logIndent(1, yellow('skipped') + ` ${testCaseLabel}: SDK does not support this feature`);
+              logIndent(1, yellow('skipped') + ` ${testCaseLabel}: SDK does not support ${error.featureName}`);
               testCaseResult.skipped = true;
             } else {
               log(red('Error1:'), error);
@@ -299,3 +315,6 @@ export default class App {
     return isEqual(subject['assignment'], results['result']);
   }
 }
+function isDynamicTypingFile(filePath: string) {
+  return filePath.indexOf('dynamic-typing') >= 0;
+}

package-testing/sdk-test-runner/src/protocol/ClientSDKRelay.ts

@@ -21,7 +21,13 @@ export class ClientSDKRelay implements SDKRelay {
   private isReadyPromise: Promise<SDKInfo>;
   private ready = false;
   private socket?: Socket;
-  private sdkInfo?: SDKInfo;
+
+  // Defaults for client SDKs
+  private sdkInfo: SDKInfo = {
+    supportsBandits: false,
+    supportsDynamicTyping: true,
+    sdkName: 'UNKNOWN',
+  };
 
   constructor(testrunnerPort: number = 3000) {
     // Run socketIO through a node http server.
@@ -41,7 +47,7 @@ export class ClientSDKRelay implements SDKRelay {
         socket.on('READY', (msg, ack) => {
           const msgObj = JSON.parse(msg);
 
-          this.sdkInfo = msgObj as SDKInfo;
+          this.sdkInfo = { ...this.sdkInfo, ...(msgObj as SDKInfo) };
 
           log(green(`Client ${this.sdkInfo?.sdkName} reports ready`));
 
@@ -60,6 +66,12 @@ export class ClientSDKRelay implements SDKRelay {
       httpServer.listen(testrunnerPort);
     });
   }
+  getSDKDetails(): SDKInfo {
+    if (this.sdkInfo == null) {
+      throw new Error('SDK Client is not connected');
+    }
+    return this.sdkInfo;
+  }
 
   reset(): Promise<void> {
     if (!this.ready) {
@@ -79,7 +91,7 @@ export class ClientSDKRelay implements SDKRelay {
       throw new Error('SDK Client is not connected');
     }
     if (!this.sdkInfo?.supportsBandits) {
-      throw new FeatureNotSupportedError('Bandits are not supported in this SDK');
+      throw new FeatureNotSupportedError('Bandits are not supported in this SDK', 'Bandits');
     }
 
     const result = await new Promise((resolve) => {

package-testing/sdk-test-runner/src/protocol/FeatureNotSupportedError.ts

@@ -1,5 +1,8 @@
 export class FeatureNotSupportedError extends Error {
-  constructor(message: string) {
+  constructor(
+    message: string,
+    public readonly featureName: string,
+  ) {
     super(message);
     this.name = 'FeatureNotSupportedError';
   }

package-testing/sdk-test-runner/src/protocol/SDKInfo.ts

@@ -1,4 +1,6 @@
 export type SDKInfo = {
   sdkName: string;
+  sdkVersion?: string;
   supportsBandits: boolean;
+  supportsDynamicTyping: boolean;
 };