chore(CTS): benchmark the search method (#3328)

Author: millotp

.github/workflows/check.yml

@@ -242,17 +242,17 @@ jobs:
           key: node-modules-tests-${{ hashFiles('tests/output/javascript/yarn.lock') }}
 
       - name: Run unit CTS
-        run: yarn cli cts run javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }} --exclude-e2e
+        run: yarn cli cts run javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }} --no-e2e
 
       - name: Run e2e CTS
         id: cts-e2e
         continue-on-error: true
         if: ${{ !github.event.pull_request.head.repo.fork }}
-        run: yarn cli cts run javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }} --exclude-unit
+        run: yarn cli cts run javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }} --no-client --no-requests
 
       - name: Retry e2e CTS
         if: ${{ !github.event.pull_request.head.repo.fork && steps.cts-e2e.outcome == 'failure' }}
-        run: yarn cli cts run javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }} --exclude-unit
+        run: yarn cli cts run javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }} --no-client --no-requests
 
       - name: Generate code snippets for documentation
         run: yarn cli snippets javascript ${{ fromJSON(needs.setup.outputs.JAVASCRIPT_DATA).toRun }}
@@ -336,17 +336,17 @@ jobs:
         run: yarn cli cts generate ${{ matrix.client.language }} ${{ matrix.client.toRun }}
 
       - name: Run unit CTS
-        run: yarn cli cts run ${{ matrix.client.language }} ${{ matrix.client.toRun }} --exclude-e2e
+        run: yarn cli cts run ${{ matrix.client.language }} ${{ matrix.client.toRun }} --no-e2e
 
       - name: Run e2e CTS
         id: cts-e2e
         continue-on-error: true
         if: ${{ !github.event.pull_request.head.repo.fork }}
-        run: yarn cli cts run ${{ matrix.client.language }} ${{ matrix.client.toRun }} --exclude-unit
+        run: yarn cli cts run ${{ matrix.client.language }} ${{ matrix.client.toRun }} --no-client --no-requests
 
       - name: Retry e2e CTS
         if: ${{ !github.event.pull_request.head.repo.fork && steps.cts-e2e.outcome == 'failure' }}
-        run: yarn cli cts run ${{ matrix.client.language }} ${{ matrix.client.toRun }} --exclude-unit
+        run: yarn cli cts run ${{ matrix.client.language }} ${{ matrix.client.toRun }} --no-client --no-requests
 
       - name: Generate code snippets for documentation
         run: yarn cli snippets ${{ matrix.client.language }} ${{ matrix.client.toRun }}
@@ -504,7 +504,7 @@ jobs:
             | 🪓 Triggered by | [`${{ github.sha }}`](${{ github.event.pull_request.base.repo.html_url }}/commit/${{ github.sha }}) |
             | 🍃 Generated commit | [`${{ steps.pushGeneratedCode.outputs.GENERATED_COMMIT }}`](${{ github.event.pull_request.base.repo.html_url }}/commit/${{ steps.pushGeneratedCode.outputs.GENERATED_COMMIT }}) |
             | 🌲 Generated branch | [`generated/${{ github.head_ref }}`](${{ github.event.pull_request.base.repo.html_url }}/tree/generated/${{ github.head_ref }}) |
-    
+
       - name: Build website
         if: ${{ github.ref == 'refs/heads/main' || github.base_ref == 'main' }}
         run: yarn website:build

generators/src/main/java/com/algolia/codegen/cts/AlgoliaCTSGenerator.java

@@ -61,7 +61,8 @@ public void processOpts() {
       ctsManager.addTestsSupportingFiles(supportingFiles);
 
       testsGenerators.add(new TestsRequest(language, client, false));
-      testsGenerators.add(new TestsClient(language, client));
+      testsGenerators.add(new TestsClient(language, client, true));
+      testsGenerators.add(new TestsClient(language, client, false));
     } else if (mode.equals("snippets")) {
       ctsManager.addSnippetsSupportingFiles(supportingFiles);
 
@@ -148,6 +149,8 @@ public Map<String, Object> postProcessSupportingFileData(Map<String, Object> obj
       if (hasRegionalHost) {
         bundle.put("defaultRegion", regionVariable.defaultValue);
       }
+      // special lambda for dynamic templates
+      bundle.put("dynamicTemplate", new DynamicTemplateLambda(this));
       bundle.put("lambda", lambda);
 
       ctsManager.addDataToBundle(bundle);
@@ -170,9 +173,6 @@ public Map<String, Object> postProcessSupportingFileData(Map<String, Object> obj
         }
       }
 
-      // special lambda for dynamic templates
-      bundle.put("dynamicTemplate", new DynamicTemplateLambda(this));
-
       // Generation notice, added on every generated files
       bundle.put("generationBanner", Helpers.setGenerationBanner(additionalProperties));
 

generators/src/main/java/com/algolia/codegen/cts/manager/PythonCTSManager.java

@@ -19,6 +19,7 @@ public void addTestsSupportingFiles(List<SupportingFile> supportingFiles) {
     supportingFiles.add(new SupportingFile("tests/__init__.mustache", "tests/output/python/tests/requests", "__init__.py"));
     supportingFiles.add(new SupportingFile("tests/__init__.mustache", "tests/output/python/tests/client", "__init__.py"));
     supportingFiles.add(new SupportingFile("tests/__init__.mustache", "tests/output/python/tests/e2e", "__init__.py"));
+    supportingFiles.add(new SupportingFile("tests/__init__.mustache", "tests/output/python/tests/benchmark", "__init__.py"));
   }
 
   @Override

generators/src/main/java/com/algolia/codegen/cts/tests/ClientTestData.java

@@ -14,6 +14,7 @@ class Step {
 
   public String type;
   public String method;
+  public int times;
   public Map<String, Object> parameters;
   public Expected expected;
 }

generators/src/main/java/com/algolia/codegen/cts/tests/TestsClient.java

@@ -16,8 +16,13 @@
 
 public class TestsClient extends TestsGenerator {
 
-  public TestsClient(String language, String client) {
+  private final boolean withBenchmark;
+  private final String testType;
+
+  public TestsClient(String language, String client, boolean withBenchmark) {
     super(language, client);
+    this.withBenchmark = withBenchmark;
+    this.testType = withBenchmark ? "benchmark" : "client";
   }
 
   @Override
@@ -27,8 +32,17 @@ public boolean available() {
       return false;
     }
 
-    File templates = new File("templates/" + language + "/tests/client/suite.mustache");
-    return templates.exists();
+    File templates = new File("templates/" + language + "/tests/client/" + testType + ".mustache");
+    if (!templates.exists()) {
+      return false;
+    }
+
+    if (withBenchmark) {
+      File benchmarkTest = new File("tests/CTS/benchmark/" + client + "/benchmark.json");
+      return benchmarkTest.exists();
+    }
+
+    return true;
   }
 
   @Override
@@ -38,15 +52,15 @@ public void addSupportingFiles(List<SupportingFile> supportingFiles, String outp
     }
     supportingFiles.add(
       new SupportingFile(
-        "tests/client/suite.mustache",
-        "tests/output/" + language + "/" + outputFolder + "/client",
+        "tests/client/" + testType + ".mustache",
+        "tests/output/" + language + "/" + outputFolder + "/" + testType,
         Helpers.createClientName(client, language) + extension
       )
     );
   }
 
   public void run(Map<String, CodegenModel> models, Map<String, CodegenOperation> operations, Map<String, Object> bundle) throws Exception {
-    Map<String, ClientTestData[]> cts = loadCTS("client", client, ClientTestData[].class);
+    Map<String, ClientTestData[]> cts = loadCTS(testType, client, ClientTestData[].class);
     ParametersWithDataType paramsType = new ParametersWithDataType(models, language, client);
 
     List<Object> blocks = new ArrayList<>();
@@ -62,17 +76,18 @@ public void run(Map<String, CodegenModel> models, Map<String, CodegenOperation>
           testOut.put("testName", test.testName);
           testOut.put("testIndex", testIndex++);
           testOut.put("autoCreateClient", test.autoCreateClient);
+          testOut.put("useEchoRequester", true);
+          testOut.put("isBenchmark", withBenchmark);
           for (Step step : test.steps) {
             Map<String, Object> stepOut = new HashMap<>();
-            stepOut.put("useEchoRequester", true);
+            if (step.times > 1) stepOut.put("times", step.times);
             CodegenOperation ope = null;
             if (step.type.equals("createClient")) {
               stepOut.put("stepTemplate", "tests/client/createClient.mustache");
               stepOut.put("isCreateClient", true); // TODO: remove once kotlin is converted
 
               boolean hasCustomHosts = step.parameters != null && step.parameters.containsKey("customHosts");
-
-              stepOut.put("useEchoRequester", !hasCustomHosts);
+              if (hasCustomHosts) testOut.put("useEchoRequester", false);
               stepOut.put("hasCustomHosts", hasCustomHosts);
               if (hasCustomHosts) {
                 stepOut.put("customHosts", step.parameters.get("customHosts"));
@@ -95,6 +110,10 @@ public void run(Map<String, CodegenModel> models, Map<String, CodegenOperation>
               stepOut.put("stepTemplate", "tests/client/method.mustache");
               stepOut.put("isMethod", true); // TODO: remove once kotlin is converted
               stepOut.put("hasParams", ope.hasParams);
+              stepOut.put("isGeneric", (boolean) ope.vendorExtensions.getOrDefault("x-is-generic", false));
+              if (ope.returnType != null && ope.returnType.length() > 0) {
+                stepOut.put("returnType", Helpers.toPascalCase(ope.returnType));
+              }
 
               // set on testOut because we need to wrap everything for java.
               testOut.put("isHelper", (boolean) ope.vendorExtensions.getOrDefault("x-helper", false));
@@ -181,6 +200,6 @@ public void run(Map<String, CodegenModel> models, Map<String, CodegenOperation>
       testObj.put("testType", blockEntry.getKey());
       blocks.add(testObj);
     }
-    bundle.put("blocksClient", blocks);
+    bundle.put(withBenchmark ? "blocksBenchmark" : "blocksClient", blocks);
   }
 }

generators/src/main/java/com/algolia/codegen/cts/tests/TestsGenerator.java

@@ -41,12 +41,11 @@ protected <T> Map<String, T> loadCTS(String path, String clientName, Class<T> js
     if (!dir.exists()) {
       throw new CTSException("CTS not found at " + dir.getAbsolutePath(), true);
     }
-    if (!commonTestDir.exists()) {
-      throw new CTSException("CTS not found at " + commonTestDir.getAbsolutePath(), true);
-    }
     List<File> allTests = new ArrayList<>();
     Collections.addAll(allTests, dir.listFiles());
-    Collections.addAll(allTests, commonTestDir.listFiles());
+    if (commonTestDir.exists()) {
+      Collections.addAll(allTests, commonTestDir.listFiles());
+    }
 
     Map<String, T> cts = new TreeMap<>();
 

scripts/ci/githubActions/createMatrix.ts

@@ -69,15 +69,15 @@ async function createClientMatrix(baseBranch: string): Promise<void> {
     const testsRootFolder = `tests/output/${language}`;
     const testsOutputBase = `${testsRootFolder}/${getTestOutputFolder(language)}`;
     // We delete tests to ensure the CI only run tests against what changed.
-    const testsToDelete = `${testsOutputBase}/client ${testsOutputBase}/requests ${testsOutputBase}/e2e`;
+    const testsToDelete = `${testsOutputBase}/client ${testsOutputBase}/requests ${testsOutputBase}/e2e ${testsOutputBase}/benchmark`;
 
     // We only store tests of clients that ran during this job, the rest stay as is
     let testsToStore = matrix[language].toRun
       .map((client) => {
         const clientName = createClientName(client, language);
         const extension = getTestExtension(language);
 
-        return `${testsOutputBase}/client/${clientName}${extension} ${testsOutputBase}/requests/${clientName}${extension} ${testsOutputBase}/e2e/${clientName}${extension}`;
+        return `${testsOutputBase}/client/${clientName}${extension} ${testsOutputBase}/requests/${clientName}${extension} ${testsOutputBase}/e2e/${clientName}${extension} ${testsOutputBase}/benchmark/${clientName}${extension}`;
       })
       .join(' ');
 

scripts/cli/index.ts

@@ -129,13 +129,15 @@ ctsCommand
   .addArgument(args.language)
   .addArgument(args.clients)
   .option(flags.verbose.flag, flags.verbose.description)
-  .option('-e, --exclude-e2e', "don't run the e2e tests, useful for offline testing")
-  .option('-u, --exclude-unit', "don't run the client and requests tests")
+  .option('-e, --no-e2e', 'run the e2e tests, that requires internet connection')
+  .option('-c, --no-client', 'run the client tests')
+  .option('-r, --no-requests', 'run the requests tests')
+  .option('-b, --benchmark', 'run the benchmarks')
   .action(
     async (
       langArg: LangArg,
       clientArg: string[],
-      { verbose, excludeE2e: excludeE2E, excludeUnit },
+      { verbose, e2e, client: includeClient, requests, benchmark },
     ) => {
       const { language, client } = transformSelection({
         langArg,
@@ -144,12 +146,12 @@ ctsCommand
 
       setVerbose(Boolean(verbose));
 
-      await runCts(
-        language === ALL ? LANGUAGES : [language],
-        client,
-        Boolean(excludeE2E),
-        Boolean(excludeUnit),
-      );
+      await runCts(language === ALL ? LANGUAGES : [language], client, {
+        client: includeClient,
+        requests,
+        e2e,
+        benchmark,
+      });
     },
   );
 

scripts/cts/runCts.ts

@@ -5,36 +5,40 @@ import { getTestOutputFolder } from '../config.js';
 import { createSpinner } from '../spinners.js';
 import type { Language } from '../types.js';
 
-import { startTestServer } from './testServer';
+import { startBenchmarkServer, startTestServer } from './testServer';
+import { printBenchmarkReport } from './testServer/benchmark.js';
 import { assertChunkWrapperValid } from './testServer/chunkWrapper.js';
 import { assertValidReplaceAllObjects } from './testServer/replaceAllObjects.js';
 import { assertValidTimeouts } from './testServer/timeout.js';
 import { assertValidWaitForApiKey } from './testServer/waitForApiKey.js';
 
-async function runCtsOne(
-  language: Language,
-  excludeE2E: boolean,
-  excludeUnit: boolean,
-): Promise<void> {
-  const cwd = `tests/output/${language}`;
+export type CTSType = 'benchmark' | 'client' | 'e2e' | 'requests';
 
-  const folders: string[] = [];
-  if (!excludeE2E) {
-    // check if the folder has files
-    const e2eFolder = toAbsolutePath(
-      `tests/output/${language}/${getTestOutputFolder(language)}/e2e`,
+async function buildFilter(
+  language: Language,
+  suites: Record<CTSType, boolean>,
+): Promise<CTSType[]> {
+  const folders: CTSType[] = [];
+  for (const [suite, include] of Object.entries(suites)) {
+    // check if the folder has files in it
+    const folder = toAbsolutePath(
+      `tests/output/${language}/${getTestOutputFolder(language)}/${suite}`,
     );
     if (
-      (await exists(e2eFolder)) &&
-      (await fsp.readdir(e2eFolder)).filter((f) => f !== '__init__.py' && f !== '__pycache__')
-        .length > 0
+      include &&
+      (await exists(folder)) &&
+      (await fsp.readdir(folder)).filter((f) => f !== '__init__.py' && f !== '__pycache__').length >
+        0
     )
-      folders.push('e2e');
-  }
-  if (!excludeUnit) {
-    folders.push('client', 'requests');
+      folders.push(suite as CTSType);
   }
 
+  return folders;
+}
+
+async function runCtsOne(language: Language, suites: Record<CTSType, boolean>): Promise<void> {
+  const cwd = `tests/output/${language}`;
+  const folders = await buildFilter(language, suites);
   const spinner = createSpinner(`running cts for '${language}' in folder(s) ${folders.join(', ')}`);
 
   if (folders.length === 0) {
@@ -81,7 +85,10 @@ async function runCtsOne(
       );
       break;
     case 'kotlin':
-      await run('./gradle/gradlew -p tests/output/kotlin allTests', { language });
+      await run(
+        `./gradle/gradlew -p tests/output/kotlin jvmTest ${filter((f) => `--tests 'com.algolia.${f}*'`)}`,
+        { language },
+      );
       break;
     case 'php':
       await runComposerInstall();
@@ -133,20 +140,28 @@ async function runCtsOne(
 export async function runCts(
   languages: Language[],
   clients: string[],
-  excludeE2E: boolean,
-  excludeUnit: boolean,
+  suites: Record<CTSType, boolean>,
 ): Promise<void> {
-  const useTestServer = !excludeUnit && (clients.includes('search') || clients.includes('all'));
-  let close: () => Promise<void> = async () => {};
+  const useTestServer = suites.client && (clients.includes('search') || clients.includes('all'));
+  const useBenchmarkServer =
+    suites.benchmark && (clients.includes('search') || clients.includes('all'));
+
+  let closeTestServer: () => Promise<void> = async () => {};
   if (useTestServer) {
-    close = await startTestServer();
+    closeTestServer = await startTestServer();
+  }
+
+  let closeBenchmarkServer: () => Promise<void> = async () => {};
+  if (useBenchmarkServer) {
+    closeBenchmarkServer = await startBenchmarkServer();
   }
+
   for (const lang of languages) {
-    await runCtsOne(lang, excludeE2E, excludeUnit);
+    await runCtsOne(lang, suites);
   }
 
   if (useTestServer) {
-    await close();
+    await closeTestServer();
 
     const skip = (lang: Language): number => (languages.includes(lang) ? 1 : 0);
 
@@ -155,4 +170,10 @@ export async function runCts(
     assertValidReplaceAllObjects(languages.length - skip('dart') - skip('scala'));
     assertValidWaitForApiKey(languages.length - skip('dart') - skip('scala'));
   }
+
+  if (useBenchmarkServer) {
+    await closeBenchmarkServer();
+
+    printBenchmarkReport();
+  }
 }