Updated the webhook to record test cases

Author: Fgerthoffert

src/config.service.ts

@@ -8,6 +8,15 @@ import * as path from 'path';
 
 import { zencrepesConfig, defaultConfig } from '@bit/zencrepes.zindexer.config';
 
+// Since bit is not working anymore, adding this extra interface manually
+interface zencrepesConfigAppend {
+  elasticsearch: {
+    dataIndices: {
+      testingCases: string;
+    }
+  }
+}
+
 export interface EnvConf {
   [key: string]: string;
 }
@@ -19,7 +28,7 @@ export class ConfigService {
   APP_VERSION: string;
 
   private readonly envConfig: EnvConf;
-  private userConfig: zencrepesConfig;
+  private userConfig: zencrepesConfig & zencrepesConfigAppend;
 
   constructor() {
     // Initialize config directory:
@@ -74,11 +83,11 @@ export class ConfigService {
     return this.envConfig[key];
   }
 
-  getUserConfig(): zencrepesConfig {
+  getUserConfig(): zencrepesConfig & zencrepesConfigAppend {
     return this.userConfig;
   }
 
-  setUserConfig(userConfig: zencrepesConfig) {
+  setUserConfig(userConfig: zencrepesConfig & zencrepesConfigAppend) {
     this.userConfig = userConfig;
   }
 }

src/testing/cases/esMapping.ts

@@ -0,0 +1,63 @@
+const yaml = `
+#https://www.elastic.co/guide/en/elasticsearch/plugins/current/mapper-size-usage.html
+_source:
+  enabled: true
+properties:
+  id:
+    type: keyword
+  name:
+    type: text
+    fields:
+      keyword:
+        type: keyword
+        ignore_above: 256
+  suite:
+    type: text
+    fields:
+      keyword:
+        type: keyword
+        ignore_above: 256
+  project:
+    type: text
+    fields:
+      keyword:
+        type: keyword
+        ignore_above: 256
+  jahia:
+    type: text
+    fields:
+      keyword:
+        type: keyword
+        ignore_above: 256
+  module:
+    type: text
+    fields:
+      keyword:
+        type: keyword
+        ignore_above: 256
+  full:
+    type: text
+    fields:
+      keyword:
+        type: keyword
+        ignore_above: 256
+  createdAt:
+    type: date
+  state:
+    type: keyword
+  caseTotal:
+    type: integer
+  caseSuccess:
+    type: integer 
+  caseSuccessRate:
+    type: integer
+  caseFailure:
+    type: integer
+  caseFailureRate:
+    type: integer
+  duration:
+    type: keyword    
+  url:
+    type: keyword
+`;
+export default yaml;

src/testing/cases/esSettings.ts

@@ -0,0 +1,27 @@
+const yaml = `
+analysis:
+  analyzer:
+    autocomplete_analyzed:
+      filter:
+        - lowercase
+        - edge_ngram
+      tokenizer: standard
+    autocomplete_prefix:
+      filter:
+        - lowercase
+        - edge_ngram
+      tokenizer: keyword
+    lowercase_keyword:
+      filter:
+        - lowercase
+      tokenizer: keyword
+  filter:
+    edge_ngram:
+      max_gram: '20'
+      min_gram: '1'
+      side: front
+      type: edge_ngram
+index.mapping.nested_fields.limit: 100
+number_of_shards: 1
+`;
+export default yaml;

src/testing/cases/index.ts

@@ -0,0 +1,4 @@
+export { default as esMapping } from './esMapping';
+export { default as esSettings } from './esSettings';
+export { getId, getCaseId } from './utils';
+export { CaseNode, Dependency } from './node.type'

src/testing/cases/node.type.ts

@@ -0,0 +1,22 @@
+
+export interface Dependency {
+  id: string;
+  name: string;
+  version: string;
+  full: string;
+  url?: string;
+}
+
+export interface CaseNode {
+  id: string;
+  name: string;
+  suite: string;
+  project: string;
+  version: string;
+  full: string;
+  dependencies: Dependency[]
+  createdAt: string;
+  state: string;
+  duration: number;
+  url: string;
+}

src/testing/cases/utils.ts

@@ -0,0 +1,28 @@
+import {v5 as uuidv5} from 'uuid'
+import {CaseNode} from './node.type'
+
+interface Dependency {
+    name: string;
+    version: string;
+}
+
+const prepString = (s: string) => {
+  return s.replace(/[^0-9a-zA-Z]/g, '').toLowerCase()
+}
+
+// We want the run ID to be unique as we want to record all runs
+export const getCaseId = (testCase: CaseNode) => {
+  const idStr = prepString(testCase.id) + prepString(testCase.createdAt)
+  const UUID_NAMESPACE = 'c72d8f12-1818-4cb9-bead-44634c441c11'
+  return uuidv5(idStr, UUID_NAMESPACE)
+}
+
+
+// This generate an unique id based on the combination the component and its dependencies
+// The ID is simply a UUID genreated from the concatenation of all elements
+// Note that the dependencies are sorted and all string are cleaned (lower case and stripped from non alphanumerical characters)
+export const getId = (dependency: Dependency) => {
+  const idStr = prepString(dependency.name) + prepString(dependency.version)
+  const UUID_NAMESPACE = 'c72d8f12-1818-4cb9-bead-44634c441c11'
+  return uuidv5(idStr, UUID_NAMESPACE)
+}

src/testing/testingStorePayload.processor.ts

@@ -4,6 +4,7 @@ import { Job } from 'bull';
 
 import { esMapping, esSettings, StateNode, getId } from '@bit/zencrepes.zindexer.testing-states';
 import { esMapping as esMappingRuns, esSettings as esSettingsRuns, RunNode, getRunId} from '@bit/zencrepes.zindexer.testing-runs';
+import { esMapping as esMappingCases, esSettings as esSettingsCases, CaseNode, getCaseId} from './cases';
 import { checkEsIndex, pushEsNodes } from '@bit/zencrepes.zindexer.es-utils';
 
 import { ConfigService } from '../config.service';
@@ -31,7 +32,7 @@ export class TestingStorePayloadProcessor {
     await checkEsIndex(esClient, userConfig.elasticsearch.dataIndices.testingStates, esMapping, esSettings, console.log);
 
     // Transforming the object, objective is just to match to GitHub's to be consistent with the rest of the app
-    const state: StateNode = {
+    let state: StateNode = {
       ...job.data,
       full: job.data.name + '_' + job.data.version,
       dependencies: {
@@ -48,6 +49,12 @@ export class TestingStorePayloadProcessor {
       }
     }
 
+    // Cases is a more recent addition, it should not be present in the state, not to pollute the index
+    // with unnecessary data
+    if ((state as any).cases !== undefined) {
+      delete (state as any).cases;
+    }
+
     // Push single document to Elasticsearch
     await pushEsNodes(esClient, userConfig.elasticsearch.dataIndices.testingStates, [state], console.log);
 
@@ -58,7 +65,7 @@ export class TestingStorePayloadProcessor {
     await checkEsIndex(esClient, userConfig.elasticsearch.dataIndices.testingRuns, esMappingRuns, esSettingsRuns, console.log);
 
     // Transforming the object, objective is just to match to GitHub's to be consistent with the rest of the app
-    const run: RunNode = {
+    let run: RunNode = {
       ...job.data,
       id: getRunId(job.data),
       full: job.data.name + '_' + job.data.version,
@@ -78,8 +85,39 @@ export class TestingStorePayloadProcessor {
       }
     }
 
+    // Cases is a more recent addition, it should not be present in the state, not to pollute the index
+    // with unnecessary data
+    if ((run as any).cases !== undefined) {
+      delete (run as any).cases;
+    }
+
     // Push single document to Elasticsearch
     await pushEsNodes(esClient, userConfig.elasticsearch.dataIndices.testingRuns, [run], console.log);
 
+    if (job.data.cases !== undefined && job.data.cases.length > 0) {
+      // Check if the index exists, create it if it does not    
+      await checkEsIndex(esClient, userConfig.elasticsearch.dataIndices.testingCases, esMappingCases, esSettingsCases, console.log);    
+
+      // Populating testing Cases
+      this.logger.log(`Event for: ${job.data.name}, version: ${job.data.version} - Pushing ${job.data.cases.length} Cases`);
+
+      const cases: CaseNode[] = job.data.cases.map((c) => {
+        const caseObj = {
+          ...c,
+          id: getCaseId(c),
+          full: c.suite + ' - ' + c.name,
+          runId: run.id,
+          url: run.url,
+          caseSuccessRate: Math.round(c.caseSuccess * 100 / c.caseTotal),
+          caseFailureRate: Math.round(c.caseFailure * 100 / c.caseTotal),
+          project: run.name
+        }
+        return caseObj
+      })
+
+      // Push single document to Elasticsearch
+      await pushEsNodes(esClient, userConfig.elasticsearch.dataIndices.testingCases, cases, console.log);
+
+    }
   }
 }