Validate BDD models in tests (#904)

* convert params to models in bdd tests

Co-authored-by: ci.datadog-api-spec <[email protected]>
Co-authored-by: Thomas Hervé <[email protected]>

Author: 3 people

.generator/src/generator/cli.py

@@ -71,6 +71,10 @@ def cli(specs, output):
         "index.ts": env.get_template("index.j2"),
     }
 
+    test_scenarios_files = {
+        "scenarios_model_mapping.ts": env.get_template("scenarios_model_mapping.j2")
+    }
+
     all_specs = {}
     all_apis = {}
     for spec_path in specs:
@@ -127,3 +131,10 @@ def cli(specs, output):
         filename.parent.mkdir(parents=True, exist_ok=True)
         with filename.open("w+") as fp:
             fp.write(template.render(apis=all_apis))
+
+    # Parameter mappings for bdd tests
+    scenarios_test_output = pathlib.Path("../features/support/")
+    for name, template in test_scenarios_files.items():
+        filename = scenarios_test_output / name
+        with filename.open("w") as fp:
+            fp.write(template.render(all_apis=all_apis))

.generator/src/generator/templates/model/ObjectSerializer.j2

@@ -3,7 +3,7 @@
 import { {{ name }} } from "./{{ name }}";
 {%- endif %}
 {%- endfor %}
-import { UnparsedObject } from "../../datadog-api-client-common/util";
+import { dateFromRFC3339String, dateToRFC3339String, UnparsedObject } from "../../datadog-api-client-common/util";
 import { logger } from "../../../logger";
 
 const primitives = [
@@ -54,6 +54,8 @@ export class ObjectSerializer {
   public static serialize(data: any, type: string, format: string): any {
     if (data == undefined || type == "any") {
       return data;
+    } else if (data instanceof UnparsedObject) {
+      return data._data;
     } else if (primitives.includes(type.toLowerCase()) && typeof data == type.toLowerCase()) {
       return data;
     } else if (type.startsWith(ARRAY_PREFIX)) {
@@ -87,12 +89,8 @@ export class ObjectSerializer {
       if ("string" == typeof data) {
         return data;
       }
-      if (format == "date") {
-        let month = data.getMonth() + 1
-        month = month < 10 ? "0" + month.toString() : month.toString()
-        let day = data.getDate();
-        day = day < 10 ? "0" + day.toString() : day.toString();
-        return data.getFullYear() + "-" + month + "-" + day;
+      if (format == "date" || format == "date-time") {
+        return dateToRFC3339String(data)
       } else {
         return data.toISOString();
       }
@@ -129,21 +127,6 @@ export class ObjectSerializer {
       const attributesMap = typeMap[type].getAttributeTypeMap();
       const instance: {[index: string]: any} = {};
 
-      const extraAttributes = Object.keys(data)
-          .filter((key) => !Object.prototype.hasOwnProperty.call(attributesMap, key))
-          .reduce((obj, key) => {
-            return Object.assign(obj, {
-              [key]: data[key]
-            });
-          }, {});
-
-      if (Object.keys(extraAttributes).length !== 0) {
-        if (!data.additionalProperties) {
-          data.additionalProperties = {};
-        }
-        Object.assign(data.additionalProperties, extraAttributes);
-      }
-
       for (const attributeName in attributesMap) {
         const attributeObj = attributesMap[attributeName];
         if (attributeName == "additionalProperties") {
@@ -163,11 +146,8 @@ export class ObjectSerializer {
         if (attributeObj?.required && instance[attributeObj.baseName] === undefined) {
           throw new Error(`missing required property '${attributeObj.baseName}'`);
         }
-
-        if (enumsMap[attributeObj.type] && !enumsMap[attributeObj.type].includes(instance[attributeObj.baseName])) {
-          instance.unparsedObject = instance[attributeObj.baseName];
-        }
       }
+
       return instance;
     }
   }
@@ -206,18 +186,20 @@ export class ObjectSerializer {
       }
       return transformedData;
     } else if (type === "Date") {
-      return new Date(data);
+      return dateFromRFC3339String(data)
     } else {
       if (enumsMap[type]) {
-         return data;
+        if (enumsMap[type].includes(data)) {
+          return data;
+        }
+        return new UnparsedObject(data)
       }
-
       if (oneOfMap[type]) {
         const oneOfs: any[] = [];
         for (const oneOf of oneOfMap[type]) {
           try {
             const d = ObjectSerializer.deserialize(data, oneOf, format);
-            if (d?.unparsedObject === undefined) {
+            if (!d?._unparsed) {
               oneOfs.push(d);
             }
           } catch (e) {
@@ -237,18 +219,49 @@ export class ObjectSerializer {
 
       const instance = new typeMap[type]();
       const attributesMap = typeMap[type].getAttributeTypeMap();
+      let extraAttributes: any = []
+      if ("additionalProperties" in attributesMap) {
+        const attributesBaseNames = Object.keys(attributesMap).reduce((o, key) => Object.assign(o, {[attributesMap[key].baseName]: ""}), {});
+        extraAttributes = Object.keys(data).filter((key) => !Object.prototype.hasOwnProperty.call(attributesBaseNames, key))
+      }
 
       for (const attributeName in attributesMap) {
         const attributeObj = attributesMap[attributeName];
+        if (attributeName == "additionalProperties") {
+          if (extraAttributes.length > 0) {
+            if (!instance.additionalProperties) {
+              instance.additionalProperties = {};
+            }
+
+            for (const key in extraAttributes) {
+              instance.additionalProperties[extraAttributes[key]] = ObjectSerializer.deserialize(
+                data[extraAttributes[key]],
+                attributeObj.type,
+                attributeObj.format
+              );
+            }
+          }
+          continue;
+        }
+
         instance[attributeName] = ObjectSerializer.deserialize(data[attributeObj.baseName], attributeObj.type, attributeObj.format);
+
         // check for required properties
         if (attributeObj?.required && instance[attributeName] === undefined) {
           throw new Error(`missing required property '${attributeName}'`);
         }
 
-        // check for enum values
-        if (enumsMap[attributeObj.type] && !enumsMap[attributeObj.type].includes(instance[attributeName])) {
-          instance.unparsedObject = instance[attributeName];
+        if (instance[attributeName] instanceof UnparsedObject || instance[attributeName]?._unparsed) {
+          instance._unparsed = true
+        }
+
+        if (Array.isArray(instance[attributeName])) {
+          for (const d of instance[attributeName]) {
+            if (d instanceof UnparsedObject || d?._unparsed) {
+              instance._unparsed = true
+              break
+            }
+          }
         }
       }
 

.generator/src/generator/templates/model/model.j2

@@ -8,8 +8,12 @@ import { {{ classname }} } from "./{{ classname }}";
 
 import { HttpFile } from "../../datadog-api-client-common/http/http";
 
-{% if "enum" not in model -%}
-import { AttributeTypeMap{%- if "oneOf" in model %}, UnparsedObject{% endif %} } from "../../datadog-api-client-common/util";
+{% if "enum" not in model and "oneOf" not in model -%}
+import { AttributeTypeMap } from "../../datadog-api-client-common/util";
+{%- endif %}
+
+{% if "enum" in model or "oneOf" in model -%}
+import { UnparsedObject } from "../../datadog-api-client-common/util";
 {%- endif %}
 
 {% if "description" in model %}
@@ -42,7 +46,7 @@ export class {{ name }} {
   /**
    * @ignore
    */
-  "unparsedObject"?:any;
+  "_unparsed"?: boolean;
   {%- if "items" not in model %}
 
   /**
@@ -116,7 +120,7 @@ export class {{ name }} {
 
 
 {%- if "enum" in model %}
-export type {{ name }} = {% for index, value in enumerate(model.enum) %}typeof {{ model["x-enum-varnames"][index] or value.upper() }} {%- if not loop.last %}| {% endif %}{%- endfor %};
+export type {{ name }} = {% for index, value in enumerate(model.enum) %}typeof {{ model["x-enum-varnames"][index] or value.upper() }} {%- if not loop.last %}| {% endif %}{%- if loop.last %} | UnparsedObject{%- endif %}{%- endfor %};
 {%- for index, value in enumerate(model.enum) %}
 export const {{ model["x-enum-varnames"][index] or value.upper() }} = {{ value|format_value }};
 {%- endfor %}

.generator/src/generator/templates/scenarios_model_mapping.j2

@@ -0,0 +1,18 @@
+export const ScenariosModelMappings: {[key: string]: {[key: string]: any}} = {
+{%- for version, apis in all_apis.items() %}
+{%- for _, operations in apis.items() %}
+{%- for path, method, operation in operations %}
+{%- set operationParams = operation|parameters|list %}
+    "{{ version }}.{{ operation['operationId'] }}": {
+        {%- for name, parameter in operation|parameters %}
+        "{{ name|attribute_name }}": {
+            "type": "{{ get_type_for_parameter(parameter) }}",
+            "format": "{{ get_format_for_schema(parameter) }}",
+            },
+        {%- endfor %}
+        "operationResponseType": "{{ operation|return_type }}",
+    },
+{%- endfor %}
+{%- endfor %}
+{%- endfor %}
+}

.generator/src/generator/templates/util.j2

@@ -1,8 +1,8 @@
 
 export class UnparsedObject {
-  unparsedObject:any;
+  _data:any;
   constructor(data:any) {
-    this.unparsedObject = data;
+    this._data = data;
   }
 }
 
@@ -18,3 +18,76 @@ export type AttributeTypeMap = {
 export const isBrowser: boolean = typeof window !== "undefined" && typeof window.document !== "undefined";
 
 export const isNode: boolean = typeof process !== "undefined" && process.release && process.release.name === 'node';
+
+export class DDate extends Date {
+  rfc3339TzOffset: string | undefined;
+}
+
+const RFC3339Re: RegExp = /^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2}):(\d{2})\.?(\d+)?(?:(?:([+-]\d{2}):?(\d{2}))|Z)?$/;
+export function dateFromRFC3339String(date: string): DDate {
+  const m = RFC3339Re.exec(date);
+  if (m) {
+    const _date = new DDate(date)
+    if( m[8] === undefined && m[9] === undefined){
+      _date.rfc3339TzOffset = 'Z'
+    } else {
+      _date.rfc3339TzOffset = `${m[8]}:${m[9]}`
+    }
+
+    return _date
+  } else {
+    throw new Error('unexpected date format: ' + date)
+  }
+}
+
+export function dateToRFC3339String(date: Date | DDate): string {
+  const offSetArr = getRFC3339TimezoneOffset(date).split(":")
+  const tzHour = offSetArr.length == 1 ? 0 : +offSetArr[0];
+  const tzMin  = offSetArr.length == 1 ? 0 : +offSetArr[1];
+
+  const year = date.getFullYear() ;
+  const month = date.getMonth();
+  const day = date.getUTCDate();
+  const hour = date.getUTCHours() + +tzHour;
+  const minute = date.getUTCMinutes() + +tzMin;
+  const second = date.getUTCSeconds();
+
+  let msec = date.getUTCMilliseconds().toString();
+  msec = +msec === 0 ? "" : `.${pad(+msec, 3)}`
+
+  return year + "-" +
+    pad(month + 1) + "-" +
+    pad(day) + "T" +
+    pad(hour) + ":" +
+    pad(minute) + ":" +
+    pad(second) +
+    msec +
+    offSetArr.join(":");
+}
+
+// Helpers
+function pad(num: number, len: number = 2): string {
+  let paddedNum = num.toString()
+  if (paddedNum.length < len) {
+    paddedNum = "0".repeat(len - paddedNum.length) + paddedNum
+  } else if (paddedNum.length > len) {
+    paddedNum = paddedNum.slice(0, len)
+  }
+
+  return paddedNum
+}
+
+function getRFC3339TimezoneOffset(date: Date | DDate): string {
+  if (date instanceof DDate && date.rfc3339TzOffset) {
+    return date.rfc3339TzOffset;
+  }
+
+  let offset = date.getTimezoneOffset()
+  if (offset === 0) {
+      return "Z";
+  }
+
+  const offsetSign = (offset > 0) ? "+" : "-";
+  offset = Math.abs(offset);
+  return offsetSign + pad(Math.floor(offset / 60)) + ":" + pad(offset % 60);
+}