IPFIX HTTP exporter: provide an option to add custom header fields to the HTTP push requests

Author: obarash-te

scripts/automation/trex_control_plane/interactive/trex/emu/emu_plugins/emu_plugin_ipfix.py

@@ -516,6 +516,7 @@ def ipfix_push(self, line):
                                         parsing_opts.IPFIX_HTTP_REPEATS_WAIT_TIME,
                                         parsing_opts.IPFIX_HTTP_SITES_PER_TENANT,
                                         parsing_opts.IPFIX_HTTP_DEVICES_PER_SITE,
+                                        parsing_opts.IPFIX_HTTP_HEADER_FIELDS_JSON_FILE,
                                         parsing_opts.IPFIX_UDP_PACKETS_WAIT_TIME,
                                         )
         opts = parser.parse_args(line.split())
@@ -524,6 +525,15 @@ def ipfix_push(self, line):
         if exporter_params is None:
             raise TRexError("Failed to create ipfix exporter")
 
+        header_fields_json = None
+
+        if opts.header_fields_json_file:
+            try:
+                with open(opts.header_fields_json_file, 'r') as file:
+                    header_fields_json = file.read()
+            except FileNotFoundError:
+                raise TRexError("Header fields JSON file doesn't exist")
+
         exporter_params.set_export_from_dir(True)
         if exporter_params.get_type() == "http":
             exporter_params.set_repeats_num(opts.http_repeats_num)
@@ -532,6 +542,7 @@ def ipfix_push(self, line):
                                                        dir_scans_num = opts.dir_scans_num,
                                                        files_wait_time = opts.files_wait_time,
                                                        files_wait_time_speedup = opts.files_wait_time_speedup)
+            exporter_params.set_header_fields(header_fields_json)
 
         if exporter_params.get_type() == "udp":
             exporter_params.set_export_from_dir_params(dir = opts.dir,

scripts/automation/trex_control_plane/interactive/trex/emu/trex_emu_ipfix_profile.py

@@ -154,16 +154,17 @@ def _create_file_obj(self, dst_url):
         return obj
 
     def _create_http_obj(self, dst_url):
-        re_pattern = r"^/api/ipfixfilecollector/([%\w-]+)/([%\w-]+)/([%\w-]+)/post_file$"
+        re_pattern = r"([/\w]+)/([%\w-]+)/([%\w-]+)/([%\w-]+)/post_file$"
         match = re.search(re_pattern, dst_url.path)
         if not match:
             return None
 
-        tenant_id = match.groups()[0]
-        site_id = match.groups()[1]
-        device_id = match.groups()[2]
+        dst_url_path = match.groups()[0]
+        tenant_id = match.groups()[1]
+        site_id = match.groups()[2]
+        device_id = match.groups()[3]
 
-        obj = IpfixHttpExporterParams(dst_url.scheme, dst_url.hostname, dst_url.port, tenant_id, site_id, device_id)
+        obj = IpfixHttpExporterParams(dst_url.scheme, dst_url.hostname, dst_url.port, tenant_id, site_id, device_id, dst_url_path)
         return obj
 
 
@@ -279,13 +280,14 @@ def set_compress(self, compress):
 
 
 class IpfixHttpExporterParams(IpfixFileExporterParams):
-    def __init__(self, scheme, dst_ip_addr, dst_port, tenant_id, site_id, device_id):
+    def __init__(self, scheme, dst_ip_addr, dst_port, tenant_id, site_id, device_id, dst_url_path):
         super(IpfixHttpExporterParams, self).__init__(None, None)
         self._dst_ip_addr = dst_ip_addr
         self._dst_port = dst_port   # urlparse can return dst_port None
         self._tenant_id = tenant_id
         self._site_id = site_id
         self._device_id = device_id
+        self._dst_url_path = dst_url_path
         self._max_posts = None
         self._tls_cert_file = None
         self._tls_key_file = None
@@ -295,6 +297,7 @@ def __init__(self, scheme, dst_ip_addr, dst_port, tenant_id, site_id, device_id)
         self._repeats_wait_time = None
         self._export_from_dir = None
         self._export_from_dir_params = None
+        self._header_fields = None
         if scheme not in ["http", "https"]:
             raise ValueError("invalid scheme {}, scheme should be http/s".format(scheme))
         self._type = scheme
@@ -313,16 +316,22 @@ def get_json(self):
         add_to_json_if_not_none(exporter_params, "repeats_wait_time", self._repeats_wait_time)
         add_to_json_if_not_none(exporter_params, "export_from_dir", self._export_from_dir)
         add_to_json_if_not_none(exporter_params, "export_from_dir_params", self._export_from_dir_params)
+        add_to_json_if_not_none(exporter_params, "header_fields", self._header_fields)
 
         return None if len(exporter_params) == 0 else exporter_params
 
     def get_dst_url(self):
-        if self._dst_port is None:
-            return "{0}://{1}/api/ipfixfilecollector/{2}/{3}/{4}/post_file".format(
-                self._type, self._dst_ip_addr, self._tenant_id, self._site_id, self._device_id)
+        dst_url_path = "/api/ipfixfilecollector"
+        if self._dst_url_path:
+            dst_url_path = self._dst_url_path
+
+        if not self._dst_port:
+            return "{0}://{1}{2}/{3}/{4}/{5}/post_file".format(
+                self._type, self._dst_ip_addr, dst_url_path, self._tenant_id, self._site_id, self._device_id)
         else:
-            return "{0}://{1}:{2}/api/ipfixfilecollector/{3}/{4}/{5}/post_file".format(
-                self._type, self._dst_ip_addr, self._dst_port, self._tenant_id, self._site_id, self._device_id)
+            return "{0}://{1}:{2}{3}/{4}/{5}/{6}/post_file".format(
+                self._type, self._dst_ip_addr, self._dst_port, 
+                dst_url_path, self._tenant_id, self._site_id, self._device_id)
 
     def set_max_posts(self, max_posts):
         self._max_posts = max_posts
@@ -363,6 +372,15 @@ def set_export_from_dir_params(
 
         self._export_from_dir_params = json
 
+    def set_header_fields(self, header_fields_json):
+        # This function is used to add custom header fields to the post requests created by EMU.
+        # The JSON should have the following format:
+        # {
+        #     "header_1": "value 1", 
+        #     "header_2": "value_2"
+        # }
+        if header_fields_json:
+            self._header_fields = json.loads(header_fields_json)
 
 ###############################################################
                         # Plugin

scripts/automation/trex_control_plane/interactive/trex/utils/parsing_opts.py

@@ -1543,6 +1543,13 @@ class OPTIONS_DB_ARGS:
          'dest': "http_devices_per_site",
          'help': "Number of devices per site to be used in dst url"})
 
+    IPFIX_HTTP_HEADER_FIELDS_JSON_FILE = ArgumentPack(
+        ['--header-fields-json-file'],
+        {'type': str,
+         'default': None,
+         'dest': "header_fields_json_file",
+         'help': "JSON file defining custom HTTP header fields to add to the post requests created by EMU"})
+
     IPFIX_UDP_PACKETS_WAIT_TIME = ArgumentPack(
         ['--udp-packets-wait-time'],
         {'type': str,