Add APIs to support configurable output (data provider)

- Add configuration service per output (data provider)
  This addition enables clients to create derived data providers from
  an existing data provider.
- Add unit tests
- Add cli to invoke new endpoints
- Update README.md for new commands

Signed-off-by: Bernd Hufmann <[email protected]>

Author: bhufmann

README.md

@@ -89,6 +89,12 @@ usage: tsp_cli_client [-h] [--ip IP] [--port PORT]
                       [--params PARAMS]
                       [--get-health]
                       [--get-identifier]
+                      [--list-output-configuration-sources OUTPUT_ID]
+                      [--list-output-configuration-source TYPE_ID]
+                      [--create-output OUTPUT_ID]
+                      [--delete-output DERIVED_OUTPUT_ID]
+                      [--output-id OUTPUT_ID]
+                      [--json-file JSON_FILE]
 
 CLI client to send Trace Server Protocol commands to a Trace Server.
 
@@ -160,6 +166,18 @@ optional arguments:
   --params PARAMS       comma separated key value pairs (key1=val1,key2=val2)
   --get-health          Get the health status of the server
   --get-identifier      Identify important information regarding the server and the system
+  --list-output-configuration-sources OUTPUT_ID
+                        Get available configuration sources for a given experiment and output
+  --list-output-configuration-source TYPE_ID
+                        Get configuration source for a given experiment, output and type
+  --create-output OUTPUT_ID
+                        Create derived output provided by --param or --json-file
+  --delete-output DERIVED_OUTPUT_ID
+                        Delete derived output
+  --output-id OUTPUT_ID
+                        The output ID
+  --json-file JSON_FILE
+                        JSON file with parameter
 ```
 
 Examples:
@@ -190,6 +208,10 @@ Examples:
   ./tsp_cli_client --load-configuration --type-id TYPE_ID --params key1:value1
   ./tsp_cli_client --update-configuration --type-id TYPE_ID --config-id CONFIG_ID --params key1=value1,key2=value2
   ./tsp_cli_client --delete-configuration CONFIGURATION_ID --type-id TYPE_ID
+  ./tsp_cli_client --list-output-configuration-sources OUTPUT_ID --uuid UUID
+  ./tsp_cli_client --list-output-configuration-source TYPE_ID --output-id OUTPUT_ID --uuid UUID
+  ./tsp_cli_client --create-output OUTPUT_ID --uuid UUID --json-file absolute-path-to-json-file
+  ./tsp_cli_client --delete-output DERIVED_OUTPUT_ID --output-id  OUTPUT_ID --uuid UUID
   ./tsp_cli_client --get-health
   ./tsp_cli_client --get-identifier
 ```

test_tsp.py

@@ -33,6 +33,9 @@
 from tsp.response import ResponseStatus
 from tsp.tsp_client import TspClient
 from tsp.virtual_table_tag import VirtualTableTag
+from tsp.configuration_source import ConfigurationSource
+from tsp.configuration_source_set import ConfigurationSourceSet
+from tsp.output_descriptor import OutputDescriptor
 
 STATISTICS_DP_ID = (
     "org.eclipse.tracecompass.analysis.timing.core.segmentstore.SegmentStoreStatisticsDataProvider:"
@@ -44,6 +47,8 @@
 )
 TIMEGRAPH_DP_ID = "org.eclipse.tracecompass.internal.analysis.os.linux.core.threadstatus.ThreadStatusDataProvider"
 
+INANDOUT_DP_ID = "org.eclipse.tracecompass.incubator.inandout.core.analysis.inAndOutdataProviderFactory"
+
 REQUESTED_TIME_START = 1332170682440133097
 REQUESTED_TIME_END = 1332170692664579801
 REQUESTED_TIME_LENGTH = 10
@@ -53,6 +58,8 @@
 
 CONFIG_SOURCE_TYPE = 'org.eclipse.tracecompass.tmf.core.config.xmlsourcetype'
 
+INANDOUT_CONFIG_SOURCE_TYPE = 'org.eclipse.tracecompass.incubator.internal.inandout.core.config'
+
 # pylint: disable=too-many-public-methods
 
 
@@ -693,6 +700,81 @@ def test_put_configuration(self, extension):
 
         self.tsp_client.delete_configuration(CONFIG_SOURCE_TYPE, self.name)
 
+    def test_fetch_output_configuration_sources(self, kernel):
+        """Expect at least one configuration source ."""
+
+        traces = []
+        response = self.tsp_client.open_trace(os.path.basename(kernel), kernel)
+        traces.append(response.model.UUID)
+        response = self.tsp_client.open_experiment(
+            os.path.basename(kernel), traces)
+        assert response.status_code == 200
+        experiment_uuid = response.model.UUID
+
+        response = self.tsp_client.fetch_output_configuration_sources(experiment_uuid, INANDOUT_DP_ID)
+        assert response.status_code == 200
+        assert isinstance(response.model, ConfigurationSourceSet)
+        assert response.model.configuration_source_set
+        assert len(response.model.configuration_source_set) > 0
+
+        response = self.tsp_client.fetch_output_configuration_source(experiment_uuid, INANDOUT_DP_ID, INANDOUT_CONFIG_SOURCE_TYPE)
+        assert response.status_code == 200
+        assert response.model
+        assert isinstance(response.model, ConfigurationSource)
+
+        assert response.model.schema != None
+
+    def test_create_delete_derived_output(self, kernel):
+        """Expect a data provider descriptor after creating it."""
+
+        traces = []
+        response = self.tsp_client.open_trace(os.path.basename(kernel), kernel)
+        traces.append(response.model.UUID)
+        response = self.tsp_client.open_experiment(
+            os.path.basename(kernel), traces)
+        assert response.status_code == 200
+        experiment_uuid = response.model.UUID
+
+        params = {
+            "name": "My new InAndOut",
+            "description": "My special configuration",
+            "sourceTypeId": "org.eclipse.tracecompass.incubator.internal.inandout.core.config",
+            "parameters": {
+                "specifiers": [{
+                    "label": "latency",
+                    "inRegex": "(\\S*)_entry",
+                    "outRegex": "(\\S*)_exit",
+                    "contextInRegex": "(\\S*)_entry",
+                    "contextOutRegex": "(\\S*)_exit",
+                    "classifier": "CPU"
+                }]
+            }
+        }
+        
+        response = self.tsp_client.create_derived_output(experiment_uuid, INANDOUT_DP_ID, params)
+        assert response.status_code == 200
+        assert response.model
+        assert isinstance(response.model, OutputDescriptor)
+        assert response.model.parent_id == INANDOUT_DP_ID
+
+        derived_id = response.model.id
+
+        response = self.tsp_client.fetch_experiment_outputs(experiment_uuid)
+        assert response.status_code == 200
+        assert response.model
+        assert len(response.model.descriptors) > 0
+        assert [desc for desc in response.model.descriptors if desc.id == derived_id]
+
+        response = self.tsp_client.fetch_experiment_output(
+            experiment_uuid, derived_id)
+        assert response.status_code == 200
+        assert response.model.id == derived_id
+
+        response = self.tsp_client.delete_derived_output(experiment_uuid, INANDOUT_DP_ID, derived_id)
+        assert response.status_code == 200
+        assert response.model
+        assert isinstance(response.model, OutputDescriptor)
+
     def test_fetch_health(self):
         """Expect a successful health response"""
         response = self.tsp_client.fetch_health()

tsp/tsp_client.py

@@ -49,6 +49,7 @@
 GET_TREE_FAILED = "failed to get tree: {0}"
 GET_STATES_FAILED = "failed to get states: {0}"
 GET_ARROWS_FAILED = "failed to get arrows: {0}"
+GET_CONFIG_SOURCE_TYPES = "failed to get config source type(s): {} {}"
 
 
 # pylint: disable=consider-using-f-string,missing-timeout
@@ -474,6 +475,86 @@ def fetch_xy(self, exp_uuid, output_id, parameters):
             print("failed to get xy: {0}".format(response.status_code))
             return TspClientResponse(None, response.status_code, response.text)
 
+
+    def fetch_output_configuration_sources(self, exp_uuid, output_id):
+        '''
+        Fetch all configuration source types for a given experiment and output
+        :param exp_uuid Experiment UUID
+        :param output_id Output ID
+        :returns :class:`TspClientResponse <ConfigurationSourceSet>` object
+        :rtype: TspClientResponse
+        '''
+        api_url = '{0}experiments/{1}/outputs/{2}/configTypes'.format(
+            self.base_url, exp_uuid, output_id)
+        response = requests.get(api_url, headers=headers)
+        if response.status_code == 200:
+            return TspClientResponse(ConfigurationSourceSet(json.loads(response.content.decode('utf-8'))),
+                                     response.status_code, response.text)
+        else:  # pragma: no cover
+            print(GET_CONFIG_SOURCE_TYPES.format(response.status_code, response.text))
+            return TspClientResponse(None, response.status_code, response.text)
+
+    def fetch_output_configuration_source(self, exp_uuid, output_id, type_id):
+        '''
+        Fetch a single configuration source type for a given experiment, output and type
+        :param exp_uuid Experiment UUID
+        :param output_id Output ID
+        :param type_id the ID of the configuration source type
+        :returns :class:`TspClientResponse <ConfigurationSource>` object
+        :rtype: TspClientResponse
+        '''
+        api_url = '{0}experiments/{1}/outputs/{2}/configTypes/{3}'.format(
+            self.base_url, exp_uuid, output_id, type_id)
+        response = requests.get(api_url, headers=headers)
+        if response.status_code == 200:
+            return TspClientResponse(ConfigurationSource(json.loads(response.content.decode('utf-8'))),
+                                     response.status_code, response.text)
+        else:  # pragma: no cover
+            print(GET_CONFIG_SOURCE_TYPES.format(response.status_code, response.text))
+            return TspClientResponse(None, response.status_code, response.text)
+
+    def create_derived_output(self, exp_uuid, output_id, params):
+        '''
+        Create a derived output for a given experiment, output and parameters
+        :param exp_uuid Experiment UUID
+        :param output_id Output ID
+        :param params output query params (JSON)
+        :returns :class:`TspClientResponse <OutputDescriptor>` object
+        :rtype: TspClientResponse
+        '''
+        api_url = '{0}experiments/{1}/outputs/{2}'.format(
+            self.base_url, exp_uuid, output_id)
+
+        response = requests.post(api_url, json=params, headers=headers)
+
+        if response.status_code == 200:
+            return TspClientResponse(OutputDescriptor(json.loads(response.content.decode('utf-8'))),
+                                     response.status_code, response.text)
+        else:  # pragma: no cover
+            print("failed to create derived output: {} {}".format(response.status_code, response.text))
+            return TspClientResponse(None, response.status_code, response.text)
+
+    def delete_derived_output(self, exp_uuid, output_id, derived_output_id):
+        '''
+        Create a derived output for a given experiment, output and parameters
+        :param exp_uuid Experiment UUID
+        :param output_id Output ID
+        :param derived_output_id the ID of the derived output
+        :returns :class:`TspClientResponse <OutputDescriptor>` object
+        :rtype: TspClientResponse
+        '''
+        api_url = '{0}experiments/{1}/outputs/{2}/{3}'.format(
+            self.base_url, exp_uuid, output_id, derived_output_id)
+
+        response = requests.delete(api_url, headers=headers_form)
+
+        if response.status_code == 200:
+            return TspClientResponse(OutputDescriptor(json.loads(response.content.decode('utf-8'))),
+                                     response.status_code, response.text)
+        else:  # pragma: no cover
+            print("delete derived output failed: {} {}".format(response.status_code, response.text))
+            return TspClientResponse(None, response.status_code, response.text)
+
     def fetch_configuration_sources(self):
         '''
         Fetch Extensions (loaded files)

tsp_cli_client

@@ -27,6 +27,7 @@
 """Manual CLI script file."""
 
 import argparse
+import json
 import sys
 
 from os.path import os
@@ -193,6 +194,14 @@ if __name__ == "__main__":
     parser.add_argument("--params", dest="params", help="semicolon separated key value pairs (key1=val1;key2=val2)")
     parser.add_argument("--get-health", dest="get_health", action='store_true', help="Get the health status of the server")
     parser.add_argument("--get-identifier", dest="get_identifier", action='store_true', help="Get important information regarding the trace server and the system")
+    parser.add_argument("--list-output-configuration-sources", dest="list_output_configuration_sources",
+                        help="Get available configuration sources for a given experiment and output", metavar="OUTPUT_ID")
+    parser.add_argument("--list-output-configuration-source", dest="list_output_configuration_source",
+                        help="Get configuration source for a given experiment, output and type", metavar="TYPE_ID")
+    parser.add_argument("--create-output", dest="create_output", help="Create derived output provided by --param or --json-file", metavar="OUTPUT_ID")
+    parser.add_argument("--delete-output", dest="delete_output", help="Delete derived output", metavar="DERIVED_OUTPUT_ID")
+    parser.add_argument("--output-id", dest="output_id", help="The output ID")
+    parser.add_argument("--json-file", dest="json_file", help="JSON file with parameter")
 
     argcomplete.autocomplete(parser)
     options = parser.parse_args()
@@ -509,7 +518,6 @@ if __name__ == "__main__":
                 sys.exit(1)
 
 
-
         if options.load_configuration:
             if options.type_id is not None:
                 if options.params is not None:
@@ -573,6 +581,94 @@ if __name__ == "__main__":
             else:
                 print("No source typeId provided to delete this configuration")
 
+        if options.list_output_configuration_sources:
+            if not options.uuid:
+                print(TRACE_MISSING)
+                sys.exit(1)
+
+            response = tsp_client.fetch_output_configuration_sources(options.uuid, options.list_output_configuration_sources)
+            if response.status_code == 200:
+                configuration_source_set = response.model
+                if not configuration_source_set or len(configuration_source_set.configuration_source_set) == 0:
+                    print('No configuration sources available')
+                else:
+                    print('Successfully listed configuration sources')
+                    print('-----------------------------------------')
+                    print(configuration_source_set.to_json())
+                sys.exit(0)
+            else:
+                sys.exit(1)
+
+        if options.list_output_configuration_source:
+            if not options.uuid:
+                print(TRACE_MISSING)
+                sys.exit(1)
+
+            if not options.output_id:
+                print("No output ID provided")
+                sys.exit(1)
+
+            response = tsp_client.fetch_output_configuration_source(options.uuid, options.output_id, options.list_output_configuration_source)
+            if response.status_code == 200:
+                configuration_source = response.model
+                if not configuration_source:
+                    print('No configuration sources available')
+                else:
+                    print('Successfully listed configuration sources')
+                    print('-----------------------------------------')
+                    print(configuration_source.to_json())
+                sys.exit(0)
+            else:
+                sys.exit(1)
+
+        if options.create_output:
+            if not options.uuid:
+                print(TRACE_MISSING)
+                sys.exit(1)
+
+            if not options.params and not options.json_file:
+                print("No parameters provided with --param or --json-file")
+                sys.exit(1)
+            
+            params = {}
+            if options.params is not None:
+                params = __parse_params(options.params)
+
+            if options.json_file is not None:
+                with open(options.json_file, 'r') as file:
+                    params = json.loads(file.read())
+
+            if (params is not None):
+                response = tsp_client.create_derived_output(options.uuid, options.create_output, params)
+                if response.status_code == 200:
+                    print('Successfully created derived output')
+                    print('-----------------------------------')
+                    print(response.model.to_json())
+                    sys.exit(0)
+                else:
+                    sys.exit(1)
+            else:
+                print("Invalid params provided")
+                sys.exit(1)
+
+        if options.delete_output:
+            if not options.uuid:
+                print(TRACE_MISSING)
+                sys.exit(1)
+
+            if not options.output_id:
+                print("No parent output ID provided")
+                sys.exit(1)
+            
+            response = tsp_client.delete_derived_output(options.uuid, options.output_id, options.delete_output)
+            if response.status_code == 200:
+                print('Successfully deleted derived output')
+                print('-----------------------------------')
+                print(response.model.to_json())
+                sys.exit(0)
+            else:
+                sys.exit(1)
+
         if options.get_health:
             response = tsp_client.fetch_health()
             if response.status_code == 200: