use enum for result

Author: floweisshardt

atf_core/scripts/analyser.py

@@ -8,6 +8,7 @@
 import rostest
 import sys
 import time
+import traceback
 import unittest
 import yaml
 
@@ -78,11 +79,13 @@ def __init__(self, package_name, test_generation_config_file = "atf/test_generat
                         break
                     except Exception as e:
                         print "general Exception in ATF analyser", type(e), e
+                        print traceback.format_exc()
                         count_error += 1
                         continue
                     bar.update(j)
             except Exception as e:
                 print "FATAL exception in bag file", type(e), e
+                print traceback.format_exc()
                 continue
             bar.finish()
 
@@ -178,18 +181,15 @@ def aggregate_results(self, atf_result):
                     for tl_test in tl_tests.keys():
                         #print "    tl_test=", tl_test
                         metric_result = MetricResult()
-                        started = True
-                        finished = True
+                        status = TestblockStatus.SUCCEEDED
                         groundtruth_result = True
                         groundtruth_error_message = ""
                         details = []
                         for test in mbt[metric][testblock].keys():
                             if test.startswith(tl_test):
-                                # aggregate started/stopped from every metric_result
-                                if not mbt[metric][testblock][test].started:
-                                    started = False
-                                if not mbt[metric][testblock][test].finished:
-                                    finished = False
+                                # aggregate status SUCCEEDED from every metric_result
+                                if mbt[metric][testblock][test].status != TestblockStatus.SUCCEEDED:
+                                    status = TestblockStatus.ERROR
 
                                 # aggregate data from every metric_result
                                 data = mbt[metric][testblock][test].data
@@ -219,8 +219,7 @@ def aggregate_results(self, atf_result):
 
                         metric_result.name          = mbt[metric][testblock][test].name
                         metric_result.mode          = MetricResult.SPAN_MEAN # aggregated metrics are always SPAN_MEAN
-                        metric_result.started       = started
-                        metric_result.finished      = finished
+                        metric_result.status        = status
                         # metric_result.series is set above
                         metric_result.data.stamp    = stamp
                         metric_result.data.data     = metrics_helper.get_mean(metric_result.series)

atf_core/src/atf_core/atf.py

@@ -92,7 +92,6 @@ def stop(self, testblock, metric_result = None):
         else:
             rospy.loginfo("no user result set for testblock \'%s\'"%testblock)
             metric_result = MetricResult()
-            metric_result.groundtruth.error_message = "!!USER ERROR!!: no user result set for testblock %s"%testblock # TODO use from global field (same as in atf.stop())
 
         rospy.loginfo("stopping testblock \'%s\'"%testblock)
         trigger = TestblockTrigger()

atf_core/src/atf_core/testblock.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 from atf_core import ATFAnalyserError
-from atf_msgs.msg import TestblockResult, TestblockStatus
+from atf_msgs.msg import TestblockResult, TestblockStatus, Groundtruth
 
 class Testblock:
     def __init__(self, name, metric_handles, recorder_handle):
@@ -12,7 +12,6 @@ def __init__(self, name, metric_handles, recorder_handle):
         self.trigger = None
         self.timestamp = None
         self.exception = None
-        self.atf_started = False
         self.status = None
 
     def get_result(self):
@@ -35,11 +34,11 @@ def get_result(self):
                 testblock_result.results.append(metric_result)
 
                 # aggregate result
-                if metric_result.groundtruth.result != None and not metric_result.groundtruth.result:
+                if metric_result.groundtruth.result != None and metric_result.groundtruth.result != Groundtruth.SUCCEEDED:
                     testblock_result.result = False
                     testblock_result.error_message += "\n     - metric '%s': %s"%(metric_result.name, metric_result.groundtruth.error_message)
                     #print testblock_result.groundtruth_error_message
-                if testblock_result.result == None and metric_result.groundtruth.result:
+                if testblock_result.result == None and metric_result.groundtruth.result == Groundtruth.SUCCEEDED:
                     testblock_result.result = True
 
         if testblock_result.result == None:

atf_metrics/src/atf_metrics/calculate_interface.py

@@ -1,11 +1,10 @@
 #!/usr/bin/env python
-import copy
 import math
 import rospy
 
 from atf_core import ATFAnalyserError, ATFConfigurationError
 from atf_msgs.msg import Api
-from atf_msgs.msg import MetricResult, Groundtruth, KeyValue, DataStamped
+from atf_msgs.msg import MetricResult, Groundtruth, KeyValue, DataStamped, TestblockStatus
 from atf_metrics import metrics_helper
 
 class CalculateInterfaceParamHandler:
@@ -61,29 +60,25 @@ def __init__(self, name, testblock_name, params, mode, series_mode):
         Class for calculating the interface type.
         """
         self.name = name
-        self.started = False
-        self.finished = False
-        self.active = False
+        self.testblock_name = testblock_name
+        self.status = TestblockStatus()
         self.groundtruth = Groundtruth()
         self.groundtruth.available = True
         self.groundtruth.data = 100         # this is the max score
         self.groundtruth.epsilon = 0        # no deviation from max score allowed
-        self.testblock_name = testblock_name
         self.mode = mode
         self.series_mode = series_mode
         self.series = []
-        self.data = DataStamped()
+
         self.interface_details = {}
         self.api_dict = {}
         self.params = params
 
     def start(self, status):
-        self.active = True
-        self.started = True
+        self.status = status
 
     def stop(self, status):
-        self.active = False
-        self.finished = True
+        self.status = status
 
     def pause(self, status):
         pass
@@ -93,13 +88,14 @@ def purge(self, status):
 
     def update(self, topic, msg, t):
         # get data if testblock is active
-        if self.active:
+        if self.status.status == TestblockStatus.ACTIVE:
             if topic == "/atf/api" and msg.testblock_name == self.testblock_name:
                 self.api_dict = self.msg_to_dict(msg)
                 interface_data, self.interface_details = self.calculate_data_and_details()
-                self.data.stamp = t
-                self.data.data = interface_data
-                self.series.append(copy.deepcopy(self.data))  # FIXME handle fixed rates
+                data = DataStamped()
+                data.stamp = t
+                data.data = interface_data
+                self.series.append(data)  # FIXME handle fixed rates
 
     def msg_to_dict(self, msg):
         api_dict = {}
@@ -196,78 +192,32 @@ def get_result(self):
         metric_result = MetricResult()
         metric_result.name = self.name
         metric_result.mode = self.mode
-        metric_result.started = self.started # FIXME remove
-        metric_result.finished = self.finished # FIXME remove
+        metric_result.status = self.status.status
         metric_result.series = []
         metric_result.groundtruth.available = self.groundtruth.available
         metric_result.groundtruth.data = self.groundtruth.data
         metric_result.groundtruth.epsilon = self.groundtruth.epsilon
 
-        if not self.started:
-            error_message = "testblock %s never started"%self.testblock_name
-            #print error_message
-            metric_result.groundtruth.result = False
-            metric_result.groundtruth.error_message = error_message
-            return metric_result
-
-        if not self.finished:
-            error_message = "testblock %s never stopped"%self.testblock_name
-            #print error_message
-            metric_result.groundtruth.result = False
-            metric_result.groundtruth.error_message = error_message
+        if self.status.status != TestblockStatus.SUCCEEDED:
+            metric_result.groundtruth.result = Groundtruth.FAILED
+            metric_result.groundtruth.error_message = metrics_helper.extract_error_message(self.status)
             return metric_result
 
         # check if result is available
         if len(self.series) == 0:
             # let the analyzer know that this test failed
-            metric_result.groundtruth.result = False
+            metric_result.groundtruth.result = Groundtruth.FAILED
             metric_result.groundtruth.error_message = "testblock %s stopped without result"%self.testblock_name
             return metric_result
 
         # at this point we're sure that any result is available
 
-        # calculate metric data
+        # set series
         if self.series_mode != None:
             metric_result.series = self.series
-        if metric_result.mode == MetricResult.SNAP:
-            metric_result.data = self.series[-1]                           # take last element from self.series for data and stamp
-            metric_result.min = metric_result.data
-            metric_result.max = metric_result.data
-            metric_result.mean = metric_result.data.data
-            metric_result.std = 0.0
-        elif metric_result.mode == MetricResult.SPAN_MEAN:
-            metric_result.min = metrics_helper.get_min(self.series)
-            metric_result.max = metrics_helper.get_max(self.series)
-            metric_result.mean = metrics_helper.get_mean(self.series)
-            metric_result.std = metrics_helper.get_std(self.series)
-            metric_result.data.data = metric_result.mean                   # take mean for data
-            metric_result.data.stamp = self.series[-1].stamp               # take stamp from last element in self.series for stamp
-        elif metric_result.mode == MetricResult.SPAN_MIN:
-            metric_result.min = metrics_helper.get_min(self.series)
-            metric_result.max = metrics_helper.get_max(self.series)
-            metric_result.mean = metrics_helper.get_mean(self.series)
-            metric_result.std = metrics_helper.get_std(self.series)
-            metric_result.data = metric_result.min
-        elif metric_result.mode == MetricResult.SPAN_ABSMIN:
-            metric_result.min = metrics_helper.get_absmin(self.series)
-            metric_result.max = metrics_helper.get_absmax(self.series)
-            metric_result.mean = metrics_helper.get_mean(self.series)
-            metric_result.std = metrics_helper.get_std(self.series)
-            metric_result.data = metric_result.min
-        elif metric_result.mode == MetricResult.SPAN_MAX:
-            metric_result.min = metrics_helper.get_min(self.series)
-            metric_result.max = metrics_helper.get_max(self.series)
-            metric_result.mean = metrics_helper.get_mean(self.series)
-            metric_result.std = metrics_helper.get_std(self.series)
-            metric_result.data = metric_result.max
-        elif metric_result.mode == MetricResult.SPAN_ABSMAX:
-            metric_result.min = metrics_helper.get_absmin(self.series)
-            metric_result.max = metrics_helper.get_absmax(self.series)
-            metric_result.mean = metrics_helper.get_mean(self.series)
-            metric_result.std = metrics_helper.get_std(self.series)
-            metric_result.data = metric_result.max
-        else: # invalid mode
-            raise ATFAnalyserError("Analysing failed, invalid mode '%s' for metric '%s'."%(metric_result.mode, metric_result.name))
+
+        # calculate metric data
+        [metric_result.data, metric_result.min, metric_result.max, metric_result.mean, metric_result.std] = metrics_helper.calculate_metric_data(metric_result.name, metric_result.mode, self.series)
 
         # fill details as KeyValue messages
         details = []
@@ -277,14 +227,14 @@ def get_result(self):
         # evaluate metric data
         if metric_result.groundtruth.available: # groundtruth available
             if math.fabs(metric_result.groundtruth.data - metric_result.data.data) <= metric_result.groundtruth.epsilon:
-                metric_result.groundtruth.result = True
+                metric_result.groundtruth.result = Groundtruth.SUCCEEDED
                 metric_result.groundtruth.error_message = "all OK"
             else:
-                metric_result.groundtruth.result = False
+                metric_result.groundtruth.result = Groundtruth.FAILED
                 metric_result.groundtruth.error_message = "groundtruth missmatch: %f not within %f+-%f"%(metric_result.data.data, metric_result.groundtruth.data, metric_result.groundtruth.epsilon)
 
         else: # groundtruth not available
-            metric_result.groundtruth.result = True
+            metric_result.groundtruth.result = Groundtruth.SUCCEEDED
             metric_result.groundtruth.error_message = "all OK (no groundtruth available)"
 
         return metric_result