Merge pull request #150 from floweisshardt/fix/user_result

rework groundtruth handling

Author: floweisshardt

atf_core/scripts/analyser.py

@@ -8,19 +8,19 @@
 import rostest
 import sys
 import time
+import traceback
 import unittest
 import yaml
 
 from atf_core import ATFConfigurationParser
-from atf_msgs.msg import AtfResult, TestResult, TestblockResult, MetricResult, TestblockStatus, KeyValue, DataStamped
+from atf_msgs.msg import AtfResult, TestResult, TestblockResult, MetricResult, TestblockStatus, KeyValue, DataStamped, Groundtruth
 from atf_metrics import metrics_helper
 
 class Analyser:
     def __init__(self, package_name, test_generation_config_file = "atf/test_generation_config.yaml"):
         print "ATF analyser: started!"
         start_time = time.time()
         self.ns = "/atf/"
-        self.error = False
         self.package_name = package_name
 
         # parse configuration
@@ -78,11 +78,13 @@ def __init__(self, package_name, test_generation_config_file = "atf/test_generat
                         break
                     except Exception as e:
                         print "general Exception in ATF analyser", type(e), e
+                        print traceback.format_exc()
                         count_error += 1
                         continue
                     bar.update(j)
             except Exception as e:
                 print "FATAL exception in bag file", type(e), e
+                print traceback.format_exc()
                 continue
             bar.finish()
 
@@ -178,18 +180,15 @@ def aggregate_results(self, atf_result):
                     for tl_test in tl_tests.keys():
                         #print "    tl_test=", tl_test
                         metric_result = MetricResult()
-                        started = True
-                        finished = True
-                        groundtruth_result = True
+                        status = TestblockStatus.SUCCEEDED
+                        groundtruth_result = Groundtruth.SUCCEEDED
                         groundtruth_error_message = ""
                         details = []
                         for test in mbt[metric][testblock].keys():
                             if test.startswith(tl_test):
-                                # aggregate started/stopped from every metric_result
-                                if not mbt[metric][testblock][test].started:
-                                    started = False
-                                if not mbt[metric][testblock][test].finished:
-                                    finished = False
+                                # aggregate status SUCCEEDED from every metric_result
+                                if mbt[metric][testblock][test].status != TestblockStatus.SUCCEEDED:
+                                    status = TestblockStatus.ERROR
 
                                 # aggregate data from every metric_result
                                 data = mbt[metric][testblock][test].data
@@ -201,8 +200,8 @@ def aggregate_results(self, atf_result):
 
                                 # aggregate groundtruth from every metric_result
                                 groundtruth = mbt[metric][testblock][test].groundtruth
-                                if groundtruth.result == False:
-                                    groundtruth_result = False
+                                if groundtruth.result != Groundtruth.SUCCEEDED:
+                                    groundtruth_result = Groundtruth.FAILED
                                     if groundtruth_error_message != "":
                                         groundtruth_error_message += "\n"
                                     groundtruth_error_message += "groundtruth missmatch in subtest %s"%(test)
@@ -219,8 +218,7 @@ def aggregate_results(self, atf_result):
 
                         metric_result.name          = mbt[metric][testblock][test].name
                         metric_result.mode          = MetricResult.SPAN_MEAN # aggregated metrics are always SPAN_MEAN
-                        metric_result.started       = started
-                        metric_result.finished      = finished
+                        metric_result.status        = status
                         # metric_result.series is set above
                         metric_result.data.stamp    = stamp
                         metric_result.data.data     = metrics_helper.get_mean(metric_result.series)
@@ -273,7 +271,7 @@ def aggregate_results(self, atf_result):
                     metric_result = tbm[test][testblock][metric]
                     testblock_result.results.append(metric_result)
                     # aggregate metric result
-                    if metric_result.groundtruth.result == False:
+                    if metric_result.groundtruth.result != Groundtruth.SUCCEEDED:
                         testblock_result.result = False
                         testblock_result.error_message += "\n     - metric '%s': %s"%(metric_result.name, metric_result.groundtruth.error_message)
 

atf_core/src/atf_core/atf.py

@@ -55,7 +55,7 @@ def stop(self, testblock, metric_result = None):
             error_msg = "testblock \'%s\' not in list of testblocks"%testblock
             self._send_error(error_msg)
             raise atf_core.ATFError(error_msg)
-        
+
         if metric_result != None:
 
             #TODO  check if metric_result is of type atf_msgs/MetricResult
@@ -91,22 +91,25 @@ def stop(self, testblock, metric_result = None):
 
         else:
             rospy.loginfo("no user result set for testblock \'%s\'"%testblock)
+            metric_result = MetricResult()
 
         rospy.loginfo("stopping testblock \'%s\'"%testblock)
         trigger = TestblockTrigger()
         trigger.stamp = rospy.Time.now()
         trigger.name = testblock
         trigger.trigger = TestblockTrigger.STOP
-        if metric_result != None:
-            trigger.user_result = metric_result
+        trigger.user_result = metric_result
         self.publisher_trigger.publish(trigger)
 
     def shutdown(self):
         rospy.loginfo("shutting down atf application")
 
         # call stop for all testblocks
         for testblock in self.test.testblocks:
-            self.stop(testblock.name)
+            rospy.sleep(1) # TODO remove sleep and wait until all states have finished their transitions
+            if testblock.name in self.sm_container_status.active_states:
+                rospy.logwarn("shutdown called but testblock %s is still active: calling stop automatically"%testblock.name)
+                self.stop(testblock.name)
 
         # check if any testblock is still running
         rospy.loginfo("waiting for all states to be in a terminal state")

atf_core/src/atf_core/configuration_parser.py

@@ -208,15 +208,20 @@ def validate_metric_parameters(self, metric_type, params):
             print "params not a dict"
             return False        
 
-        if "groundtruth" in params and "groundtruth_epsilon" in params: # groundtruth specified
-            pass
-        elif "groundtruth" not in params and "groundtruth_epsilon" not in params: # no groundtruth specified
-            pass
-        else: # invalid configuration
-            # e.g. (params["groundtruth"] == None and params["groundtruth_epsilon"] != None) or (params["groundtruth"] != None and params["groundtruth_epsilon"] == None)
+        try:
+            params["groundtruth"]
+        except (TypeError, KeyError):
+            return True # no groundtruth specified
+
+        try:
+            params["groundtruth"]["data"]
+            params["groundtruth"]["epsilon"]
+            return True # groundtruth specified
+        except (TypeError, KeyError):
+            # e.g. (params["groundtruth"]["data"] == None and params["groundtruth"]["epsilon"] != None) or (params["groundtruth"]["data"] != None and params["groundtruth"]["epsilon"] == None)
             print "invalid groundtruth specified:", params
             return False
-        
+
         return True # all checks successfull
 
     def load_data(self, filename):

atf_core/src/atf_core/testblock.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 from atf_core import ATFAnalyserError
-from atf_msgs.msg import TestblockResult, TestblockStatus
+from atf_msgs.msg import TestblockResult, TestblockStatus, Groundtruth
 
 class Testblock:
     def __init__(self, name, metric_handles, recorder_handle):
@@ -12,8 +12,7 @@ def __init__(self, name, metric_handles, recorder_handle):
         self.trigger = None
         self.timestamp = None
         self.exception = None
-        self.atf_started = False
-        self.status = None
+        self.status = TestblockStatus.INACTIVE
 
     def get_result(self):
 
@@ -35,11 +34,11 @@ def get_result(self):
                 testblock_result.results.append(metric_result)
 
                 # aggregate result
-                if metric_result.groundtruth.result != None and not metric_result.groundtruth.result:
+                if metric_result.groundtruth.result != Groundtruth.SUCCEEDED:
                     testblock_result.result = False
                     testblock_result.error_message += "\n     - metric '%s': %s"%(metric_result.name, metric_result.groundtruth.error_message)
                     #print testblock_result.groundtruth_error_message
-                if testblock_result.result == None and metric_result.groundtruth.result:
+                if testblock_result.result == None and metric_result.groundtruth.result == Groundtruth.SUCCEEDED:
                     testblock_result.result = True
 
         if testblock_result.result == None: