testwise checks in automated bucket testing

Author: ngsrinivas

pyretic/examples/bucket.py

@@ -67,9 +67,10 @@ def static_fwding_cycle_3_3():
 
 class QueryTest(CountBucket):
 
-    def __init__(self):
+    def __init__(self, test_num=None):
         super(QueryTest, self).__init__()
         self.register_callback(self.query_callback)
+        self.test_num = test_num if test_num else id(self)
         import threading
         self.query_thread = threading.Thread(target=self.query_thread)
         self.query_thread.daemon = True
@@ -84,14 +85,14 @@ def query_thread(self):
             output += self.get_matches()
             # print output
             self.pull_stats()
-            print ">>>", str(datetime.now()), ('issued query %d, sleeping for %f' %
-                                               (id(self), interval))
+            print ">>>", str(datetime.now()), ('issued query %s, sleeping for %f' %
+                                               (str(self.test_num), interval))
             time.sleep(interval)
 
     def query_callback(self, counts):
         print "***", str(datetime.now()), "| In user callback for bucket",
-        print id(self)
-        print "Bucket", id(self), "(packet, byte) counts:", counts
+        print self.test_num
+        print "Bucket", self.test_num, "(packet, byte) counts:", counts
         print "-----------------------------------"
 
 def test0():
@@ -112,7 +113,7 @@ def test0():
     number of packets/bytes displayed by wireshark or tshark with these
     filters. They are designed to work under topology and policy updates.
     """
-    test_bucket = QueryTest()
+    test_bucket = QueryTest(0)
     return test_bucket
 
 def test1():
@@ -127,7 +128,7 @@ def test1():
     arp.dst.proto_ipv4 == 10.0.2.0/24) ) or ipv6 ) ) and ( ip.src ==
     10.0.0.1 || (arp && arp.src.proto_ipv4 == 10.0.0.1) )
     """
-    test_bucket = QueryTest()
+    test_bucket = QueryTest(1)
     return (match(srcip=ip1) >> test_bucket)
 
 def test2():
@@ -153,7 +154,7 @@ def test2():
     """
     b = [] # counting buckets
     for i in range(0,2):
-        b.append(QueryTest())
+        b.append(QueryTest('2.b%d' % i))
         time.sleep(0.2)
 
     pol1 = (match(ethtype=IP_TYPE) & match(srcip=ip1)) >> b[0]
@@ -186,7 +187,7 @@ def test3():
     """
     b = [] # counting buckets
     for i in range(0,2):
-        b.append(QueryTest())
+        b.append(QueryTest('3.b%d' % i))
         time.sleep(0.2)
 
     pol1 = match(srcip=ip1) >> b[0]
@@ -221,7 +222,7 @@ def test4():
     """
     b = [] # counting buckets
     for i in range(0,3):
-        b.append(QueryTest())
+        b.append(QueryTest('4.b%d' % i))
         time.sleep(0.2)
 
     query1 = match(srcip=ip1) >> match(dstip=ip2) >> b[0]
@@ -245,7 +246,7 @@ def test5():
     (arp.dst.proto_ipv4 == 10.0.0.1 || arp.dst.proto_ipv4 == 10.0.0.2 ||
     arp.dst.proto_ipv4 == 10.0.0.3) ) )
     """
-    test_bucket = QueryTest()
+    test_bucket = QueryTest(5)
     matched_traffic = ( (~match(srcip=ip1) & match(dstip=ip2)) +
                         (~match(srcip=ip1) & match(dstip=ip3)) +
                         (~match(srcip=ip1) & match(dstip=ip1)) )
@@ -264,7 +265,7 @@ def test6():
     ( (ip && ip.src == 10.0.0.1 ) or (arp && arp.src.proto_ipv4 == 10.0.0.1) or
     ipv6))
     """
-    test_bucket = QueryTest()
+    test_bucket = QueryTest(6)
     matched_traffic = ~match(srcip=ip1)
     return (matched_traffic >> test_bucket)
 
@@ -280,7 +281,7 @@ def test7():
     arp.dst.proto_ipv4 == 10.0.2.0/24) ) or ipv6 ) ) and (ip.dst == 10.0.0.1 ||
     (arp && (arp.dst.proto_ipv4 == 10.0.0.1 ) ) )
     """
-    return ( (match(dstip=ip1) >> QueryTest()) +
+    return ( (match(dstip=ip1) >> QueryTest(7)) +
              (match(dstip=ip1) >> Controller) )
 
 def parse_args(kwargs, defaults):

pyretic/tests/test_bucket.py

@@ -327,12 +327,22 @@ def write_passfail_info(success_file, tshark_counts, buckets_counts, ctlr):
 def bucket_write_passfail_info(success_file, tshark_counts, buckets_counts):
     passfail = open(success_file, 'w')
     output_str = ''
-    if set(tshark_counts.values()) == set(buckets_counts.values()):
-        output_str += "PASS\n"
-    else:
+    if set(tshark_counts.keys()) != set(buckets_counts.keys()):
         output_str += "FAIL\n"
-        output_str += "TShark: %s\n" % str(tshark_counts)
-        output_str += "Bucket: %s\n" % str(buckets_counts)
+        output_str += "Query references mismatch:\n"
+        output_str += "TShark: %s\n" % str(tshark_counts.keys())
+        output_str += "Bucket: %s\n" % str(buckets_counts.keys())
+    elif True:
+        for q in tshark_counts.keys():
+            tc = tshark_counts[q]
+            bc = buckets_counts[q]
+            if tc != bc:
+                output_str += "FAIL\n"
+                output_str += "Query: %s\n" % q
+                output_str += "TShark: %s\n" % str(tc)
+                output_str += "Bucket: %s\n" % str(bc)
+    if output_str == '':
+        output_str += "PASS\n"
     print output_str
     passfail.write(output_str)
     passfail.close()