Merge pull request altMITgcm#4 from edoddridge/multiple_tests

allow for multiple subtests, expand list of tests

Author: jahn

.travis.yml

@@ -14,5 +14,39 @@ before_install:
 
 script: 
  - echo `pwd`
- - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; ./testreport -t tutorial_barotropic_gyre | tee tutorial_barotropic_gyre/testreport_out.txt"
- - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; python verification_parser.py -filename 'tutorial_barotropic_gyre/testreport_out.txt' -threshold 15"
+ # aim.5l_cs
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; ./testreport -t aim.5l_cs | tee aim.5l_cs/testreport_out.txt"
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; python verification_parser.py -filename 'aim.5l_cs/testreport_out.txt' -threshold 14 16"
+ # global_ocean.cs32x15
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; ./testreport -t global_ocean.cs32x15 | tee global_ocean.cs32x15/testreport_out.txt"
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; python verification_parser.py -filename 'global_ocean.cs32x15/testreport_out.txt' -threshold 16 16 16 16 16"
+ # global_ocean.90x40x15
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; ./testreport -t global_ocean.90x40x15 | tee global_ocean.90x40x15/testreport_out.txt"
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; python verification_parser.py -filename 'global_ocean.90x40x15/testreport_out.txt' -threshold 16 16 16"
+ # hs94.cs-32x32x5
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; ./testreport -t hs94.cs-32x32x5 | tee hs94.cs-32x32x5/testreport_out.txt"
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; python verification_parser.py -filename 'hs94.cs-32x32x5/testreport_out.txt' -threshold 13 16" 
+ # isomip
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; ./testreport -t isomip | tee isomip/testreport_out.txt"
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; python verification_parser.py -filename 'isomip/testreport_out.txt' -threshold 16 16 16"
+ # offline_exf_seaice
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; ./testreport -t offline_exf_seaice | tee offline_exf_seaice/testreport_out.txt"
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; python verification_parser.py -filename 'offline_exf_seaice/testreport_out.txt' -threshold 16 16 16 16 16"
+ # tutorial advection in gyre
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; ./testreport -t tutorial_advection_in_gyre | tee tutorial_advection_in_gyre/testreport_out.txt"
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; python verification_parser.py -filename 'tutorial_advection_in_gyre/testreport_out.txt' -threshold 16"
+ # tutorial cfc offline
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; ./testreport -t tutorial_cfc_offline | tee tutorial_cfc_offline/testreport_out.txt"
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; python verification_parser.py -filename 'tutorial_cfc_offline/testreport_out.txt' -threshold 16"
+ # tutorial deep convection
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; ./testreport -t tutorial_deep_convection | tee tutorial_deep_convection/testreport_out.txt"
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; python verification_parser.py -filename 'tutorial_deep_convection/testreport_out.txt' -threshold 16 16"
+ # tutorial global oce biogeo
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; ./testreport -t tutorial_global_oce_biogeo | tee tutorial_global_oce_biogeo/testreport_out.txt"
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; python verification_parser.py -filename 'tutorial_global_oce_biogeo/testreport_out.txt' -threshold 16"
+ # tutorial global oce in p
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; ./testreport -t tutorial_global_oce_in_p | tee tutorial_global_oce_in_p/testreport_out.txt"
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; python verification_parser.py -filename 'tutorial_global_oce_in_p/testreport_out.txt' -threshold 16"
+ # tutorial_plume_on_slope
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; ./testreport -t tutorial_plume_on_slope | tee tutorial_plume_on_slope/testreport_out.txt"
+ - docker exec -i fc11-testreport bash -c "cd /MITgcm/verification; python verification_parser.py -filename 'tutorial_plume_on_slope/testreport_out.txt' -threshold 16"

verification/verification_parser.py

@@ -1,8 +1,32 @@
 """Parse verification outputs."""
 
 import re
+import glob
+import os
 
 def verification_parser(filename, threshold):
+
+    # function should be given a threshold value for each sub test
+    (directory, _) = os.path.split(filename)
+    # how many additional tests are run with tweaks to this configuration
+    num_exps = len(glob.glob(directory+'/input.*'))+1
+
+    # check that the correct number of values for `threshold` have been given
+    if len(threshold) != num_exps:
+        # some if statements to deal with grammar
+        if len(threshold)==1:
+            error_message = '{0} value given for threshold, '.format(len(threshold))
+        else:
+            error_message = '{0} values given for threshold, '.format(len(threshold))
+        
+        if num_exps==1:
+            error_message = error_message + 'but {0} subtest found.'.format(num_exps)        
+        else:
+            error_message = error_message + 'but {0} subtests found.'.format(num_exps)
+
+        raise ValueError(error_message)
+        
+    # open the testreport output to assess pass/fail            
     with open(filename) as f:
         lines = f.readlines()
 
@@ -16,23 +40,38 @@ def verification_parser(filename, threshold):
                     # but set to false to catch next matches.
                     first_match = False
                 else:
-                    test_results = lines[i+2]
-
-        # split test_results into a list with values for each number. 
-        # this uses spaces and the < > characters to separate the numbers.
-        test_results = re.split('[ ><]',test_results)
-        # ignore the Genmake, depend, make, and run checks, as well as
-        # the "pass" or "fail" and test name at the end of the line
-        test_results = test_results[4:-3]
-        # convert to floats
-        dp_similarity = []
-        for i, x in enumerate(test_results):
-            try:
-                dp_similarity.append(float(x))
-            except ValueError:
-                pass
-
-        assert all(elements > threshold for elements in dp_similarity)
+                    # save the line number where the output is found
+                    output_line = i
+                    break
+
+        # loop through each of the subexperiments:
+        for j in xrange(len(threshold)):
+            test_results = lines[output_line+2+j]
+
+            # split test_results into a list with values for each number. 
+            # this uses spaces and the < > characters to separate the numbers.
+            test_results = re.split('[ ><]',test_results)
+            # ignore the Genmake, depend, make, and run checks, as well as
+            # the "pass" or "fail" and test name at the end of the line
+            test_results = test_results[4:-3]
+            # convert to floats
+            dp_similarity = []
+            for i, x in enumerate(test_results):
+                try:
+                    dp_similarity.append(float(x))
+                except ValueError:
+                    pass
+
+
+            if len(dp_similarity) >= 17:
+                # this means that the test wasn't an offline advection test.
+                # Remove the means of u and v since they are constrained 
+                # to ~0 by domain geometry and can cause the test to fail 
+                # when it shouldn't.
+                del dp_similarity[15]
+                del dp_similarity[11]
+
+            assert all(elements >= threshold[j] for elements in dp_similarity)
 
 if __name__ == '__main__':
 
@@ -43,8 +82,8 @@ def verification_parser(filename, threshold):
     parser.add_argument('-filename', type=str, 
                         help='path to output file from the verification test')
 
-    parser.add_argument('-threshold', type=int, default=15, 
-                        help='number of decimal places of similarity required for test to pass')
+    parser.add_argument('-threshold',nargs='+', type=int, default=15, 
+                        help='number of decimal places of similarity required for test to pass. Requires a value for each sub test. Separate values with a space.')
 
     args = parser.parse_args()