Merge pull request #6 from blsemo/master

Merge over to blackseamonster

Author: blsemo

.gitignore

@@ -1,6 +1,12 @@
+.idea
 .DS_Store
 depthmapX-build
 build
 RegressionTest/rundir
 __pycache__
 depthmapX.pro.user
+*_BACKUP_*
+*_BASE_*
+*_LOCAL_*
+*_REMOTE_*
+

.travis.yml

@@ -18,5 +18,5 @@ script:
   - mkdir build && cd build
   - /opt/qt57/bin/qmake ../depthmapX.pro
   - make
-  - ./cliTest/cliTest && ./GuiUnitTest/GuiUnitTest && ./salaTest/salaTest
-  - cd ../RegressionTest && python3.5 test_main.py && python3.5 RegressionTestRunner.py
+  - ./cliTest/cliTest && ./GuiUnitTest/GuiUnitTest && ./salaTest/salaTest && ./genlibTest/genlibTest
+  - cd ../RegressionTest/test && python3.5 test_main.py && cd .. && python3.5 RegressionTestRunner.py

RegressionTest/RegressionTestRunner.py

@@ -1,7 +1,9 @@
 import runhelpers
 import config
 import depthmaprunner
+import performancerunner
 import os
+import sys
 
 defaultConfigFile = "regressionconfig.json"
 
@@ -17,12 +19,17 @@ def __init__(self, configfile, runfunc):
     def run(self):
         if not os.path.exists(self.config.rundir):
             os.makedirs(self.config.rundir)
-        runner = depthmaprunner.DepthmapRegressionRunner( self.runfunc, self.baseBinary, self.testBinary, self.config.rundir )
+        if self.config.performanceRegression.enabled:
+            print("Performance regression runs enabled")
+            runner = performancerunner.PerformanceRunner(self.runfunc, self.baseBinary, self.testBinary, self.config.rundir,self.config.performanceRegression )
+        else:
+            print("Default regression runs - no performance")
+            runner = depthmaprunner.DepthmapRegressionRunner( self.runfunc, self.baseBinary, self.testBinary, self.config.rundir )
 
         good = True
         for name, case in self.config.testcases.items():
             print("Running test case " + name)
-            success, output = runner.runTestCase(name, case.cmd.infile, case.cmd.outfile, case.cmd.mode, case.cmd.simpleMode, case.cmd.modeLines)
+            success, output = runner.runTestCase(name, case)
             if not success:
                 good = False
                 print ("Failed:\n" + output)
@@ -31,7 +38,10 @@ def run(self):
         return good        
 
 if __name__ == "__main__":
-    r = RegressionTestRunner(defaultConfigFile, runhelpers.runExecutable)
+    configFile = defaultConfigFile
+    if len(sys.argv) == 2:
+        configFile = sys.argv[1]
+    r = RegressionTestRunner(configFile, runhelpers.runExecutable)
     if not r.run():
         exit(-1)
 

RegressionTest/cmdlinewrapper.py

@@ -9,7 +9,7 @@ def __init__(self):
         self.outfile = None
         self.simpleMode = False
         self.mode = None
-        self.modeLines = []
+        self.extraArgs = {}
         self.timingFile = None
 
 
@@ -27,42 +27,15 @@ def toCmdArray(self):
         if self.timingFile:
             args.extend(["-t", self.timingFile])
 
-        for modeLine in self.modeLines:
-            args.extend(modeLine.toCmdArray())
+        for key, value in self.extraArgs.items():
+            if isinstance(value, list):
+                for v in value:
+                    args.append(key)
+                    args.append(v)
+            else:
+                args.append(key)
+                if value:
+                    args.append(value)
 
         return args    
 
-class VisibilityCmd():
-    visibilityMode = None
-    globalMeasures = False
-    localMeasures = False
-    radius = None
-
-    def toCmdArray(self):
-        if self.visibilityMode == None:
-            raise CommandLineError("visibility mode must be defined")
-        args = ["-vm", self.visibilityMode ]
-        if self.globalMeasures:
-            args.append("-vg")
-        if self.localMeasures:
-            args.append("-vl")
-        if not self.radius == None:
-            args.extend(["-vr", self.radius])
-        return args
-        
-class LinkCmd():
-    linksFile = None
-    manualLinks = None
-
-    def toCmdArray(self):
-        if self.linksFile == None and self.manualLinks == None:
-            raise CommandLineError("links must be given as a file or each one manually")
-        if self.linksFile != None and self.manualLinks != None:
-            raise CommandLineError("links must be given as a file or each one manually")
-        args = [];
-        if self.manualLinks:
-            for manualLink in self.manualLinks:
-                args.extend(["-lnk", manualLink])
-        if self.linksFile:
-            args.extend(["-lf", self.linksFile]);
-        return args

RegressionTest/config.py

@@ -1,52 +1,23 @@
 import json
 import os.path
 import cmdlinewrapper
+from performanceregressionconfig import PerformanceRegressionConfig
+
 
 class ConfigError(Exception):
     def __init__(self, message):
         self.message = message
 
-def buildSubCmd( subType, subConfig ):
-    if subType == "Visibility":
-        scmd = cmdlinewrapper.VisibilityCmd()
-        scmd.visibilityMode = subConfig["visibilityMode"]
-        if "globalMeasures" in subConfig:
-            scmd.globalMeasures = True
-        if "localMeasures" in subConfig:
-            scmd.localMeasures = True
-        if "radius" in subConfig:
-            scmd.radius = subConfig["radius"]
-        return scmd;
-    elif subType == "Link":
-        scmd = cmdlinewrapper.LinkCmd()
-        if "linksFile" in subConfig:
-            scmd.linksFile = subConfig["linksFile"]
-        if "manualLinks" in subConfig:
-            scmd.manualLinks = subConfig["manualLinks"]
-        return scmd;
-    else:
-        raise ConfigError("Unknown sub commandline config " + subType)
-    
 def buildCmd(testcase):
     cmd = cmdlinewrapper.DepthmapCmd()
     cmd.infile = testcase["infile"]
     cmd.outfile = testcase["outfile"]
     cmd.mode = testcase["mode"]
     if "simple" in testcase and not testcase["simple"]  == "false":
         cmd.simpleMode = True
-    if "subcmds" in testcase:
-        for (key, value) in testcase["subcmds"].items():
-            cmd.modeLines.append(buildSubCmd(key, value))
+    cmd.extraArgs = testcase.get("extraArgs", {})
     return cmd
 
-class TestCase():
-    def __init__(self, cmd):
-        self.cmd = cmd
-
-def buildTestcase(testcase, rundir, configdir):
-    return TestCase(buildCmd(testcase)) 
-
-
 class RegressionConfig():
     def __init__(self, filename):
         with open(filename, "r") as f:
@@ -55,8 +26,9 @@ def __init__(self, filename):
         self.rundir = config["rundir"]
         self.basebinlocation = config["basebinlocation"]
         self.testbinlocation = config["testbinlocation"]
+        self.performanceRegression = PerformanceRegressionConfig(config.get("performance", None))
         self.testcases = {}
         for (name, tc) in config["testcases"].items():
-            self.testcases[name] = buildTestcase(tc, self.rundir, configdir)
+            self.testcases[name] = buildCmd(tc)
 
 

RegressionTest/depthmaprunner.py

@@ -2,7 +2,9 @@
 import cmdlinewrapper
 import difflib
 import pprint
-import csv
+
+import runhelpers
+
 
 class DepthmapRunner():
     def __init__(self, runFunc, binary ):
@@ -22,88 +24,40 @@ def diffBinaryFiles(file1, file2):
     gen = difflib.diff_bytes(difflib.unified_diff, [content1], [content2])
     return not(list(gen))
 
-def checkPerformance(baseFile, testFile):
-    """ 
-    Check the performance of 2 depthmap runs against each other
-    This function expects the timing from a base and test run and  parses them
-    as CSV. For now, it expects the entries to be the same. It will return an
-    error message if
-    * one or both of the files are missing
-    * the number of lines or the labels don't match
-    * the test run is more than 5 seconds or 5% slower than the baseline
-      (whatever is greater)
-    """
-    if not os.path.exists(baseFile):
-        return "Base performance timing file {0} is missing".format(baseFile)
-    if not os.path.exists(testFile):
-        return "Test performance timing file {0} is missing".format(testFile)
-    with open(baseFile) as baseHandle, open(testFile) as testHandle:
-        baseReader = csv.DictReader(baseHandle)
-        testReader = csv.DictReader(testHandle)
-
-        baseDone = False
-        testDone = False
-        
-        while True:
-            try:
-                baseLine = next(baseReader)
-            except StopIteration:
-                baseDone = True
-
-            try:
-                testLine = next(testReader)
-            except StopIteration:
-                testDone = True
-
-            if baseDone and testDone:
-                return ""
-            if baseDone and not testDone:
-                return "baseline performance file {0} has fewer lines than the test one {1}".format(baseFile, testFile)
-            if testDone and not baseDone:
-                return "baseline performance file {0} has more lines than the test one {1}".format(baseFile, testFile)
-
-            if not baseLine["action"] == testLine["action"]:
-                return "performance line mismatch: base '{0}', test '{1}'".format(baseLine["action"], testLine["action"])
-
-            baseTime = float(baseLine["duration"])
-            testTime = float(testLine["duration"])
-
-            allowance = max(5.0, baseTime / 20.0 )
-            if testTime > baseTime + allowance:
-                return "Performance regression: {0} took {1}s instead of {2}s".format(baseLine["action"], testLine["duration"], baseLine["duration"])
-            
-        
-    
 class DepthmapRegressionRunner():
     def __init__(self, runFunc, baseBinary, testBinary, workingDir):
         self.__baseRunner = DepthmapRunner(runFunc, baseBinary)
         self.__testRunner = DepthmapRunner(runFunc, testBinary)
         self.__workingDir = workingDir
 
-    def runTestCase(self, name, infile, outfile, mode, simpleMode = False, subcmds = [], timingFile = "runtimes.csv"):
-        cmd = cmdlinewrapper.DepthmapCmd()
-        cmd.infile = infile
-        cmd.outfile = outfile
-        cmd.mode = mode
-        cmd.simpleMode = simpleMode
-        cmd.modeLines = subcmds
-        cmd.timingFile = timingFile
+    def makeBaseDir(self, name):
+        return os.path.join(self.__workingDir, name + "_base")
+
+    def makeTestDir(self, name):
+        return os.path.join(self.__workingDir, name + "_test")
+
+    def runTestCase(self, name, cmd):
+        runhelpers.prepareDirectory(self.makeBaseDir(name))
+        runhelpers.prepareDirectory(self.makeTestDir(name))
+        return self.runTestCaseImpl(name, cmd)
 
-        baseDir = os.path.join(self.__workingDir, name + "_base")
+    def runTestCaseImpl(self, name, cmd):
+        baseDir = self.makeBaseDir(name)
         (baseSuccess, baseOut) = self.__baseRunner.runDepthmap(cmd, baseDir)
         if not baseSuccess:
             print("Baseline run failed with arguments " + pprint.pformat(cmd.toCmdArray()))
             print(baseOut)
             return (False, "Baseline run failed")
-        testDir = os.path.join(self.__workingDir, name + "_test")
+
+        testDir = self.makeTestDir(name)
         (testSuccess, testOut) = self.__testRunner.runDepthmap(cmd, testDir)
         if not testSuccess:
             print("Test run failed with arguments " + pprint.pformat(cmd.toCmdArray()))
             print(testOut)
             return (False, "Test run failed")
 
-        baseFile = os.path.join(baseDir, outfile)
-        testFile = os.path.join(testDir, outfile)
+        baseFile = os.path.join(baseDir, cmd.outfile)
+        testFile = os.path.join(testDir, cmd.outfile)
         if not os.path.exists(baseFile):
             message = "Baseline output {0} does not exist".format(baseFile)
             print (message)
@@ -118,11 +72,6 @@ def runTestCase(self, name, infile, outfile, mode, simpleMode = False, subcmds =
             print (message)
             return (False, message)
 
-        message = checkPerformance( os.path.join(baseDir, timingFile), os.path.join(testDir, timingFile))
-        if message:
-            print(message)
-            return (False,message)
-
         return (True, "")
 
 

RegressionTest/performance_regression.json

@@ -0,0 +1,18 @@
+{
+  "rundir": "rundir",
+  "basebinlocation": "../../BaselineBinaries",
+  "testbinlocation": "../../../build",
+  "performance":{},
+  "testcases": {
+    "visibility_global_n": {
+      "infile": "../../../testdata/gallery_connected.graph",
+      "outfile": "out.graph",
+      "mode": "VGA",
+      "extraArgs": {
+        "-vm": "visibility",
+        "-vg": "1",
+        "-vr": "n"
+      }
+    }
+  }
+}

RegressionTest/performanceregressionconfig.py

@@ -0,0 +1,24 @@
+
+
+class PerformanceRegressionConfig:
+    """ Encapsulate performance regression config
+        This takes an optional performance regression config. All Elements
+        in the performance regression config are optional. If it is None
+        completly, performance regresssion will be disabled
+        "peformanceRegression":
+        {
+            enabled = 1,   <- enabling performance regression - only required if nothing else is set, the existence of a non null config will turn on performance regression
+            runsPerInstance = 3, <- how many runs of each run def to run and average over
+            relativeThresholdInPercent = 1.5, <- how many percent can the new version be slower without failing
+            absoluteThresholdInSeconds = 2.3, <- how many seconds the new version can be slower without failing
+                                                 of the two above, breaching the lower one will lead to failure.
+        }
+    """
+    def __init__(self, perfConfig):
+        if None == perfConfig or ("enabled" in perfConfig and perfConfig["enabled"] not in ["True", "true", "1", "yes"]):
+            self.enabled = False
+            return
+        self.enabled = True;
+        self.runsPerInstance = int(perfConfig.get("runsPerInstance", 3))
+        self.relativeThresholdInPercent = float(perfConfig.get("relativeThresholdInPercent", 1))
+        self.absoluteThresholdInSeconds = float(perfConfig.get("absoluteThresholdInSeconds",1))