Updates from Jenkins 0.79 scripts

Author: TimSVector

.gitignore

@@ -0,0 +1,4 @@
+/.idea
+**/__pycache__
+/*.log
+/*.pyc

cobertura.py

@@ -32,7 +32,10 @@
 except:
     pass
 
-from vector.apps.DataAPI.cover_api import CoverApi
+try:
+    from vector.apps.DataAPI.cover_api import CoverApi
+except:
+    pass
 
 try:
     from vector.apps.DataAPI.unit_test_api import UnitTestApi
@@ -393,10 +396,10 @@ def procesCoverage(coverXML, coverApi, extended = False, source_root = ""):
 
 def runCoverageResultsMP(packages, mpFile, verbose = False, extended=False, source_root = ""):
 
-    with VCProjectApi(mpFile) as vcproj:
-        api = vcproj.project.cover_api
-        
-        results = runCoberturaResults(packages, api, verbose = False, extended = extended, source_root = source_root)
+    vcproj = VCProjectApi(mpFile)
+    api = vcproj.project.cover_api    
+    results = runCoberturaResults(packages, api, verbose = False, extended = extended, source_root = source_root)
+    vcproj.close()
 
     return results
 

copy_build_dir.py

@@ -30,6 +30,12 @@
 import shutil
 import argparse
 
+from vcast_utils import dump, checkVectorCASTVersion, getVectorCASTEncoding
+
+if not checkVectorCASTVersion(19):
+    print ("Copy build directory for pipelines not supported. Please upgrade VectorCAST")
+    sys.exit(0)
+
 try:
     ## This tests to see if 2018 is present.
     from vector.apps.ReportBuilder.custom_report import CustomReport
@@ -43,7 +49,7 @@
 try:
     from vector.apps.DataAPI.vcproject_api import VCProjectApi
 except:
-    pass
+    raise ImportError("Full reports generated by previous call to generate-results.py")
 
 def make_relative(path, workspace, vCastProjectWorkspace, ManageProjectName):
 
@@ -237,14 +243,17 @@ def run(ManageProjectName, Level, BaseName, Env, workspace, vCastProjectWorkspac
 
 def getVcastProjectWorkspace(args):
 
-    with VCProjectApi(args.ManageProject) as vcproj:
-        vCastProjectWorkspace = vcproj.project.workspace
-    
+    vcproj = VCProjectApi(args.ManageProject)
+    vCastProjectWorkspace = vcproj.project.workspace
+    vcproj.close()
     return vCastProjectWorkspace
 
 if __name__ == '__main__':
 
-    
+    if not checkVectorCASTVersion(19):
+        print ("Pipeline's copy build directory not supported under. Please upgrade VectorCAST")
+        sys.exit(0)
+
     parser = argparse.ArgumentParser()
     parser.add_argument('ManageProject',       help='Manager Project Name')
     parser.add_argument('-l', '--level',       help='Environment Name if only doing single environment. Should be in the form of level/env', default="NotProvided/NotProvided")

create_index_html.py

@@ -117,7 +117,6 @@ def getReportName(filename):
 baseOutputDir = ""
 
 def create_index_html(mpName, isGitLab = False, output_dir = ""):
-    import pathlib
     from vector.apps.DataAPI.vcproject_api import VCProjectApi
     from vector.apps.ReportBuilder.custom_report import CustomReport
 
@@ -127,25 +126,24 @@ def create_index_html(mpName, isGitLab = False, output_dir = ""):
     global baseOutputDir
     baseOutputDir = output_dir
 
-    with VCProjectApi(mpName) as vcproj: 
-        # Set custom report directory to the where this script was
-        # found. Must contain sections/index_section.py
-        rep_path = pathlib.Path(__file__).parent.resolve()
-
-        if usingGitLabCI:
-            output_file=os.path.join(baseOutputDir,"index.html")
-        else:
-            output_file=os.path.join(baseOutputDir,"index.html")
-            
-        CustomReport.report_from_api(
-                api=vcproj,
-                title="HTML Reports",
-                report_type="INDEX_FILE",
-                formats=["HTML"],
-                output_file=output_file,
-                sections=['CUSTOM_HEADER', 'REPORT_TITLE', 'TABLE_OF_CONTENTS','INDEX_SECTION', 'CUSTOM_FOOTER'],
-                customization_dir=rep_path)
+    vcproj = VCProjectApi(mpName)
 
+    # Set custom report directory to the where this script was
+    # found. Must contain sections/index_section.py
+    rep_path = os.path.abspath(os.path.dirname(__file__))
+
+    output_file=os.path.join(baseOutputDir,"index.html")
+        
+    CustomReport.report_from_api(
+            api=vcproj,
+            title="HTML Reports",
+            report_type="INDEX_FILE",
+            formats=["HTML"],
+            output_file=output_file,
+            sections=['CUSTOM_HEADER', 'REPORT_TITLE', 'TABLE_OF_CONTENTS','INDEX_SECTION', 'CUSTOM_FOOTER'],
+            customization_dir=rep_path)
+    vcproj.close()
+   
 def create_index_html_body ():
 
     tempHtmlReportList =  glob.glob(os.path.join(baseOutputDir,"*.html"))

generate_lcov.py

@@ -144,9 +144,10 @@ def get_function_name_line_number(file_path, function, initial_guess):
 
 def runCoverageResultsMP(mpFile, verbose = False, testName = "", source_root = ""):
 
-    with VCProjectApi(mpFile) as vcproj:
-        api = vcproj.project.cover_api
-        results = runGcovResults(api, verbose = verbose, testName = vcproj.project.name, source_root=source_root)
+    vcproj = VCProjectApi(mpFile)
+    api = vcproj.project.cover_api
+    results = runGcovResults(api, verbose = verbose, testName = vcproj.project.name, source_root=source_root)
+    vcproj.close()
 
     return results
 

generate_pclp_reports.py

@@ -330,7 +330,6 @@ def generate_html_report(mpName, input_xml, output_html):
         print("{} was not found. Skipping PCLP HTML reporting".format(input_xml))
         return
 
-    import pathlib
     from vector.apps.DataAPI.vcproject_api import VCProjectApi
     from vector.apps.ReportBuilder.custom_report import CustomReport
 
@@ -340,19 +339,21 @@ def generate_html_report(mpName, input_xml, output_html):
     if output_html is None:
         output_html = "pclp_findings.html"
 
-    with VCProjectApi(mpName) as vcproj:
-
-        # Set custom report directory to the where this script was
-        # found. Must contain sections/index_section.py
-        rep_path = pathlib.Path(__file__).parent.resolve()
-        CustomReport.report_from_api(
-                api=vcproj,
-                title="PC-Lint Plus Results",
-                report_type="INDEX_FILE",
-                formats=["HTML"],
-                output_file=output_html,
-                sections=['CUSTOM_HEADER', 'REPORT_TITLE', 'TABLE_OF_CONTENTS','PCLP_SUMMARY_SECTION','PCLP_DETAILS_SECTION','PCLP_SOURCE_SECTION', 'CUSTOM_FOOTER'],
-                customization_dir=rep_path)
+    vcproj = VCProjectApi(mpName)
+
+    # Set custom report directory to the where this script was
+    # found. Must contain sections/index_section.py
+    rep_path = os.path.abspath(os.path.dirname(__file__))
+    CustomReport.report_from_api(
+            api=vcproj,
+            title="PC-Lint Plus Results",
+            report_type="INDEX_FILE",
+            formats=["HTML"],
+            output_file=output_html,
+            sections=['CUSTOM_HEADER', 'REPORT_TITLE', 'TABLE_OF_CONTENTS','PCLP_SUMMARY_SECTION','PCLP_DETAILS_SECTION','PCLP_SOURCE_SECTION', 'CUSTOM_FOOTER'],
+            customization_dir=rep_path)
+            
+    vcproj.close()
 
 def has_any_coverage(line):
 

generate_qa_results_xml.py

@@ -23,9 +23,6 @@
 #
 
 from __future__ import print_function
-
-# adding path
-
 import datetime
 try:
     from html import escape
@@ -210,36 +207,31 @@ def processSystemTestResultsData(lines, encoding = 'utf-8'):
 
     return passed, failed
 
-def saveQATestStatus(mp):
-    callStr = os.environ.get('VECTORCAST_DIR') + os.sep + "manage -p " + mp + " --system-tests-status=" + os.path.basename(mp)[:-4] + "_system_tests_status.html"
-    p = subprocess.Popen(callStr, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
-    out, err = p.communicate()
-
 def genQATestResults(mp, level = None, envName = None, verbose = False, encoding = 'utf-8'):
+    passed_count = 0
+    failed_count = 0
+
     try:
         from vector.apps.DataAPI.manage_models import SystemTest
-        if verbose:
-            print("No need to process system test results using --system-tests-status")
-        return
     except:
-        pass
-
-    print("   Processing QA test results for " + mp)
-    callStr = os.environ.get('VECTORCAST_DIR') + os.sep + "manage -p " + mp + " --system-tests-status"
-    if level:
-        callStr += " --level " + level
-        if envName:
-            callStr += " -e " + envName
-        
-    p = subprocess.Popen(callStr, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
-    out, err = p.communicate()
+        if verbose:
+            print("No QA Environment that can be processed using --system-tests-status")
+        return passed_count, failed_count
+
+    if level and envName:
+        nameLevel = level + "_" + envName
+        nameLevel = nameLevel.replace("\\","/").replace("/","_")
+        report_name = "{}_{}_system_tests_status.html".format(os.path.basename(mp)[:-4], nameLevel)
+    else:
+        report_name = os.path.basename(mp)[:-4] + "_system_tests_status.html"
+
+    if os.path.exists(report_name):
+        with open(report_name,"rb") as fd:
+            raw = fd.read()
+            out = raw.decode(encoding, 'replace')
+                            
+        passed_count, failed_count = processSystemTestResultsData(out.splitlines(), encoding)
 
-    if err:
-        print("{} {}".format(out, err))
-    passed_count, failed_count = processSystemTestResultsData(out.splitlines(), encoding)
-    
-    saveQATestStatus(mp)
-    
     return passed_count, failed_count
 
 if __name__ == '__main__':

generate_sonarqube_testresults.py

@@ -558,11 +558,11 @@ def generate_unit(self):
         if isinstance(self.api, CoverApi):
             try:
                 if self.topLevelAPI == None:
-                    api = VCProjectApi(self.FullManageProjectName)
+                    vcproj = VCProjectApi(self.FullManageProjectName)
                 else:
-                    api = self.topLevelAPI
+                    vcproj = self.topLevelAPI
 
-                for env in api.Environment.all():
+                for env in vcproj.Environment.all():
                     if env.compiler.name == self.compiler and env.testsuite.name == self.testsuite and env.name == self.env and env.system_tests:
                         for st in env.system_tests:
                             pass_fail_rerun = ""
@@ -580,12 +580,8 @@ def generate_unit(self):
                                 print ("{} {} {}".format(level, st.name, pass_fail_rerun))
                             self.write_testcase(st, level, st.name, env.definition.is_monitored)
 
-                # callStr = os.getenv('VECTORCAST_DIR') + os.sep + "manage -p " + self.FullManageProjectName + " --system-tests-status=" + self.manageProjectName + "_system_tests_status.html"
-                # p = subprocess.Popen(callStr, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
-                # out, err = p.communicate()
-
                 if self.topLevelAPI == None:
-                    api.close()
+                    vcproj.close()
 
             except ImportError as e:
                 from generate_qa_results_xml import genQATestResults

generate_xml.py

@@ -82,7 +82,32 @@ def __init__(self, cover_report_name, verbose, use_ci):
             self.use_ci = " --ci "
         else:
             self.use_ci = ""
+        self.system_tests_status_report_generated = False
 
+    def generate_system_test_status_report(self):
+        if self.system_tests_status_report_generated:
+            return
+
+        report_name = os.path.basename(self.FullManageProjectName)[:-4] + "_system_tests_status.html"
+
+        print("   Creating System Test Status " + self.FullManageProjectName)
+        callStr = os.environ.get('VECTORCAST_DIR') + os.sep + "manage -p " + self.FullManageProjectName + " --system-tests-status=" + report_name
+
+        import subprocess
+
+        print(callStr)
+        p = subprocess.Popen(callStr, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+        out, err = p.communicate()
+
+        if os.path.exists(report_name):
+            print("File exists: " + report_name)
+        else:
+            print("File not exists: " + report_name)
+
+        if err:
+            print("Cannot create system test status report{} {}".format(out, err))
+
+        self.system_tests_status_report_generated = True
 #
 # BaseGenerateXml - calculate coverage value
 #
@@ -542,7 +567,8 @@ def __init__(self, FullManageProjectName, build_dir, env, compiler, testsuite, c
         self.cbtDict = None
 
         if os.path.exists(cov_path) and os.path.exists(cov_path[:-4]):
-            self.using_cover = True
+            self.using_cover = True     
+            self.generate_system_test_status_report()            
             try:
                 self.api = CoverApi(cov_path)
             except:
@@ -621,18 +647,16 @@ def generate_unit(self):
                             if self.verbose:
                                 print (level, st.name, pass_fail_rerun)
                             self.write_testcase(st, level, st.name)
-                from generate_qa_results_xml import saveQATestStatus
-                saveQATestStatus(self.FullManageProjectName)
-
                 api.close()
 
             except ImportError as e:
-                from generate_qa_results_xml import genQATestResults
-                pc,fc = genQATestResults(self.FullManageProjectName, self.compiler+ "/" + self.testsuite, self.env, True, self.encFmt)
-                self.failed_count += fc
-                self.passed_count += pc
-                return
+                pass
 
+            from generate_qa_results_xml import genQATestResults
+            pc,fc = genQATestResults(self.FullManageProjectName, self.compiler + "/" + self.testsuite, self.env, True, self.encFmt)
+            self.failed_count += fc
+            self.passed_count += pc 
+                       
         else:
 
             try:

getjobs.py

@@ -162,12 +162,14 @@ def printEnvInfoNoDataAPI(ManageProjectName, printData = True, printEnvType = Fa
 
     max_indent = 0
     veryMax = 8
-
+    last_index_max_line = 0
+    
     for line in enabledList:
         indent = len(line) - len(line.lstrip())
 
         if indent > max_indent and indent <= veryMax:
             max_indent = indent
+            last_index_max_line = line
 
     if max_indent == 8:
         source_match_string    = "^   [^\s]"
@@ -190,6 +192,9 @@ def printEnvInfoNoDataAPI(ManageProjectName, printData = True, printEnvType = Fa
         group_match_string     = None
         env_match_string       = "^     [^\s]"
     else:
+        print("[DEBUG] Error with full-status report")
+        print("[DEBUG] Highest indent is with this line\n" + str(last_index_max_line))
+        print("\n".join(enabledList))
         raise ValueError("Error deciphering max_index: " +  str(max_indent))
 
     source = None
@@ -234,12 +239,12 @@ def printEnvironmentInfo(ManageProjectName, printData = True, printEnvType = Fal
     try:
 
         from vector.apps.DataAPI.vcproject_api import VCProjectApi
-        api = VCProjectApi(ManageProjectName)
-        ret_info = printEnvInfoDataAPI(api, printData, printEnvType)
-        api.close()
+
+        vcproj = VCProjectApi(ManageProjectName)
+        ret_info = printEnvInfoDataAPI(vcproj, printData, printEnvType)
+        vcproj.close()
         return ret_info
 
-    
     except:    
         return printEnvInfoNoDataAPI(ManageProjectName, printData, printEnvType)