diff --git a/CHANGELOG.md b/CHANGELOG.md index c0efbb5..3021ec7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.2.0] - Unreleased + +### Added +- #29: Track code coverage for embedded python methods in .cls files +- #42: Added a listener interface and manager with an associated user parameter, allowing the user to broadcast output on test method/case/suite completion. ## [3.1.1] - 2024-07-31 diff --git a/cls/TestCoverage/Data/CodeSubUnit.cls b/cls/TestCoverage/Data/CodeSubUnit.cls index 073c3c1..f8eb8ca 100644 --- a/cls/TestCoverage/Data/CodeSubUnit.cls +++ b/cls/TestCoverage/Data/CodeSubUnit.cls @@ -9,6 +9,9 @@ Property Mask As TestCoverage.DataType.Bitstring; /// Cyclomatic complexity of the code subunit Property Complexity As %Integer [ InitialExpression = 1 ]; +/// 1 if it's a python class method, 0 if not +Property IsPythonMethod As %Boolean; + Method UpdateComplexity() As %Status { Quit $$$OK @@ -26,6 +29,9 @@ Storage Default Complexity + +IsPythonMethod + {%%PARENT}("SubUnits") CodeSubUnitDefaultData @@ -36,4 +42,3 @@ Storage Default } } - diff --git a/cls/TestCoverage/Data/CodeUnit.cls b/cls/TestCoverage/Data/CodeUnit.cls index 7266342..0ac96c3 100644 --- a/cls/TestCoverage/Data/CodeUnit.cls +++ b/cls/TestCoverage/Data/CodeUnit.cls @@ -14,7 +14,7 @@ Index NameTypeHash On (Name, Type, Hash) [ Data = ExecutableLines, Unique ]; /// Name of the code unit Property Name As %String(MAXLEN = 255) [ Required ]; -/// Type (3-letter extension) of the code unit +/// Type (2 or 3-letter extension) of the code unit Property Type As TestCoverage.DataType.RoutineType [ Required ]; /// Hash of the code unit; for methods for determining this, see GetCurrentHash @@ -28,15 +28,23 @@ Property ExecutableLines As TestCoverage.DataType.Bitstring; /// For classes, map of method names in the code to their associated line numbers /// For routines, map of labels to associated line numbers +/// For python, map of method names to associated starting line number Property MethodMap As array Of %Integer; +/// Only for python: map of method names to associated ending line number of the method +Property MethodEndMap As array Of %Integer; + /// For classes, map of line numbers in code to associated method names /// For routines, map of labels to associated line numbers Property LineToMethodMap As array Of %Dictionary.CacheIdentifier [ Private ]; +/// For each line, whether or not it belongs to a python method, only populated for .cls CodeUnits +Property LineIsPython As array Of %Boolean; + /// Set to true if this class/routine is generated Property Generated As %Boolean [ InitialExpression = 0 ]; +/// /// Methods, branches, etc. within this unit of code. Relationship SubUnits As TestCoverage.Data.CodeSubUnit [ Cardinality = children, Inverse = Parent ]; @@ -98,10 +106,21 @@ ClassMethod GetCurrentByName(pInternalName As %String, pSourceNamespace As %Stri } Set $Namespace = pSourceNamespace + If (tType = "CLS") { Do ##class(TestCoverage.Utils).GetClassLineExecutableFlags(tName,.tCodeArray,.tExecutableFlags) - } Else { + } ElseIf ((tType = "INT") || (tType = "MAC")) { Do ##class(TestCoverage.Utils).GetRoutineLineExecutableFlags(.tCodeArray,.tExecutableFlags) + } ElseIf (tType="PY") { + Do ##class(TestCoverage.Utils).CodeArrayToList(.tCodeArray, .pDocumentText) + Set tExecutableFlagsPyList = ##class(TestCoverage.Utils).GetPythonLineExecutableFlags(pDocumentText) + Kill tExecutableFlags + for i=1:1:tExecutableFlagsPyList."__len__"()-1 { + set tExecutableFlags(i) = tExecutableFlagsPyList."__getitem__"(i) + } + } + Else { + return $$$ERROR($$$GeneralError,"File type not supported") } Set $Namespace = tOriginalNamespace @@ -110,56 +129,98 @@ ClassMethod GetCurrentByName(pInternalName As %String, pSourceNamespace As %Stri Set pCodeUnit.Type = tType Set pCodeUnit.Hash = tHash - Set tBitString = "" - For tLine=1:1:$Get(tCodeArray,0) { - Set $Bit(tBitString,tLine) = $Get(tExecutableFlags(tLine),0) - } - Set pCodeUnit.ExecutableLines = tBitString + If (tType = "CLS") { Set pCodeUnit.Generated = ($$$comClassKeyGet(tName,$$$cCLASSgeneratedby) '= "") + } - Set tMethod = "" - Set tMethodSignature = "" - Set tMethodMask = "" - For tLineNumber=1:1:$Get(tCodeArray,0) { - Set tLine = tCodeArray(tLineNumber) - Do pCodeUnit.Lines.Insert(tLine) - - If (tType = "CLS") { - // Extract line offset of methods in classes - Set tStart = $Piece(tLine," ") - If (tStart = "ClassMethod") || (tStart = "Method") { - Set tMethod = $Piece($Piece(tLine,"(")," ",2) - Set tMethodSignature = tLine - Do pCodeUnit.MethodMap.SetAt(tLineNumber,tMethod) - Do pCodeUnit.LineToMethodMap.SetAt(tMethod,tLineNumber) - } ElseIf ($Extract(tStart) = "{") { - // Ignore the opening bracket for a method. - } ElseIf ($Extract(tStart) = "}") && (tMethod '= "") { - // End of method. Add method subunit to class. - Set tSubUnit = ##class(TestCoverage.Data.CodeSubUnit.Method).%New() - Set tSubUnit.Name = tMethod - Set tSubUnit.DisplaySignature = tMethodSignature - Set tSubUnit.Mask = tMethodMask - Do pCodeUnit.SubUnits.Insert(tSubUnit) - Set tMethod = "" - Set tMethodSignature = "" - Set tMethodMask = "" - } ElseIf (tMethod '= "") { - Set $Bit(tMethodMask,tLineNumber) = 1 - } - } Else { - // Extract line offset of labels in routines - If ($ZStrip($Extract(tLine),"*PWC") '= "") { - Set tLabel = $Piece($Piece(tLine," "),"(") - Do pCodeUnit.MethodMap.SetAt(tLineNumber,tLabel) - Do pCodeUnit.LineToMethodMap.SetAt(tLabel,tLineNumber) + + If (tType = "PY") { + // fill in the Lines property of this CodeUnit + set tPointer = 0 + While $ListNext(pDocumentText, tPointer, tCurLine) { + do pCodeUnit.Lines.Insert(tCurLine) + } + + do pCodeUnit.Lines.Insert("") + + // Filling in the MethodMap and LineToMethodMap properties + Set tMethodInfo = ##class(TestCoverage.Utils).GetPythonMethodMapping(pDocumentText) + // tMethodInfo is a python tuple of (line to method info, method map info) + Set tLineToMethodInfo = tMethodInfo."__getitem__"(0) // a python builtins list where the item at index i is the name of the method that line i is a part of + Set tMethodMapInfo = tMethodInfo."__getitem__"(1) // a python builtins dict with key = method name, value = the line number of its definition + for i=1:1:$listlength(pDocumentText) { + Set tMethod = tLineToMethodInfo."__getitem__"(i) + Do pCodeUnit.LineToMethodMap.SetAt(tMethod,i) + } + Set iterator = tMethodMapInfo."__iter__"() + for i=1:1:tMethodMapInfo."__len__"() { // when iterator passes the last element, it throws a Python exception: StopIteration error, so I think the current pattern is preferable over that + Set tMethod = iterator."__next__"() + Set tStartEnd = tMethodMapInfo."__getitem__"(tMethod) + Set tStartLine = tStartEnd."__getitem__"(0) + Set tEndLine = tStartEnd."__getitem__"(1) + Do pCodeUnit.MethodMap.SetAt(tStartLine,tMethod) + Set tExecutableFlags(tStartLine) = 0 + Do pCodeUnit.MethodEndMap.SetAt(tEndLine, tMethod) + } + } + Else { + Set tMethod = "" + Set tMethodSignature = "" + Set tMethodMask = "" + For tLineNumber=1:1:$Get(tCodeArray,0) { + Set tLine = tCodeArray(tLineNumber) + Do pCodeUnit.Lines.Insert(tLine) + + If (tType = "CLS") { + // initialize each line to not python (we'll update this later) + Do pCodeUnit.LineIsPython.SetAt(0, tLineNumber) + + // Extract line offset of methods in classes + Set tStart = $Piece(tLine," ") + If (tStart = "ClassMethod") || (tStart = "Method") { + Set tMethod = $Piece($Piece(tLine,"(")," ",2) + Set tMethodSignature = tLine + Do pCodeUnit.MethodMap.SetAt(tLineNumber,tMethod) + Do pCodeUnit.LineToMethodMap.SetAt(tMethod,tLineNumber) + } ElseIf ($Extract(tStart) = "{") { + // Ignore the opening bracket for a method. + } ElseIf ($Extract(tStart) = "}") && (tMethod '= "") { + // End of method. Add method subunit to class. + Set tSubUnit = ##class(TestCoverage.Data.CodeSubUnit.Method).%New() + Set tSubUnit.Name = tMethod + Set tSubUnit.DisplaySignature = tMethodSignature + Set tSubUnit.Mask = tMethodMask + set NormalizedSignature = $zconvert($zstrip(tMethodSignature, "*W"), "l") + set tSubUnit.IsPythonMethod = (NormalizedSignature [ "[language=python]") + Do pCodeUnit.SubUnits.Insert(tSubUnit) + Set tMethod = "" + Set tMethodSignature = "" + Set tMethodMask = "" + } ElseIf (tMethod '= "") { + Set $Bit(tMethodMask,tLineNumber) = 1 + } + } + Else { + // Extract line offset of labels in routines + If ($ZStrip($Extract(tLine),"*PWC") '= "") { + Set tLabel = $Piece($Piece(tLine," "),"(") + Do pCodeUnit.MethodMap.SetAt(tLineNumber,tLabel) + Do pCodeUnit.LineToMethodMap.SetAt(tLabel,tLineNumber) + } } } } + Set tBitString = "" + For tLine=1:1:$Get(tCodeArray,0) { + Set $Bit(tBitString,tLine) = $Get(tExecutableFlags(tLine),0) + } + Set pCodeUnit.ExecutableLines = tBitString + + Set tSC = pCodeUnit.%Save() If $$$ISERR(tSC) && $System.Status.Equals(tSC,$$$ERRORCODE($$$IDKeyNotUnique)) { // Some other process beat us to it. @@ -183,10 +244,107 @@ ClassMethod GetCurrentByName(pInternalName As %String, pSourceNamespace As %Stri Quit tSC } +/// Fill in the LineIsPython property of .cls files +Method UpdatePythonLines(pName As %String, ByRef pPyCodeUnit) As %Status +{ + + Set tSC = $$$OK + Set tOriginalNamespace = $Namespace + Set tInitTLevel = $TLevel + + Try { + TSTART + + If (##class(TestCoverage.Manager).HasPython(pName)) { + + Set tFromHash = pPyCodeUnit.Hash + Set tToHash = ..Hash + set sql = "SELECT map.ToLine FROM TestCoverage_Data.CodeUnitMap map " _ + "JOIN TestCoverage_Data.CodeUnit fromCodeUnit " _ + "ON fromCodeUnit.Hash = map.FromHash " _ + "WHERE map.FromHash = ? " _ + "AND map.ToHash = ? " + set resultSet = ##class(%SQL.Statement).%ExecDirect(, sql, tFromHash, tToHash) + If (resultSet.%SQLCODE < 0) { + Throw ##class(%Exception.SQL).CreateFromSQLCODE(resultSet.%SQLCODE, resultSet.%Message) + } + while resultSet.%Next(.tSC) { + $$$ThrowOnError(tSC) + Set hToLine = resultSet.%GetData(1) + do ..LineIsPython.SetAt(1, hToLine) + } + If (resultSet.%SQLCODE < 0) { + Throw ##class(%Exception.SQL).CreateFromSQLCODE(resultSet.%SQLCODE, resultSet.%Message) + } + } + Set tSC = ..%Save() + $$$ThrowOnError(tSC) + + TCOMMIT + } Catch e { + Set pCodeUnit = $$$NULLOREF + Set tSC = e.AsStatus() + } + While ($TLevel > tInitTLevel) { + TROLLBACK 1 + } + Quit tSC +} + +/// Get the executable lines of code in python over to the .cls CodeUnit +Method UpdatePyExecutableLines(pName As %String, ByRef pPyCodeUnit) As %Status +{ + Set tSC = $$$OK + Set tOriginalNamespace = $Namespace + Set tInitTLevel = $TLevel + Try { + TSTART + + Set tBitString = "" + If (##class(TestCoverage.Manager).HasPython(pName)) { + + Set tFromHash = pPyCodeUnit.Hash + Set tToHash = ..Hash + set sql = "SELECT map.ToLine FROM TestCoverage_Data.CodeUnitMap map " _ + "JOIN TestCoverage_Data.CodeUnit fromCodeUnit " _ + "ON fromCodeUnit.Hash = map.FromHash " _ + "WHERE map.FromHash = ? " _ + "AND map.ToHash = ? " _ + "AND TestCoverage.BIT_VALUE(fromCodeUnit.ExecutableLines,map.FromLine) <> 0" + + set resultSet = ##class(%SQL.Statement).%ExecDirect(, sql, tFromHash, tToHash) + If (resultSet.%SQLCODE < 0) { + Throw ##class(%Exception.SQL).CreateFromSQLCODE(resultSet.%SQLCODE, resultSet.%Message) + } + while resultSet.%Next(.tSC) { + $$$ThrowOnError(tSC) + Set hToLine = resultSet.%GetData(1) + Set $Bit(tBitString, hToLine) = 1 + } + If (resultSet.%SQLCODE < 0) { + Throw ##class(%Exception.SQL).CreateFromSQLCODE(resultSet.%SQLCODE, resultSet.%Message) + } + } + Set ..ExecutableLines = $BITLOGIC(..ExecutableLines | tBitString) + Set tSC = ..%Save() + $$$ThrowOnError(tSC) + + TCOMMIT + } Catch e { + Set pCodeUnit = $$$NULLOREF + Set tSC = e.AsStatus() + } + While ($TLevel > tInitTLevel) { + TROLLBACK 1 + } + Quit tSC +} + Method UpdateSourceMap(pSourceNamespace As %String, ByRef pCache) As %Status { Set tSC = $$$OK Try { + // First, build local array (tMap) of all maps from the .INT file to other files. If (..Type = "INT") { For tLineNumber=1:1:..Lines.Count() { @@ -225,6 +383,39 @@ Method UpdateSourceMap(pSourceNamespace As %String, ByRef pCache) As %Status } } } + If (..Type = "PY") { + + set tClass = ..Name + Set tSourceUnits(tClass_".CLS") = "" + // we'll need the MethodMap from the .CLS CodeUnit to figure out the line mappings + $$$ThrowOnError(..GetCurrentByName(tClass _ ".CLS", pSourceNamespace, .pCLSCodeUnit, .pCLSCache)) + // we'll do the mappings from the .py to the .cls direction, so that we don't iterate over objectscript lines + Set tMethod = "" + Do ..MethodMap.GetNext(.tMethod) + while (tMethod '= "") + { + // for each method in the .py file, we'll find the line number of the corresponding method (guaranteed to be unique) in the .cls file + // and then map each line in the .py file to each line in the .cls file by just going 1 by 1 down the lines + Set tCLSMethodNum = pCLSCodeUnit.MethodMap.GetAt(tMethod) + Set tMethodStart = ..MethodMap.GetAt(tMethod) + Set tMethodEnd = ..MethodEndMap.GetAt(tMethod) + Set tMethodName = tMethod + Set tFullMap(tMethodStart) = $lb("CLS", tClass,tMethodName, -1, -1) ; -1 because the class + ; definition doesn't have the +1 offset from the { + For i = tMethodStart+1:1:tMethodEnd { + Set tClassLineNum = i-tMethodStart + Set tFullMap(i) = $lb("CLS", tClass,tMethodName, tClassLineNum, tClassLineNum) + + // extra check to make sure that the lines we're mapping between are the same as expected + Set tClassLineCode = $zstrip(pCLSCodeUnit.Lines.GetAt(tCLSMethodNum + tClassLineNum + 1), "<>W") + Set tPyLineCode = $zstrip(..Lines.GetAt(i), "<>W") + if (tPyLineCode '= tClassLineCode) { + Set tSC = $$$ERROR($$$GeneralError,"Compiled .py code doesn't match .CLS python code at line " _ $char(10,13) _ tPyLineCode) + } + } + Do ..MethodMap.GetNext(.tMethod) + } + } // If we are a generator .INT file, ensure that we have source for the original class populated. // In such files, the second line looks like (for example): @@ -270,7 +461,7 @@ Method UpdateSourceMap(pSourceNamespace As %String, ByRef pCache) As %Status Set tCodeUnits(tCodeUnit.Type,tCodeUnit.Name) = tCodeUnit } - // Create CodeUnitMap data based on .INT->.CLS mapping. + // Create CodeUnitMap data based on .INT / .py ->.CLS mapping. Set tFromHash = ..Hash Set tLineNumber = "" For { @@ -349,15 +540,28 @@ Method UpdateComplexity() As %Status If (..Type '= "CLS") { Quit } - + + // python methods + If (##class(TestCoverage.Manager).HasPython(..Name)) { + do ##class(TestCoverage.Data.CodeUnit).GetCurrentByName(..Name _ ".PY", , .pPyCodeUnit, ) // need the source code for the python + set tDocumentText = pPyCodeUnit.Lines.Serialize() + set tMethodComplexities = ..GetPythonComplexities(tDocumentText) + } + Set tKey = "" For { Set tSubUnit = ..SubUnits.GetNext(.tKey) If (tKey = "") { Quit } - $$$ThrowOnError(tSubUnit.UpdateComplexity()) + If (tSubUnit.IsPythonMethod) { + set tSubUnit.Complexity = tMethodComplexities."__getitem__"(tSubUnit.Name) + $$$ThrowOnError(tSubUnit.%Save(0)) + } Else { + $$$ThrowOnError(tSubUnit.UpdateComplexity()) + } } + $$$ThrowOnError(..%Save()) } Catch e { @@ -366,6 +570,21 @@ Method UpdateComplexity() As %Status Quit tSC } +/// returns a python dict with (key, value) = (method name, complexity) for each python method +ClassMethod GetPythonComplexities(pDocumentText) [ Language = python ] +{ + from radon.complexity import cc_visit + import iris + source_lines = iris.cls('%SYS.Python').ToList(pDocumentText) + source_code = "\n".join(source_lines) + visitor = cc_visit(source_code) + class_info = visitor[0] + method_complexities = {} + for method in class_info.methods: + method_complexities[method.name] = method.complexity + return method_complexities +} + Method GetMethodOffset(pAbsoluteLine As %Integer, Output pMethod As %String, Output pOffset As %Integer) { } @@ -396,6 +615,10 @@ ClassMethod GetCurrentHash(pName As %String, pType As %String, Output pHash As % // Skip header (lines 1-4) which, for .INT routines generated from classes, // includes the class compilation signature. Set pHash = ..HashArrayRange(.pCodeArray,5,pName_"."_pType,.tSizeHint) + } ElseIf (pType = "PY") { + Merge pCodeArray = ^ROUTINE(pName_".py",0) // the python source code + set tSizeHint = ^ROUTINE(pName_".py",0,0) // the number of lines in the python code + set pHash = ..HashArrayRange(.pCodeArray, ,pName_".py", .tSizeHint) } Else { // Give standard descriptive error about the type being invalid. $$$ThrowStatus(..TypeIsValid(pType)) @@ -468,6 +691,11 @@ Storage Default Generated + +LineIsPython +subnode +"LineIsPython" + LineToMethodMap subnode @@ -478,6 +706,11 @@ Storage Default subnode "Lines" + +MethodEndMap +subnode +"MethodEndMap" + MethodMap subnode @@ -492,4 +725,3 @@ Storage Default } } - diff --git a/cls/TestCoverage/Data/Coverage.cls b/cls/TestCoverage/Data/Coverage.cls index 9c8e2be..237ca90 100644 --- a/cls/TestCoverage/Data/Coverage.cls +++ b/cls/TestCoverage/Data/Coverage.cls @@ -1,3 +1,5 @@ +Include TestCoverage + IncludeGenerator TestCoverage Class TestCoverage.Data.Coverage Extends %Persistent @@ -50,14 +52,14 @@ Property Time As array Of TestCoverage.DataType.Timing [ SqlFieldName = _TIME ]; /// List of "TotalTime" measurements from line-by-line monitor, subscripted by line number Property TotalTime As array Of TestCoverage.DataType.Timing; -ClassMethod StoreIntCoverage(pRun As %Integer, pTestPath As %String, pRoutineName As %String, ByRef pCache) As %Status +ClassMethod StoreIntCoverage(pRun As %Integer, pTestPath As %String, pName As %String, pType As %String, ByRef pCache) As %Status { + // pType must be either INT or PY Set tSC = $$$OK Try { #dim tResult As %SQL.StatementResult - Set tSC = ##class(TestCoverage.Data.CodeUnit).GetCurrentByName(pRoutineName_".INT",,.tCodeUnit,.pCache) + Set tSC = ##class(TestCoverage.Data.CodeUnit).GetCurrentByName(pName_"."_pType,,.tCodeUnit,.pCache) $$$ThrowOnError(tSC) - If ..UniqueCoverageDataExists(pRun,tCodeUnit.Hash,pTestPath,.tID) { Set tInstance = ..%OpenId(tID,,.tSC) $$$ThrowOnError(tSC) @@ -67,36 +69,57 @@ ClassMethod StoreIntCoverage(pRun As %Integer, pTestPath As %String, pRoutineNam $$$ThrowOnError(tSC) Set tInstance.TestPath = pTestPath Set tInstance.Hash = tCodeUnit + For tLineNumber=1:1:tCodeUnit.Lines.Count() { + do tInstance.RtnLine.SetAt(0, tLineNumber) // initialized to 0 hits of each line + // necessary for the python coverages because they don't track lines that aren't covered, only lines that are covered + } } Set tCoveredLines = tInstance.CoveredLines - - Set tAvailableMetrics = ..GetAvailableMetrics() - Set tPointer = 0 - While $ListNext(tAvailableMetrics,tPointer,tMetricKey) { - If tInstance.Run.Metrics.Find(tMetricKey) { - Set tMetrics(tMetricKey) = $Property(tInstance,tMetricKey) + if (pType = "INT") + { + Set tAvailableMetrics = ..GetAvailableMetrics() + Set tPointer = 0 + While $ListNext(tAvailableMetrics,tPointer,tMetricKey) { + If tInstance.Run.Metrics.Find(tMetricKey) { + Set tMetrics(tMetricKey) = $Property(tInstance,tMetricKey) + } } - } - - // ROWSPEC = "LineNumber:%Integer,LineCovered:%Boolean,RtnLine:%Integer,Time:%Numeric,TotalTime:%Numeric" - Set tResult = ##class(TestCoverage.Utils).LineByLineMonitorResultFunc(pRoutineName) - While tResult.%Next(.tSC) { - $$$ThrowOnError(tSC) - Set tLineNumber = tResult.%Get("LineNumber") - If tResult.%Get("LineCovered") { - Set $Bit(tCoveredLines,tLineNumber) = 1 + // ROWSPEC = "LineNumber:%Integer,LineCovered:%Boolean,RtnLine:%Integer,Time:%Numeric,TotalTime:%Numeric" + Set tResult = ##class(TestCoverage.Utils).LineByLineMonitorResultFunc(pName) + While tResult.%Next(.tSC) { + $$$ThrowOnError(tSC) + Set tLineNumber = tResult.%Get("LineNumber") + If tResult.%Get("LineCovered") { + Set $Bit(tCoveredLines,tLineNumber) = 1 + } + Set tMetricKey = "" + For { + Set tMetricKey = $Order(tMetrics(tMetricKey),1,tMetric) + If (tMetricKey = "") { + Quit + } + Do tMetric.SetAt(tResult.%Get(tMetricKey) + tMetric.GetAt(tLineNumber),tLineNumber) + } } - Set tMetricKey = "" - For { - Set tMetricKey = $Order(tMetrics(tMetricKey),1,tMetric) - If (tMetricKey = "") { - Quit + $$$ThrowOnError(tSC) + } + Else { // If pType = "PY" + // $$$PyMonitorResults(classname, linenumber) = the number of times that linenumber in that class was covered + + if $Data($$$PyMonitorResults(pName)) { + Set tLine = "" + for { + Set tLine = $Order($$$PyMonitorResults(pName, tLine), 1, tLineCount) + if (tLine = "") { + quit + } + Set $Bit(tCoveredLines, tLine) = 1 + Do tInstance.RtnLine.SetAt(tInstance.RtnLine.GetAt(tLine) + tLineCount, tLine) } - Do tMetric.SetAt(tResult.%Get(tMetricKey) + tMetric.GetAt(tLineNumber),tLineNumber) } + } - $$$ThrowOnError(tSC) Set tInstance.CoveredLines = $BitLogic(tInstance.CoveredLines|tCoveredLines) @@ -167,4 +190,3 @@ Storage Default } } - diff --git a/cls/TestCoverage/Data/Run.cls b/cls/TestCoverage/Data/Run.cls index 09dadc9..1afd79d 100644 --- a/cls/TestCoverage/Data/Run.cls +++ b/cls/TestCoverage/Data/Run.cls @@ -101,6 +101,17 @@ ClassMethod MapRunCoverage(pRunIndex As %Integer) As %Status Do tCoverage.RunSetObjectId(pRunIndex) Do tCoverage.HashSetObjectId(hToHash) Set tCoverage.TestPath = hTestPath + // also set all of its metrics to 0 to start with + Set tCodeUnit = ##class(TestCoverage.Data.CodeUnit).%OpenId(hToHash) + For i=1:1:tRun.Metrics.Count() { + Set tMetricKey = tRun.Metrics.GetAt(i) + Set tMetric = $PROPERTY(tCoverage, tMetricKey) + for tLineNumber = 1:1:tCodeUnit.Lines.Count() { + Do tMetric.SetAt(0, tLineNumber) + } + } + + } Set tCoverage.Ignore = hIgnore Set tCoverage.CoveredLines = $BitLogic(tCoverage.CoveredLines|hCoveredLines) @@ -191,4 +202,3 @@ Storage Default } } - diff --git a/cls/TestCoverage/DataType/RoutineType.cls b/cls/TestCoverage/DataType/RoutineType.cls index 4dd999a..11b4cc9 100644 --- a/cls/TestCoverage/DataType/RoutineType.cls +++ b/cls/TestCoverage/DataType/RoutineType.cls @@ -3,7 +3,6 @@ Class TestCoverage.DataType.RoutineType Extends %String [ ClassType = datatype ] Parameter MAXLEN = 3; -Parameter VALUELIST = ",CLS,MAC,INT"; +Parameter VALUELIST = ",CLS,MAC,INT,PY"; } - diff --git a/cls/TestCoverage/Listeners/ListenerInterface.cls b/cls/TestCoverage/Listeners/ListenerInterface.cls new file mode 100644 index 0000000..44479a7 --- /dev/null +++ b/cls/TestCoverage/Listeners/ListenerInterface.cls @@ -0,0 +1,8 @@ +Class TestCoverage.Listeners.ListenerInterface Extends %RegisteredObject +{ + +Method Broadcast(pMessage As %DynamicObject) As %Status [ Abstract ] +{ +} + +} diff --git a/cls/TestCoverage/Listeners/ListenerManager.cls b/cls/TestCoverage/Listeners/ListenerManager.cls new file mode 100644 index 0000000..a76f64b --- /dev/null +++ b/cls/TestCoverage/Listeners/ListenerManager.cls @@ -0,0 +1,48 @@ +Class TestCoverage.Listeners.ListenerManager Extends %RegisteredObject +{ + +Property listeners As list Of TestCoverage.Listeners.ListenerInterface; + +Method BroadCastToAll(pMessage As %DynamicObject) As %Status +{ + set tSC = $$$OK + try { + for i = 1:1:..listeners.Count() { + set tListener = ..listeners.GetAt(i) + $$$ThrowOnError(tListener.Broadcast(pMessage)) + } + } + catch e { + Set tSC = e.AsStatus() + } + quit tSC +} + +Method AddListener(pListener As TestCoverage.Listeners.ListenerInterface) As %Status +{ + set tSC = $$$OK + try { + do ..listeners.Insert(pListener) + } catch e { + set tSC = e.AsStatus() + } + quit tSC +} + +Method RemoveListener(pListener As TestCoverage.Listeners.ListenerInterface) As %Status +{ + set tSC = $$$OK + try { + set tIndex = ..listeners.FindOref(pListener) + if (tIndex = "") { + Set tMsg = "Listener not found" + $$$ThrowStatus($$$ERROR($$$GeneralError,tMsg)) + } + do ..listeners.RemoveAt(tIndex) + } catch e { + set tSC = e.AsStatus() + } + quit tSC +} + +} diff --git a/cls/TestCoverage/Manager.cls b/cls/TestCoverage/Manager.cls index 13aec20..47514ad 100644 --- a/cls/TestCoverage/Manager.cls +++ b/cls/TestCoverage/Manager.cls @@ -26,9 +26,12 @@ Property Timing As %Boolean [ InitialExpression = 0, Internal, Private ]; /// Set to true (1) if coverage targets should be loaded dynamically from the unit test root. Property DynamicTargets As %Boolean [ InitialExpression = 1, Internal, Private ]; -/// Current list of targets (routines, classes, .int code, etc.) for line-by-line monitoring +/// Current list of objectscript targets (routines, classes, .int code, etc.) for line-by-line monitoring Property CoverageTargets As %List [ Internal, Private ]; +/// Current list of compiled Python targets for line-by-line monitoring +Property PyCoverageTargets As %List [ Internal, Private ]; + /// All classes considered at any point during this unit test run /// Top-level node has $ListBuild list; also has subscripts with individual class names for a quicker lookup Property CoverageClasses As %List [ Internal, MultiDimensional, Private ]; @@ -37,6 +40,10 @@ Property CoverageClasses As %List [ Internal, MultiDimensional, Private ]; /// Top-level node has $ListBuild list; also has subscripts with individual routine names for a quicker lookup Property CoverageRoutines As %List [ Internal, MultiDimensional, Private ]; +/// All compiled python files considered at any point during this unit test run +/// Top-level node has $ListBuild list; also has subscripts with individual routine names for a quicker lookup +Property CoveragePythons As %List [ Internal, MultiDimensional, Private ]; + /// Last known coverage.list file Property LastCoverageListFile As %String(MAXLEN = "") [ Internal, Private ]; @@ -59,11 +66,18 @@ Property Run As TestCoverage.Data.Run; /// Value at subscript is set to 1 if there are executable lines of code in the target, 0 if not. Property KnownCoverageTargets [ MultiDimensional, Private ]; +/// Known python coverage targets (already snapshotted).
+/// Value at subscript is set to 1 if there are executable lines of code in the target, 0 if not. +Property KnownPyCoverageTargets [ MultiDimensional, Private ]; + /// Cache of (name, type) -> hash Property Hashes [ MultiDimensional ]; Property Monitor As TestCoverage.Utils.LineByLineMonitor [ InitialExpression = {##class(TestCoverage.Utils.LineByLineMonitor).%New()}, Private ]; +/// keeps track of all the listeners that may need information broadcasted about the unit test progress +Property ListenerManager As TestCoverage.Listeners.ListenerManager; + /// Runs unit tests that have been loaded, with code coverage enabled.
/// Note that if coverage is to be tracked for lots of code, it may be necessary to increase the "gmheap" setting /// (under Configuration - Additional Settings - Advanced Memory in the Management Portal).
@@ -142,19 +156,31 @@ ClassMethod ArrayToList(pDynamicArray As %DynamicArray) As %List Quit tList } +/// write ##class(TestCoverage.Manager).HasPython("EP.sysSettrace") +/// 1 +ClassMethod HasPython(pClassName As %String) As %Boolean +{ + return $data(^ROUTINE(pClassName _ ".py", 0, 0)) > 0 +} + Method SetCoverageTargets(pClasses As %List = "", pRoutines As %List = "", pInit As %Boolean = 0) [ Private ] { Set tList = "", tPtr = 0 + Set tPyList = "" While $ListNext(pClasses,tPtr,tClass) { // Use a wildcard to include all .int files associated with the class. Set tList = tList_$ListBuild(tClass_".CLS") Do ..AddCoverageClass(tClass) + If (..HasPython(tClass)) { + set tPyList = tPyList _ $ListBuild(tClass) + } } While $ListNext(pRoutines,tPtr,tRoutine) { Set tList = tList_$ListBuild(tRoutine_".MAC") Do ..AddCoverageRoutine(tRoutine) } Set ..CoverageTargets = ..GetObjectCodeForSourceNames(tList) + Set ..PyCoverageTargets = tPyList If pInit { // Set flag to determine code coverage dynamically. Set ..DynamicTargets = (tList = "") @@ -210,7 +236,7 @@ Method StartCoverageTracking() As %Status [ Private ] Set tSC = $$$OK New $Namespace Try { - If (..CoverageTargets '= "") { + If ((..CoverageTargets '= "" ) || (..PyCoverageTargets '= "")) { Set $Namespace = ..SourceNamespace Set tRelevantTargets = "" Set tNewTargets = "" @@ -222,13 +248,23 @@ Method StartCoverageTracking() As %Status [ Private ] Set tRelevantTargets = tRelevantTargets_$ListBuild(tCoverageTarget) } } + + Set tPyRelevantTargets = "" + Set tNewPyTargets = "" + Set tPointer = 0 + While $ListNext(..PyCoverageTargets,tPointer,tPyCoverageTarget) { + If '$Data(..KnownPyCoverageTargets(tPyCoverageTarget),tIsRelevant)#2 { + Set tNewPyTargets = tNewPyTargets_$ListBuild(tPyCoverageTarget) + } ElseIf tIsRelevant { + Set tPyRelevantTargets = tPyRelevantTargets_$ListBuild(tPyCoverageTarget) + } + } - If (tNewTargets '= "") { + If ((tNewTargets '= "") || (tNewPyTargets '= "")) { $$$StartTimer("Taking snapshot of code and CLS/MAC/INT mappings") - Set tSC = ##class(TestCoverage.Utils).Snapshot(tNewTargets, .tNewRelevantTargets) + Set tSC = ##class(TestCoverage.Utils).Snapshot(tNewTargets, tNewPyTargets, .tNewRelevantTargets, .tNewPyRelevantTargets) $$$StopTimer $$$ThrowOnError(tSC) - Set tPointer = 0 While $ListNext(tNewTargets,tPointer,tNewTarget) { Set ..KnownCoverageTargets(tNewTarget) = 0 @@ -239,9 +275,20 @@ Method StartCoverageTracking() As %Status [ Private ] Set ..KnownCoverageTargets(tRelevantTarget) = 1 Set tRelevantTargets = tRelevantTargets_$ListBuild(tRelevantTarget) } + + Set tPointer = 0 + While $ListNext(tNewPyTargets,tPointer,tNewPyTarget) { + Set ..KnownPyCoverageTargets(tNewPyTarget) = 0 + } + + Set tPointer = 0 + While $ListNext(tNewPyRelevantTargets,tPointer,tPyRelevantTarget) { + Set ..KnownPyCoverageTargets(tPyRelevantTarget) = 1 + Set tPyRelevantTargets = tPyRelevantTargets_$ListBuild(tPyRelevantTarget) + } } - If (tRelevantTargets = "") { + If ((tRelevantTargets = "") && (tPyRelevantTargets = "")) { Write !,"WARNING: Nothing found to monitor for routine(s): "_$ListToString(tNewTargets) } @@ -275,7 +322,7 @@ Method StartCoverageTracking() As %Status [ Private ] do ..GetInteropProcesses(.tProcessIDs) } Set tMetrics = $ListBuild("RtnLine") _ $Select(..Timing:$ListBuild("Time","TotalTime"),1:"") - $$$ThrowOnError(..Monitor.StartWithScope(tRelevantTargets,tMetrics,tProcessIDs)) + $$$ThrowOnError(..Monitor.StartWithScope(tRelevantTargets, tPyRelevantTargets, tMetrics,tProcessIDs)) } } Catch e { Set tSC = e.AsStatus() @@ -316,6 +363,7 @@ Method UpdateCoverageTargetsForTestDirectory(pDirectory As %String) As %Status [ // If we found it, read in coverage list from there. Set tCoverageTargetList = "" + Set tPyCoverageTargetList = "" If (tListFile '= "") { Do ..PrintLine("Tracking code coverage on resources listed in "_tListFile) Do ..GetCoverageTargetsForFile(tListFile, .tCoverageTargets) @@ -331,15 +379,18 @@ Method UpdateCoverageTargetsForTestDirectory(pDirectory As %String) As %Status [ If (tType = "CLS") { Do ..AddCoverageClass(tCoverageTargetKey) + If (..HasPython(tCoverageTargetKey)) { + Set tPyCoverageTargetList = $get(tPyCoverageTargetList) _ $ListBuild(tCoverageTargetKey) + } } Else { Do ..AddCoverageRoutine(tCoverageTargetKey) } } } } - Set tObjectCodeList = ..GetObjectCodeForSourceNames(tCoverageTargetList) Set ..CoverageTargets = tObjectCodeList // Also restarts the monitor if it is running and updates data on covered routines/classes + Set ..PyCoverageTargets = tPyCoverageTargetList // no need to get the compiled names, it's already correct } Catch e { Set tSC = e.AsStatus() } @@ -351,6 +402,10 @@ Method AddCoverageClass(pClassName As %Dictionary.CacheClassname) [ Private ] If '$Data(..CoverageClasses(pClassName)) { Set ..CoverageClasses = $Get(..CoverageClasses) _ $ListBuild(pClassName) Set ..CoverageClasses(pClassName) = "" + If (..HasPython(pClassName)) { + Set ..CoveragePythons = $Get(..CoveragePythons) _ $ListBuild(pClassName) + Set ..CoveragePythons(pClassName) = "" + } } } @@ -462,7 +517,7 @@ Method GetObjectCodeForSourceNames(pSourceNameList As %List) As %List [ Private While $ListNext(tOthers,tOtherPointer,tOtherName) { Set tOutputNameList = tOutputNameList_$ListBuild($Piece(tOtherName,".",1,*-1)) } - } +} Quit tOutputNameList } @@ -470,9 +525,10 @@ Method EndCoverageTracking(pTestSuite As %String = "", pTestClass As %String = " { Set tSC = $$$OK Try { - If (..CoverageTargets '= "") { + If ((..CoverageTargets '= "") || (..PyCoverageTargets '= "")) { // Pause the monitor. Set tSC = ..Monitor.Pause() + Do ##class(TestCoverage.Utils.LineByLineMonitor).PyStop() If $$$ISERR(tSC) { If $System.Status.GetErrorCodes(tSC) = $$$MonitorNotRunning { // Not really an error, and nothing to do in this case. @@ -503,10 +559,18 @@ Method EndCoverageTracking(pTestSuite As %String = "", pTestClass As %String = " Set tRtnCount = ..Monitor.GetRoutineCount() For i=1:1:tRtnCount { Set tRtnName = ..Monitor.GetRoutineName(i) - Set tSC = ##class(TestCoverage.Data.Coverage).StoreIntCoverage(tTestIndex,tTarget,tRtnName,.tCache) + Set tSC = ##class(TestCoverage.Data.Coverage).StoreIntCoverage(tTestIndex,tTarget,tRtnName,"INT",.tCache) + $$$ThrowOnError(tSC) + } + + Set tPyClasses = ..Monitor.PythonClassList + Set tPointer = 0 + While $ListNext(tPyClasses, tPointer, tClass) { + Set tSC = ##class(TestCoverage.Data.Coverage).StoreIntCoverage(tTestIndex,tTarget,tClass,"PY",.tCache) $$$ThrowOnError(tSC) } Merge ..Hashes = tCache + $$$StopTimer } } Catch e { @@ -563,6 +627,7 @@ ClassMethod OnBeforeAllTests(manager As TestCoverage.Manager, dir As %String, By Set tCoverageRoutines = $Get(userparam("CoverageRoutines")) Set tCoverageDetail = $Get(userparam("CoverageDetail")) Set tSourceNamespace = $Get(userparam("SourceNamespace"),$Namespace) + Set tListenerManager = $Get(userparam("ListenerManager")) Set tProcessIDs = $Get(userparam("ProcessIDs"),$ListBuild($Job)) If (tProcessIDs = "*") { Set tProcessIDs = "" @@ -584,6 +649,7 @@ ClassMethod OnBeforeAllTests(manager As TestCoverage.Manager, dir As %String, By Set manager.ProcessIDs = tProcessIDs Set manager.Timing = tTiming Do manager.SetCoverageTargets(tCoverageClasses,tCoverageRoutines,1) + Set manager.ListenerManager = tListenerManager If (tCoverageDetail '= "") { If (tCoverageDetail '= +tCoverageDetail) { // If we were passed a display value... @@ -619,6 +685,10 @@ ClassMethod OnBeforeAllTests(manager As TestCoverage.Manager, dir As %String, By } If (manager.CoverageDetail = 0) { + if (manager.ListenerManager) { + set tObj = {"message": "Starting tests"} + Do manager.ListenerManager.BroadCastToAll(tObj) + } Set tSC = manager.StartCoverageTracking() $$$ThrowOnError(tSC) } @@ -637,6 +707,10 @@ ClassMethod OnAfterAllTests(manager As TestCoverage.Manager, dir As %String, ByR Try { If (manager.CoverageDetail = 0) { Set tSC = manager.EndCoverageTracking() + if (manager.ListenerManager) { + set tObj = {"message": "All tests complete"} + Do manager.ListenerManager.BroadCastToAll(tObj) + } } Do manager.Monitor.Stop() } Catch e { @@ -673,9 +747,15 @@ Method OnBeforeTestSuite(dir As %String, suite As %String, testspec As %String, Set ..CurrentTestSuite = $Case(suite,"":"(root)",:suite) Set ..CurrentTestClass = "" Set ..CurrentTestMethod = "" + if (..ListenerManager) { + set tObj = {"message": "Starting test suite: "} + do tObj.%Set("suite", suite) + Do ..ListenerManager.BroadCastToAll(tObj) + } If (..CoverageDetail = 1) { Set tSC = ..StartCoverageTracking() } + } Catch e { Set tSC = e.AsStatus() } @@ -688,9 +768,15 @@ Method OnAfterTestSuite(dir As %String, suite As %String, testspec As %String, B { Set tSC = $$$OK Try { + If (..CoverageDetail = 1) { Set tSC = ..EndCoverageTracking($Case(suite,"":"(root)",:suite)) } + if (..ListenerManager) { + set tObj = {"message": "Finished test suite: "} + do tObj.%Set("suite", suite) + Do ..ListenerManager.BroadCastToAll(tObj) + } } Catch e { Set tSC = e.AsStatus() } @@ -705,6 +791,12 @@ Method OnBeforeTestCase(suite As %String, class As %String) As %Status Try { Set ..CurrentTestClass = class Set ..CurrentTestMethod = "" + if (..ListenerManager) { + set tObj = {"message": "Starting test case: "} + do tObj.%Set("suite", suite) + do tObj.%Set("class", class) + Do ..ListenerManager.BroadCastToAll(tObj) + } If (..CoverageDetail = 2) { Set tSC = ..StartCoverageTracking() } @@ -723,6 +815,12 @@ Method OnAfterTestCase(suite As %String, class As %String) As %Status If (..CoverageDetail = 2) { Set tSC = ..EndCoverageTracking(suite, class) } + if (..ListenerManager) { + set tObj = {"message": "Finished test case: "} + do tObj.%Set("suite", suite) + do tObj.%Set("class", class) + Do ..ListenerManager.BroadCastToAll(tObj) + } } Catch e { Set tSC = e.AsStatus() } @@ -736,6 +834,13 @@ Method OnBeforeOneTest(suite As %String, class As %String, method As %String) As Set tSC = $$$OK Try { Set ..CurrentTestMethod = method + if (..ListenerManager) { + set tObj = {"message": "Starting test method: "} + do tObj.%Set("suite", suite) + do tObj.%Set("class", class) + do tObj.%Set("method", method) + Do ..ListenerManager.BroadCastToAll(tObj) + } If (..CoverageDetail = 3) { Set tSC = ..StartCoverageTracking() } @@ -754,6 +859,13 @@ Method OnAfterOneTest(suite As %String, class As %String, method As %String) As If (..CoverageDetail = 3) { Set tSC = ..EndCoverageTracking(suite, class, method) } + if (..ListenerManager) { + set tObj = {"message": "Finished test method: "} + do tObj.%Set("suite", suite) + do tObj.%Set("class", class) + do tObj.%Set("method", method) + Do ..ListenerManager.BroadCastToAll(tObj) + } } Catch e { Set tSC = e.AsStatus() } diff --git a/cls/TestCoverage/Utils.cls b/cls/TestCoverage/Utils.cls index 2cd8662..16ddf19 100644 --- a/cls/TestCoverage/Utils.cls +++ b/cls/TestCoverage/Utils.cls @@ -78,7 +78,7 @@ ClassMethod GetTestCoverageTableList() As %List /// This is parallelized using %SYSTEM.WorkMgr for better performance.
/// pRelevantRoutines is a $ListBuild list of .INT routines that map back to a .CLS or .MAC /// routine with at least one executable line. -ClassMethod Snapshot(pIntRoutines As %List, Output pRelevantRoutines As %List = "") As %Status +ClassMethod Snapshot(pIntRoutines As %List, pPyRoutines As %List, Output pRelevantRoutines As %List = "", Output pPyRelevantRoutines As %List = "") As %Status { Set tSC = $$$OK Try { @@ -94,7 +94,9 @@ ClassMethod Snapshot(pIntRoutines As %List, Output pRelevantRoutines As %List = Set tSC = tSnapshotQueue.WaitForComplete() $$$ThrowOnError(tSC) - + + + // See which routines are actually relevant (one or more lines mapping back to a class with 1 or more executable lines) // There's no point in optimizing out .MAC routines; they'll always have code Set tPointer = 0 @@ -102,18 +104,58 @@ ClassMethod Snapshot(pIntRoutines As %List, Output pRelevantRoutines As %List = Set tOther = ##class(%Library.RoutineMgr).GetOther(tIntRoutine,"INT",-1) If (tOther '= "") && ($Piece(tOther,".",*) = "CLS") { // With the code already cached, this will be faster. + // This also snapshots the compiled python routine with it if there is one #dim tCodeUnit As TestCoverage.Data.CodeUnit Set tSC = ##class(TestCoverage.Data.CodeUnit).GetCurrentByName(tOther,,.tCodeUnit) + If $$$ISERR(tSC) { Continue // Non-fatal. Just skip it. } - If '$BitCount(tCodeUnit.ExecutableLines,1) { + Set tName = tCodeUnit.Name // should be the same as tOther without the .cls, but if I have it already why not + Set SnapshottedClasses(tName) = 1 + If ##class(TestCoverage.Manager).HasPython(tName) { + // take a snapshot of the compiled python file + $$$ThrowOnError(##class(TestCoverage.Data.CodeUnit).GetCurrentByName(tName_".PY",,.tPyCodeUnit)) + + // update the executable lines for the .cls file's python + $$$ThrowOnError(tCodeUnit.UpdatePyExecutableLines(tName, .tPyCodeUnit)) + + // update the pythonicity of the lines for the .cls file + $$$ThrowOnError(tCodeUnit.UpdatePythonLines(tName, .tPyCodeUnit)) + + // update the relevant python routines + If ($BitCount(tPyCodeUnit.ExecutableLines, 1)) { + set pPyRelevantRoutines = pPyRelevantRoutines _ $ListBuild(tName) + } ElseIf '$BitCount(tCodeUnit.ExecutableLines,1) { + // if there's no executable python and no executable objectscript, skip it + Continue + } + + } ElseIf '$BitCount(tCodeUnit.ExecutableLines,1) { // Skip it - no executable lines. Continue } } + Set pRelevantRoutines = pRelevantRoutines _ $ListBuild(tIntRoutine) } + + // Snapshot all the python routines and their corresponding classes that haven't already been snapshotted + Set tPointer = 0 + While $ListNext(pPyRoutines, tPointer, tPyRoutine) { + If ('$Data(SnapshottedClasses(tPyRoutine))) { + $$$ThrowOnError(##class(TestCoverage.Data.CodeUnit).GetCurrentByName(tPyRoutine_".CLS",,.tCodeUnit)) + $$$ThrowOnError(##class(TestCoverage.Data.CodeUnit).GetCurrentByName(tPyRoutine_".PY",,.tPyCodeUnit)) + $$$ThrowOnError(tCodeUnit.UpdatePyExecutableLines(tPyRoutine, .tPyCodeUnit)) + $$$ThrowOnError(tCodeUnit.UpdatePythonLines(tPyRoutine, .tPyCodeUnit)) + + If ($BitCount(tPyCodeUnit.ExecutableLines, 1)) { + set pPyRelevantRoutines = pPyRelevantRoutines _ $ListBuild(tPyRoutine) + } + Set SnapshottedClasses(tPyRoutine) = 1 + } + } + Write ! } Catch e { Set tSC = e.AsStatus() @@ -391,6 +433,122 @@ ClassMethod GetClassLineExecutableFlags(pClassName As %String, ByRef pDocumentTe } } +ClassMethod CodeArrayToList(ByRef pCodeArray, Output pDocumentText As %List) +{ + set pDocumentText = "" + for i=1:1:$get(pCodeArray(0)) { + set pDocumentText = pDocumentText _ $ListBuild(pCodeArray(i)) + } + quit +} + +/// returns a python tuple of (line to method info, method map info) +/// linetomethodinfo: a python builtins list where the item at index i is the name of the method that line i is a part of +/// methodmapinfo: a python builtins dict with key = method name, value = the line number of its definition +ClassMethod GetPythonMethodMapping(pDocumentText) [ Language = python ] +{ + import iris + import ast + source_lines = iris.cls('%SYS.Python').ToList(pDocumentText) + source_lines = [line + "\n" for line in source_lines] # contains a list of each line of the source code + + source = ''.join(source_lines) + tree = ast.parse(source) + line_function_map = [None] * (len(source_lines)+2) + method_map = {} # dictionary from the method name to its start and ending line number + + class FunctionMapper(ast.NodeVisitor): + def __init__(self): + self.current_class = None + self.current_function = None + self.outermost_function = None # for objectscript purposes, we only care about the outer level functions/methods + + def visit_ClassDef(self, node): + prev_class = self.current_class + self.current_class = node.name + self.generic_visit(node) + self.current_class = prev_class + + def visit_FunctionDef(self, node): + if self.outermost_function is None: + self.outermost_function = node.name + method_map[node.name] = (node.lineno-1, node.end_lineno-1) + + self.current_function = node.name + for lineno in range(node.lineno, node.end_lineno + 1): + line_function_map[lineno-1] = self.outermost_function + + self.generic_visit(node) + self.current_function = None + if self.outermost_function == node.name: + self.outermost_function = None + + # preprocessing the ending line number for each function + tree_with_line_numbers = ast.increment_lineno(tree, n=1) + for node in ast.walk(tree_with_line_numbers): + if isinstance(node, ast.FunctionDef): + node.end_lineno = node.body[-1].end_lineno + + FunctionMapper().visit(tree) + return (line_function_map, method_map) +} + +/// returns a python list with a 1 or 0 for subscript i indicating if line i is executable or not +ClassMethod GetPythonLineExecutableFlags(pDocumentText) [ Language = python ] +{ + import iris + import ast + source_lines = iris.cls('%SYS.Python').ToList(pDocumentText) + source_lines = [line + "\n" for line in source_lines] # contains a list of each line of the source code + + # create the abstract syntax tree for the code, and walk through it, getting each line of code in its context + source = ''.join(source_lines) + tree = ast.parse(source) + executable_lines = set() # stores the 1-indexed line numbers of the executable lines + + class ExecutableLineVisitor(ast.NodeVisitor): + def __init__(self): + self.function_depth = 0 + + def visit(self, node): + if hasattr(node, 'lineno'): + + # decorators for functions and class definitions are executable + if isinstance(node, (ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef)): + decorators = [element.id for element in node.decorator_list] + num_decorators = len(decorators) + for i, element in enumerate(decorators): + conjectured_line = (node.lineno-1)-num_decorators+i # change this back if the line numbers aren't 0 indexed + if "@" + element in source_lines[conjectured_line]: + executable_lines.add(conjectured_line+1) # because back to 1-indexing + executable_lines.add(node.lineno) + elif isinstance(node, (ast.Call, + ast.Return, ast.Assign, ast.AugAssign, ast.AnnAssign, + ast.For, ast.AsyncFor, ast.While, ast.If, ast.With, + ast.AsyncWith, ast.Raise, ast.Try, ast.Assert, + ast.Import, ast.ImportFrom, ast.Pass, + ast.Break, ast.Continue, ast.Delete, ast.Yield, + ast.YieldFrom, ast.Await, ast.Nonlocal)): # all executable (determined manually) + executable_lines.add(node.lineno) + elif isinstance(node, ast.ExceptHandler): # except (but not finally) is executable + executable_lines.add(node.lineno) + elif isinstance(node, ast.Expr) and not isinstance(node.value, ast.Constant): # expressions that aren't docstrings are executable + executable_lines.add(node.lineno) + self.generic_visit(node) + ExecutableLineVisitor().visit(tree) + + output = [0] * (len(source_lines)+1) + for line in executable_lines: + output[line] = 1 + output[1] = 0 # manually set the class definition to be not executable + def print_executable_lines(): + for i, line in enumerate(source_lines, start=1): + is_exec = output[i] + print(f"{i:2d} {'*' if is_exec else ' '} {line.rstrip()}") + # print_executable_lines() + return output +} + /// For a routine (.MAC/.INT) with code in pDocumentText as an integer-subscripted array of lines, /// returns an array (pExecutableFlags) subscripted by line with boolean flags indicating whether the corresponding line is executable. ClassMethod GetRoutineLineExecutableFlags(ByRef pDocumentText, Output pExecutableFlags) @@ -400,7 +558,6 @@ ClassMethod GetRoutineLineExecutableFlags(ByRef pDocumentText, Output pExecutabl For tDocLine=1:1:$Get(pDocumentText) { Do tSourceStream.WriteLine(pDocumentText(tDocLine)) } - Set tSC = ##class(%Library.SyntaxColorReader).FromCode(tSourceStream,"COS","A",.tSCReader) $$$ThrowOnError(tSC) @@ -540,4 +697,3 @@ ClassMethod LineByLineMonitorResultClose(ByRef qHandle As %Binary) As %Status [ } } - diff --git a/cls/TestCoverage/Utils/LineByLineMonitor.cls b/cls/TestCoverage/Utils/LineByLineMonitor.cls index e219f71..fe929d9 100644 --- a/cls/TestCoverage/Utils/LineByLineMonitor.cls +++ b/cls/TestCoverage/Utils/LineByLineMonitor.cls @@ -1,4 +1,4 @@ -Include %occErrors +Include (%occErrors, TestCoverage) /// Wrapper around %Monitor.System.LineByLine to ensure that the monitor is stopped when it should be, and also /// to wrap the decision about whether to stop/start the monitor or to just clear counters. @@ -21,6 +21,9 @@ Method PausedGet() As %Boolean [ CodeMode = expression ] ..Started && '$zu(84,1) } +/// The current python classes being tracked, so that we know what to store the coverage for +Property PythonClassList As %List; + Property LastRoutineList As %List [ Private ]; Property LastMetricList As %List [ Private ]; @@ -51,22 +54,64 @@ ClassMethod CheckAvailableMemory(pProcessCount As %Integer, pRoutineCount As %In Quit tSC } +ClassMethod PyStartWithScope(pCoverageClasses As %List) [ Language = python ] +{ + + from sys import settrace + import iris + + tCoverageClasses = set(iris.cls('%SYS.Python').ToList(pCoverageClasses)) + def my_tracer(frame, event, arg = None): + # extracts frame code + code = frame.f_code + # extracts calling function name and the class that the function is in + class_name = frame.f_globals['__name__'] + # extracts the line number + line_no = frame.f_lineno + if class_name in tCoverageClasses and line_no != 1: # if this is in a covered class + tGlob = iris.gref('^IRIS.Temp.TestCoveragePY') # python doesn't have macros -- this is $$$PyMonitorResults + # $$$PyMonitorResults(classname, linenumber) = the number of times that linenumber in that class was covered + + curCount = tGlob.get([class_name, line_no]) + if not curCount: + curCount = 0 + tGlob[class_name, line_no] = curCount + 1 + + return my_tracer + settrace(my_tracer) +} + +ClassMethod PyClearCounters() +{ + Kill $$$PyMonitorResults +} + +ClassMethod PyStop() [ Language = python ] +{ + from sys import settrace + settrace(None) +} + /// Tracks current monitoring context and stops/starts or resets counters depending on whether it has changed -Method StartWithScope(pRoutineList As %List, pMetricList As %List, pProcessList As %List) As %Status +Method StartWithScope(pRoutineList As %List, pPyClasses As %List, pMetricList As %List, pProcessList As %List) As %Status { Set tSC = $$$OK Try { + Set ..PythonClassList = pPyClasses Set tDifferentScope = (..LastRoutineList '= pRoutineList) || (..LastMetricList '= pMetricList) || (..LastProcessList '= pProcessList) If tDifferentScope && ..Started { // If we need to track different routines/metrics/processes, need to stop the monitor before restarting with the new context. Do ..Stop() + Do ..PyStop() Set ..LastRoutineList = pRoutineList Set ..LastMetricList = pMetricList Set ..LastProcessList = pProcessList } If '..Started { + Do ..PyClearCounters() Set tSC = ..Start(pRoutineList, pMetricList, pProcessList) + Do ..PyStartWithScope(pPyClasses) If $System.Status.Equals(tSC,$$$ERRORCODE($$$MonitorMemoryAlloc)) { // Construct a more helpful error message. Set tSC = $$$EMBEDSC(..CheckAvailableMemory($ListLength(pProcessList),$ListLength(pRoutineList),1),tSC) @@ -158,4 +203,3 @@ ClassMethod GetError(key As %String, args...) } } - diff --git a/inc/TestCoverage.inc b/inc/TestCoverage.inc index ba88615..47611ca 100644 --- a/inc/TestCoverage.inc +++ b/inc/TestCoverage.inc @@ -3,3 +3,4 @@ ROUTINE TestCoverage [Type=INC] #define StopTimer If (..Display["log") { Write ($zh-tStartTime)," seconds" } #define METRICS "RtnLine","Time","TotalTime" #define TestPathAllTests "all tests" +#define PyMonitorResults ^IRIS.Temp.TestCoveragePY \ No newline at end of file diff --git a/internal/testing/unit_tests/UnitTest/TestCoverage/Unit/CodeUnit.cls b/internal/testing/unit_tests/UnitTest/TestCoverage/Unit/CodeUnit.cls index 42f50a3..1174a48 100644 --- a/internal/testing/unit_tests/UnitTest/TestCoverage/Unit/CodeUnit.cls +++ b/internal/testing/unit_tests/UnitTest/TestCoverage/Unit/CodeUnit.cls @@ -16,6 +16,7 @@ Method TestCodeUnitCreation() #dim tCodeUnit As TestCoverage.Data.CodeUnit Set tClsName = $classname()_".CLS" Set tIntName = $classname()_".1.INT" + Set tPyName = $classname()_".PY" Set tSC = ##class(TestCoverage.Data.CodeUnit).GetCurrentByName(tIntName,$Namespace,.tIntCodeUnit) Do $$$AssertStatusOK(tSC,"Found test coverage data for "_tIntName) Do $$$AssertEquals(tIntCodeUnit.Name,$classname()_".1") @@ -27,28 +28,47 @@ Method TestCodeUnitCreation() Do $$$AssertEquals(tGenCodeUnit.Name,$classname()_".G1") Do $$$AssertEquals(tGenCodeUnit.Type,"INT") + Set tSC = ##class(TestCoverage.Data.CodeUnit).GetCurrentByName(tClsName,$Namespace,.tClsCodeUnit) Do $$$AssertStatusOK(tSC,"Found test coverage data for "_tClsName) Do $$$AssertEquals(tClsCodeUnit.Name,$classname()) Do $$$AssertEquals(tClsCodeUnit.Type,"CLS") + Set tSC = ##class(TestCoverage.Data.CodeUnit).GetCurrentByName(tPyName,$Namespace,.tPyCodeUnit) + Do $$$AssertStatusOK(tSC,"Found test coverage data for "_tPyName) + Do $$$AssertEquals(tPyCodeUnit.Name,$classname()) + Do $$$AssertEquals(tPyCodeUnit.Type,"PY") + + Set tSC = tClsCodeUnit.UpdatePyExecutableLines($classname(),.tPyCodeUnit) + Do $$$AssertStatusOK(tSC,"Found updated executable line data for "_tClsName) + + Set tSC = tClsCodeUnit.UpdatePythonLines($classname(),.tPyCodeUnit) + Do $$$AssertStatusOK(tSC,"Found updated pythonicity line data for "_tClsName) + Set tConstantReturnValueLine = tClsCodeUnit.MethodMap.GetAt("SampleConstantReturnValue") Set tCodeGeneratorLine = tClsCodeUnit.MethodMap.GetAt("SampleCodeGenerator") Set tNormalMethodLine = tClsCodeUnit.MethodMap.GetAt("SampleNormalMethod") + Set tPythonMethodLine = tClsCodeUnit.MethodMap.GetAt("SamplePythonMethod") Do $$$AssertNotEquals(tConstantReturnValueLine,"") Do $$$AssertNotEquals(tCodeGeneratorLine,"") Do $$$AssertNotEquals(tNormalMethodLine,"") + Do $$$AssertNotEquals(tPythonMethodLine,"") + + // test if LineIsPython is working properly + Do $$$AssertEquals(tClsCodeUnit.LineIsPython.GetAt(tPythonMethodLine+2), 1) + Do $$$AssertEquals(tClsCodeUnit.LineIsPython.GetAt(tNormalMethodLine+2), 0) // tTestLines(line number) = $ListBuild(description, executable (default 1), mapped (default 1), mapped from hash (if relevant), mapped from line (if relevant)) Set tTestLines(tConstantReturnValueLine+2) = $ListBuild("SampleConstantReturnValue+1",0,0) - Set tTestLines(tCodeGeneratorLine+2) = $ListBuild("SampleCodeGenerator+1",,,tGenCodeUnit.Hash,tGenCodeUnit.MethodMap.GetAt("SampleCodeGenerator")+1) - Set tTestLines(tCodeGeneratorLine+3) = $ListBuild("SampleCodeGenerator+2",,,tGenCodeUnit.Hash,tGenCodeUnit.MethodMap.GetAt("SampleCodeGenerator")+2) - Set tTestLines(tCodeGeneratorLine+4) = $ListBuild("SampleCodeGenerator+3",,,tGenCodeUnit.Hash,tGenCodeUnit.MethodMap.GetAt("SampleCodeGenerator")+3) + Set tTestLines(tCodeGeneratorLine+2) = $ListBuild("SampleCodeGenerator+1",,,tGenCodeUnit.Hash,tGenCodeUnit.MethodMap.GetAt("SampleCodeGenerator")+1, "INT") + Set tTestLines(tCodeGeneratorLine+3) = $ListBuild("SampleCodeGenerator+2",,,tGenCodeUnit.Hash,tGenCodeUnit.MethodMap.GetAt("SampleCodeGenerator")+2, "INT") + Set tTestLines(tCodeGeneratorLine+4) = $ListBuild("SampleCodeGenerator+3",,,tGenCodeUnit.Hash,tGenCodeUnit.MethodMap.GetAt("SampleCodeGenerator")+3, "INT") Set methodLabel = $Select($System.Version.GetMajor()<2023:"z",1:"")_"SampleNormalMethod" - Set tTestLines(tNormalMethodLine+2) = $ListBuild("SampleNormalMethod+1",,,tIntCodeUnit.Hash,tIntCodeUnit.MethodMap.GetAt(methodLabel)+1) - Set tTestLines(tNormalMethodLine+3) = $ListBuild("SampleNormalMethod+2",,,tIntCodeUnit.Hash,tIntCodeUnit.MethodMap.GetAt(methodLabel)+2) - + Set tTestLines(tNormalMethodLine+2) = $ListBuild("SampleNormalMethod+1",,,tIntCodeUnit.Hash,tIntCodeUnit.MethodMap.GetAt(methodLabel)+1, "INT") + Set tTestLines(tNormalMethodLine+3) = $ListBuild("SampleNormalMethod+2",,,tIntCodeUnit.Hash,tIntCodeUnit.MethodMap.GetAt(methodLabel)+2, "INT") + Set tTestLines(tPythonMethodLine+2) = $ListBuild("SamplePythonMethod+1",,,tPyCodeUnit.Hash,tPyCodeUnit.MethodMap.GetAt("SamplePythonMethod")+1, "PY") + Set tTestLines(tPythonMethodLine+3) = $ListBuild("SamplePythonMethod+2",,,tPyCodeUnit.Hash,tPyCodeUnit.MethodMap.GetAt("SamplePythonMethod")+2, "PY") Set tLine = "" For { Set tLine = $Order(tTestLines(tLine),1,tInfo) @@ -57,9 +77,9 @@ Method TestCodeUnitCreation() } // Overwrite with defined values, leave defaults in AssertLineHandledCorrectly for undefined values (passing byref) - Kill tDescription, tExecutable, tMapped, tExpectedFromHash, tExpectedFromLine - Set $ListBuild(tDescription, tExecutable, tMapped, tExpectedFromHash, tExpectedFromLine) = tInfo - Do ..AssertLineHandledCorrectly(tClsCodeUnit, tLine, .tDescription, .tExecutable, .tMapped, .tExpectedFromHash, .tExpectedFromLine) + Kill tDescription, tExecutable, tMapped, tExpectedFromHash, tExpectedFromLine, tType + Set $ListBuild(tDescription, tExecutable, tMapped, tExpectedFromHash, tExpectedFromLine, tType) = tInfo + Do ..AssertLineHandledCorrectly(tClsCodeUnit, tLine, .tDescription, .tExecutable, .tMapped, .tExpectedFromHash, .tExpectedFromLine, .tType) } } Catch e { Set tSC = e.AsStatus() @@ -67,7 +87,7 @@ Method TestCodeUnitCreation() Do $$$AssertStatusOK(tSC,"No unexpected errors occurred.") } -Method AssertLineHandledCorrectly(pClassCodeUnit As TestCoverage.Data.CodeUnit, pLine As %Integer, pDescription As %String = {"Line "_pLine}, pExecutable As %Boolean = 1, pMapped As %Boolean = 1, pExpectedFromHash As %String = "", pExpectedFromLine As %Integer = "") As %Boolean +Method AssertLineHandledCorrectly(pClassCodeUnit As TestCoverage.Data.CodeUnit, pLine As %Integer, pDescription As %String = {"Line "_pLine}, pExecutable As %Boolean = 1, pMapped As %Boolean = 1, pExpectedFromHash As %String = "", pExpectedFromLine As %Integer = "", pType As %String = "INT") As %Boolean { Set tAllGood = 1 If pExecutable { @@ -78,7 +98,7 @@ Method AssertLineHandledCorrectly(pClassCodeUnit As TestCoverage.Data.CodeUnit, &sql(select count(*),FromHash,FromLine into :tCount,:tFromHash,:tFromLine from TestCoverage_Data.CodeUnitMap - where ToHash = :pClassCodeUnit.Hash and ToLine = :pLine and FromHash->Type = 'INT') + where ToHash = :pClassCodeUnit.Hash and ToLine = :pLine and FromHash->Type = :pType) If (SQLCODE < 0) { Throw ##class(%Exception.SQL).CreateFromSQLCODE(SQLCODE,%msg) } @@ -123,5 +143,10 @@ ClassMethod SampleNormalMethod() Quit y } +ClassMethod SamplePythonMethod() [ Language = python ] +{ + import iris + return 50 } +} diff --git a/internal/testing/unit_tests/UnitTest/TestCoverage/Unit/TestComplexity.cls b/internal/testing/unit_tests/UnitTest/TestCoverage/Unit/TestComplexity.cls index e8942eb..1801d6c 100644 --- a/internal/testing/unit_tests/UnitTest/TestCoverage/Unit/TestComplexity.cls +++ b/internal/testing/unit_tests/UnitTest/TestCoverage/Unit/TestComplexity.cls @@ -10,6 +10,8 @@ Method TestMethodsInThisClass() Do $$$AssertStatusOK($System.OBJ.Compile($classname(),"ck-d")) Do $$$AssertStatusOK(##class(TestCoverage.Data.CodeUnit).GetCurrentByName(tClass_".1.INT")) If $$$AssertStatusOK(##class(TestCoverage.Data.CodeUnit).GetCurrentByName(tClass_".CLS",,.tCodeUnit)) { + Do $$$AssertStatusOK(##class(TestCoverage.Data.CodeUnit).GetCurrentByName(tClass_".PY",,.tPyCodeUnit)) + Do tCodeUnit.UpdatePyExecutableLines(tClass, .tPyCodeUnit) Set tKey = "" For { #dim tMethod As TestCoverage.Data.CodeSubUnit.Method @@ -243,5 +245,13 @@ Method MethodWithComplexMacros(pStatus As %Status) $$$ThrowOnError(pStatus) } +/// Complexity: 3 (1 + for + if) +ClassMethod ForLoopPython() [ Language = python ] +{ + for i in range(5): + if i > 4: + continue + return 1 } +} diff --git a/internal/testing/unit_tests/UnitTest/TestCoverage/Unit/TestCoverageList.cls b/internal/testing/unit_tests/UnitTest/TestCoverage/Unit/TestCoverageList.cls index f43a0e6..2658e6b 100644 --- a/internal/testing/unit_tests/UnitTest/TestCoverage/Unit/TestCoverageList.cls +++ b/internal/testing/unit_tests/UnitTest/TestCoverage/Unit/TestCoverageList.cls @@ -4,6 +4,7 @@ Class UnitTest.TestCoverage.Unit.TestCoverageList Extends %UnitTest.TestCase /// helper function to find the samplecovlist.list's path ClassMethod FindCoverageList(directory As %String = "") As %String { + set stmt = ##class(%SQL.Statement).%New() set status = stmt.%PrepareClassQuery("%File", "FileSet") if $$$ISERR(status) {write "%Prepare failed:" do $SYSTEM.Status.DisplayError(status) quit} @@ -22,10 +23,14 @@ ClassMethod FindCoverageList(directory As %String = "") As %String return name } } elseif (type = "D"){ - do ..FindCoverageList(name) + set retVal = ..FindCoverageList(name) + if (retVal '= "") { + return retVal + } } } if (rset.%SQLCODE < 0) {write "%Next failed:", !, "SQLCODE ", rset.%SQLCODE, ": ", rset.%Message quit} + return "" // didn't find it in this directory } Method TestGettingCoverageList() diff --git a/module.xml b/module.xml index 9425df6..55a559a 100644 --- a/module.xml +++ b/module.xml @@ -2,7 +2,7 @@ TestCoverage - 3.1.1 + 3.2.0 Run your typical ObjectScript %UnitTest tests and see which lines of your code are executed. Includes Cobertura-style reporting for use in continuous integration tools. module diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..703cbdb --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +radon==6.* \ No newline at end of file