Embedded Python Coverage Support

cls/TestCoverage/Data/CodeUnit.cls

@@ -14,7 +14,7 @@ Index NameTypeHash On (Name, Type, Hash) [ Data = ExecutableLines, Unique ];
 /// Name of the code unit
 Property Name As %String(MAXLEN = 255) [ Required ];
 
-/// Type (3-letter extension) of the code unit
+/// Type (2 or 3-letter extension) of the code unit
 Property Type As TestCoverage.DataType.RoutineType [ Required ];
 
 /// Hash of the code unit; for methods for determining this, see <method>GetCurrentHash</method>
@@ -28,8 +28,12 @@ Property ExecutableLines As TestCoverage.DataType.Bitstring;
 
 /// For classes, map of method names in the code to their associated line numbers
 /// For routines, map of labels to associated line numbers
+/// For python, map of method names to associated starting line number
 Property MethodMap As array Of %Integer;
 
+/// Only for python: map of method names to associated ending line number of the method 
+Property MethodEndMap As array Of %Integer;
+
 /// For classes, map of line numbers in code to associated method names
 /// For routines, map of labels to associated line numbers
 Property LineToMethodMap As array Of %Dictionary.CacheIdentifier [ Private ];
@@ -98,10 +102,21 @@ ClassMethod GetCurrentByName(pInternalName As %String, pSourceNamespace As %Stri
 		}
 
 		Set $Namespace = pSourceNamespace
+
 		If (tType = "CLS") {
 			Do ##class(TestCoverage.Utils).GetClassLineExecutableFlags(tName,.tCodeArray,.tExecutableFlags)
-		} Else {
+		} ElseIf ((tType = "INT") || (tType = "MAC")) {
 			Do ##class(TestCoverage.Utils).GetRoutineLineExecutableFlags(.tCodeArray,.tExecutableFlags)
+		} ElseIf (tType="PY") {
+			Do ##class(TestCoverage.Utils).CodeArrayToList(.tCodeArray, .pDocumentText) 
+			Set tExecutableFlagsPyList = ##class(TestCoverage.Utils).GetPythonLineExecutableFlags(pDocumentText)
+			Kill tExecutableFlags
+			for i=1:1:tExecutableFlagsPyList."__len__"()-1 {
+				set tExecutableFlags(i) = tExecutableFlagsPyList."__getitem__"(i)
+			}
+		}
+		Else {
+			return $$$ERROR($$$GeneralError,"File type not supported")
 		}
 
 		Set $Namespace = tOriginalNamespace
@@ -120,46 +135,81 @@ ClassMethod GetCurrentByName(pInternalName As %String, pSourceNamespace As %Stri
 			Set pCodeUnit.Generated = ($$$comClassKeyGet(tName,$$$cCLASSgeneratedby) '= "")
 		}
 
-		Set tMethod = ""
-		Set tMethodSignature = ""
-		Set tMethodMask = ""
-		For tLineNumber=1:1:$Get(tCodeArray,0) {
-			Set tLine = tCodeArray(tLineNumber)
-			Do pCodeUnit.Lines.Insert(tLine)
-
-			If (tType = "CLS") {
-				// Extract line offset of methods in classes
-				Set tStart = $Piece(tLine," ")
-				If (tStart = "ClassMethod") || (tStart = "Method") {
-					Set tMethod = $Piece($Piece(tLine,"(")," ",2)
-					Set tMethodSignature = tLine
-					Do pCodeUnit.MethodMap.SetAt(tLineNumber,tMethod)
-					Do pCodeUnit.LineToMethodMap.SetAt(tMethod,tLineNumber)
-				} ElseIf ($Extract(tStart) = "{") {
-					// Ignore the opening bracket for a method.
-				} ElseIf ($Extract(tStart) = "}") && (tMethod '= "") {
-					// End of method. Add method subunit to class.
-					Set tSubUnit = ##class(TestCoverage.Data.CodeSubUnit.Method).%New()
-					Set tSubUnit.Name = tMethod
-					Set tSubUnit.DisplaySignature = tMethodSignature
-					Set tSubUnit.Mask = tMethodMask
-					Do pCodeUnit.SubUnits.Insert(tSubUnit)
-					Set tMethod = ""
-					Set tMethodSignature = ""
-					Set tMethodMask = ""
-				} ElseIf (tMethod '= "") {
-					Set $Bit(tMethodMask,tLineNumber) = 1
+
+		If (tType = "PY") {
+			Set ClassName = $Piece(tName,".", *)
+			Set tMethodInfo = ##class(TestCoverage.Utils).GetPythonMethodMapping(pDocumentText, ClassName)
+			Set tLineToMethodInfo = tMethodInfo."__getitem__"(0)
+			Set tMethodMapInfo = tMethodInfo."__getitem__"(1)
+			for i=1:1:$listlength(pDocumentText) {
+				Set tMethod = tLineToMethodInfo."__getitem__"(i)
+				Do pCodeUnit.LineToMethodMap.SetAt(tMethod,i)
+			}
+			Set iterator = tMethodMapInfo."__iter__"()
+			for i=1:1:tMethodMapInfo."__len__"() {
+				Set tMethod = iterator."__next__"()
+				Set tStartEnd = tMethodMapInfo."__getitem__"(tMethod)
+				Set tStartLine = tStartEnd."__getitem__"(0)
+				Set tEndLine = tStartEnd."__getitem__"(1)
+				Do pCodeUnit.MethodMap.SetAt(tStartLine,tMethod)
+				Do pCodeUnit.MethodEndMap.SetAt(tEndLine, tMethod)
+
+				Set tMethodMask = ""
+				for j = tStartLine:1:tEndLine {
+					Set $Bit(tMethodMask,j) = 1
 				}
-			} Else {
-				// Extract line offset of labels in routines
-				If ($ZStrip($Extract(tLine),"*PWC") '= "") {
-					Set tLabel = $Piece($Piece(tLine," "),"(")
-					Do pCodeUnit.MethodMap.SetAt(tLineNumber,tLabel)
-					Do pCodeUnit.LineToMethodMap.SetAt(tLabel,tLineNumber)
+				Set tMethodSignature = $list(pDocumentText, tStartLine)
+				Set tSubUnit = ##class(TestCoverage.Data.CodeSubUnit.Method).%New()
+				Set tSubUnit.Name = tMethod
+				Set tSubUnit.DisplaySignature = tMethodSignature
+				Set tSubUnit.Mask = tMethodMask
+				Do pCodeUnit.SubUnits.Insert(tSubUnit)
+			}
+		}
+		Else {
+			Set tMethod = ""
+			Set tMethodSignature = ""
+			Set tMethodMask = ""
+			For tLineNumber=1:1:$Get(tCodeArray,0) {
+				Set tLine = tCodeArray(tLineNumber)
+				Do pCodeUnit.Lines.Insert(tLine)
+
+				If (tType = "CLS") {
+					// Extract line offset of methods in classes
+					Set tStart = $Piece(tLine," ")
+					If (tStart = "ClassMethod") || (tStart = "Method") {
+						Set tMethod = $Piece($Piece(tLine,"(")," ",2)
+						Set tMethodSignature = tLine
+						Do pCodeUnit.MethodMap.SetAt(tLineNumber,tMethod)
+						Do pCodeUnit.LineToMethodMap.SetAt(tMethod,tLineNumber)
+					} ElseIf ($Extract(tStart) = "{") {
+						// Ignore the opening bracket for a method.
+					} ElseIf ($Extract(tStart) = "}") && (tMethod '= "") {
+						// End of method. Add method subunit to class.
+						Set tSubUnit = ##class(TestCoverage.Data.CodeSubUnit.Method).%New()
+						Set tSubUnit.Name = tMethod
+						Set tSubUnit.DisplaySignature = tMethodSignature
+						Set tSubUnit.Mask = tMethodMask
+						Do pCodeUnit.SubUnits.Insert(tSubUnit)
+						Set tMethod = ""
+						Set tMethodSignature = ""
+						Set tMethodMask = ""
+					} ElseIf (tMethod '= "") {
+						Set $Bit(tMethodMask,tLineNumber) = 1
+					}
+				} 
+				Else {
+					// Extract line offset of labels in routines
+					If ($ZStrip($Extract(tLine),"*PWC") '= "") {
+						Set tLabel = $Piece($Piece(tLine," "),"(")
+						Do pCodeUnit.MethodMap.SetAt(tLineNumber,tLabel)
+						Do pCodeUnit.LineToMethodMap.SetAt(tLabel,tLineNumber)
+					}
 				}
 			}
 		}
 
+
 		Set tSC = pCodeUnit.%Save()
 		If $$$ISERR(tSC) && $System.Status.Equals(tSC,$$$ERRORCODE($$$IDKeyNotUnique)) {
 			// Some other process beat us to it.
@@ -183,10 +233,82 @@ ClassMethod GetCurrentByName(pInternalName As %String, pSourceNamespace As %Stri
 	Quit tSC
 }
 
+/// Get the executable lines of code in python over to the .cls CodeUnit
+Method UpdatePyExecutableLines(pName As %String, ByRef pPyCodeUnit) As %Status
+{
+	Set tSC = $$$OK
+	Set tOriginalNamespace = $Namespace
+	Set tInitTLevel = $TLevel
+	Try {
+		TSTART
+
+		Set tBitString = ""
+		If (##class(TestCoverage.Manager).HasPython(pName)) {
+
+			// snapshot the Python CodeUnit for it 
+			Set tFromHash = pPyCodeUnit.Hash
+			Set tToHash = ..Hash 
+			// now bring over the executable lines 
+
+			&sql(
+				DECLARE C1 CURSOR FOR
+				SELECT map.ToLine
+				INTO :hToLine
+				FROM TestCoverage_Data.CodeUnitMap map 
+				JOIN TestCoverage_Data.CodeUnit fromCodeUnit 
+				ON map.FromHash = fromCodeUnit.Hash 
+				JOIN TestCoverage_Data.CodeUnit toCodeUnit 
+				ON map.ToHash = toCodeUnit.Hash 
+				AND fromCodeUnit.Hash = :tFromHash
+				AND toCodeUnit.Hash = :tToHash
+				WHERE TestCoverage.BIT_VALUE(fromCodeUnit.ExecutableLines,map.FromLine) <> 0
+				)
+			&sql(OPEN C1)
+			If (SQLCODE < 0) {
+				Throw ##class(%Exception.SQL).CreateFromSQLCODE(SQLCODE,%msg)
+			}
+			For {
+				&SQL(FETCH C1)
+				If (SQLCODE < 0) {
+					Throw ##class(%Exception.SQL).CreateFromSQLCODE(SQLCODE,%msg)
+				} ElseIf (SQLCODE) {
+					Quit
+				}
+				// Process the fetched rows
+				// hToLine contains the line number in the .cls file corresponding to executable lines in the .py file
+				Set $Bit(tBitString, hToLine) = 1
+			}
+			&sql(CLOSE C1)
+			If (SQLCODE < 0) {
+				Throw ##class(%Exception.SQL).CreateFromSQLCODE(SQLCODE,%msg)
+			}
+		}
+		if (pName = "TestCoverage.Utils") {
+			set ^IRIS.TEMPCG($i(^IRIS.TEMPCG)) = tBitString
+			set ^IRIS.TEMPCG($i(^IRIS.TEMPCG)) = ..ExecutableLines
+		}
+		Set ..ExecutableLines = $BITLOGIC(..ExecutableLines | tBitString)
+		Set tSC = ..%Save()
+		$$$ThrowOnError(tSC)
+		if (pName = "TestCoverage.Utils") {
+			set ^IRIS.TEMPCG($i(^IRIS.TEMPCG)) = ..ExecutableLines
+		}
+		TCOMMIT
+	} Catch e {
+		Set pCodeUnit = $$$NULLOREF
+		Set tSC = e.AsStatus()
+	}
+	While ($TLevel > tInitTLevel) {
+		TROLLBACK 1
+	}
+	Quit tSC
+}
+
 Method UpdateSourceMap(pSourceNamespace As %String, ByRef pCache) As %Status
 {
 	Set tSC = $$$OK
 	Try {
+
 		// First, build local array (tMap) of all maps from the .INT file to other files.
 		If (..Type = "INT") {
 			For tLineNumber=1:1:..Lines.Count() {
@@ -225,6 +347,37 @@ Method UpdateSourceMap(pSourceNamespace As %String, ByRef pCache) As %Status
 				}
 			}
 		}
+		If (..Type = "PY") {
+
+			set tClass = ..Name
+			Set tSourceUnits(tClass_".CLS") = ""
+			$$$ThrowOnError(..GetCurrentByName(tClass _ ".CLS", pSourceNamespace, .pCLSCodeUnit, .pCLSCache))
+			// we'll do the mappings from the .py to the .cls direction, so that we don't iterate over objectscript lines
+			Set tMethod = ""
+			Do ..MethodMap.GetNext(.tMethod)
+			while (tMethod '= "") 
+			{
+				Set tCLSMethodNum = pCLSCodeUnit.MethodMap.GetAt(tMethod)
+				Set tMethodStart = ..MethodMap.GetAt(tMethod)
+				Set tMethodEnd = ..MethodEndMap.GetAt(tMethod)
+				Set tMethodName = tMethod
+				Set tFullMap(tMethodStart) = $lb("CLS", tClass,tMethodName, -1, -1) ; -1 because the class 
+				; definition doesn't have the +1 offset from the { 
+
+				Do ..MethodMap.GetNext(.tMethod)
+				For i = tMethodStart+1:1:tMethodEnd {
+					Set tClassLineNum = i-tMethodStart
+					Set tFullMap(i) = $lb("CLS", tClass,tMethodName, tClassLineNum, tClassLineNum)
+
+					// extra check to make sure that the lines we're mapping between are the same as expected
+					Set tClassLineCode = $zstrip(pCLSCodeUnit.Lines.GetAt(tCLSMethodNum + tClassLineNum + 1), "<>W")
+					Set tPyLineCode = $zstrip(..Lines.GetAt(i), "<>W")
+					if (tPyLineCode '= tClassLineCode) {
+						Set tSC = $$$ERROR($$$GeneralError,"Compiled .py code doesn't match .CLS python code ")
+					}
+				}
+			}
+		}
 
 		// If we are a generator .INT file, ensure that we have source for the original class populated.
 		// In such files, the second line looks like (for example):
@@ -270,7 +423,7 @@ Method UpdateSourceMap(pSourceNamespace As %String, ByRef pCache) As %Status
 			Set tCodeUnits(tCodeUnit.Type,tCodeUnit.Name) = tCodeUnit
 		}
 
-		// Create CodeUnitMap data based on .INT->.CLS mapping.
+		// Create CodeUnitMap data based on .INT / .py ->.CLS mapping.
 		Set tFromHash = ..Hash
 		Set tLineNumber = ""
 		For {
@@ -396,6 +549,10 @@ ClassMethod GetCurrentHash(pName As %String, pType As %String, Output pHash As %
 				// Skip header (lines 1-4) which, for .INT routines generated from classes,
 				// includes the class compilation signature.
 				Set pHash = ..HashArrayRange(.pCodeArray,5,pName_"."_pType,.tSizeHint)
+			} ElseIf (pType = "PY") {
+				Merge pCodeArray = ^ROUTINE(pName_".py",0)
+				set tSizeHint = ^ROUTINE(pName_".py",0,0)
+				set pHash = ..HashArrayRange(.pCodeArray, ,pName_".py", .pSizeHint)
 			} Else {
 				// Give standard descriptive error about the type being invalid.
 				$$$ThrowStatus(..TypeIsValid(pType))
@@ -478,6 +635,11 @@ Storage Default
 <Structure>subnode</Structure>
 <Subscript>"Lines"</Subscript>
 </Data>
+<Data name="MethodEndMap">
+<Attribute>MethodEndMap</Attribute>
+<Structure>subnode</Structure>
+<Subscript>"MethodEndMap"</Subscript>
+</Data>
 <Data name="MethodMap">
 <Attribute>MethodMap</Attribute>
 <Structure>subnode</Structure>
@@ -492,4 +654,3 @@ Storage Default
 }
 
 }
-