Add ClrBubbledException to handle exceptions bubbled from .Net to Python (#76)

* Add ClrBubbledException to handle exceptions bubbled from .Net to Python and back to .Net

* Bump version to 2.0.23

* Minor unit tests fixes

Author: jhonabreul

src/embed_tests/Codecs.cs

@@ -335,8 +335,9 @@ public void ExceptionDecoded()
         {
             PyObjectConversions.RegisterDecoder(new ValueErrorCodec());
             using var scope = Py.CreateScope();
-            var error = Assert.Throws<ValueErrorWrapper>(()
+            var error = Assert.Throws<ClrBubbledException>(()
                 => PythonEngine.Exec($"raise ValueError('{TestExceptionMessage}')"));
+            Assert.IsInstanceOf<ValueErrorWrapper>(error.InnerException);
             Assert.AreEqual(TestExceptionMessage, error.Message);
         }
 

src/embed_tests/TestPropertyAccess.cs

@@ -1339,7 +1339,9 @@ def CallDynamicMethodCatchingExceptions(self, fixture, defaultValue):
 
             using (Py.GIL())
             {
-                Assert.Throws<ArgumentException>(() => model.CallDynamicMethodWithoutCatchingExceptions(fixture));
+                var exception = Assert.Throws<ClrBubbledException>(() => model.CallDynamicMethodWithoutCatchingExceptions(fixture));
+                Assert.IsInstanceOf<ArgumentException>(exception.InnerException);
+
                 Assert.AreEqual(property, model.CallDynamicMethodCatchingExceptions(fixture, property).AsManagedObject(property.GetType()));
             }
         }

src/embed_tests/TestPythonException.cs

@@ -1,4 +1,7 @@
 using System;
+using System.IO;
+using System.Linq;
+
 using NUnit.Framework;
 using Python.Runtime;
 
@@ -10,6 +13,16 @@ public class TestPythonException
         public void SetUp()
         {
             PythonEngine.Initialize();
+
+            // Add scripts folder to path in order to be able to import the test modules
+            string testPath = Path.Combine(TestContext.CurrentContext.TestDirectory, "fixtures");
+            TestContext.Out.WriteLine(testPath);
+
+            using var str = Runtime.Runtime.PyString_FromString(testPath);
+            Assert.IsFalse(str.IsNull());
+            BorrowedReference path = Runtime.Runtime.PySys_GetObject("path");
+            Assert.IsFalse(path.IsNull);
+            Runtime.Runtime.PyList_Append(path, str.Borrow());
         }
 
         [OneTimeTearDown]
@@ -195,5 +208,69 @@ public void TestPythonException_Normalize_ThrowsWhenErrorSet()
             Assert.Throws<InvalidOperationException>(() => pythonException.Normalize());
             Exceptions.Clear();
         }
+
+        [Test]
+        public void TestGetsPythonCodeInfoInStackTrace()
+        {
+            using (Py.GIL())
+            {
+                dynamic testClassModule = PyModule.FromString("TestGetsPythonCodeInfoInStackTrace_Module", @"
+from clr import AddReference
+AddReference(""Python.EmbeddingTest"")
+
+from Python.EmbeddingTest import *
+
+class TestPythonClass(TestPythonException.TestClass):
+    def CallThrow(self):
+        super().ThrowException()
+");
+
+                try
+                {
+                    var instance = testClassModule.TestPythonClass();
+                    dynamic module = Py.Import("PyImportTest.SampleScript");
+                    module.invokeMethod(instance, "CallThrow");
+                }
+                catch (ClrBubbledException ex)
+                {
+                    Assert.AreEqual("Test Exception Message", ex.InnerException.Message);
+
+                    var pythonTracebackLines = ex.PythonTraceback.TrimEnd('\n').Split('\n').Select(x => x.Trim()).ToList();
+                    Assert.AreEqual(5, pythonTracebackLines.Count);
+
+                    Assert.AreEqual("File \"none\", line 9, in CallThrow", pythonTracebackLines[0]);
+
+                    Assert.IsTrue(new[]
+                    {
+                        "File ",
+                        "fixtures\\PyImportTest\\SampleScript.py",
+                        "line 5",
+                        "in invokeMethodImpl"
+                    }.All(x => pythonTracebackLines[1].Contains(x)));
+                    Assert.AreEqual("getattr(instance, method_name)()", pythonTracebackLines[2]);
+
+                    Assert.IsTrue(new[]
+                    {
+                        "File ",
+                        "fixtures\\PyImportTest\\SampleScript.py",
+                        "line 2",
+                        "in invokeMethod"
+                    }.All(x => pythonTracebackLines[3].Contains(x)));
+                    Assert.AreEqual("invokeMethodImpl(instance, method_name)", pythonTracebackLines[4]);
+                }
+                catch (Exception ex)
+                {
+                    Assert.Fail($"Unexpected exception: {ex}");
+                }
+            }
+        }
+
+        public class TestClass
+        {
+            public void ThrowException()
+            {
+                throw new ArgumentException("Test Exception Message");
+            }
+        }
     }
 }

src/embed_tests/fixtures/PyImportTest/SampleScript.py

@@ -0,0 +1,5 @@
+def invokeMethod(instance, method_name):
+    invokeMethodImpl(instance, method_name)
+
+def invokeMethodImpl(instance, method_name):
+    getattr(instance, method_name)()

src/embed_tests/pyimport.cs

@@ -96,7 +96,8 @@ import clr
 clr.AddReference('{path}')
 ";
 
-            Assert.Throws<BadImageFormatException>(() => PythonEngine.Exec(code));
+            var exception = Assert.Throws<ClrBubbledException>(() => PythonEngine.Exec(code));
+            Assert.IsInstanceOf<BadImageFormatException>(exception.InnerException);
         }
     }
 }

src/perf_tests/Python.PerformanceTests.csproj

@@ -13,7 +13,7 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.*" />
-    <PackageReference Include="quantconnect.pythonnet" Version="2.0.22" GeneratePathProperty="true">
+    <PackageReference Include="quantconnect.pythonnet" Version="2.0.23" GeneratePathProperty="true">
       <IncludeAssets>compile</IncludeAssets>
     </PackageReference>
   </ItemGroup>
@@ -25,7 +25,7 @@
   </Target>
 
   <Target Name="CopyBaseline" AfterTargets="Build">
-    <Copy SourceFiles="$(NuGetPackageRoot)quantconnect.pythonnet\2.0.22\lib\net5.0\Python.Runtime.dll" DestinationFolder="$(OutDir)baseline" />
+    <Copy SourceFiles="$(NuGetPackageRoot)quantconnect.pythonnet\2.0.23\lib\net5.0\Python.Runtime.dll" DestinationFolder="$(OutDir)baseline" />
   </Target>
 
   <Target Name="CopyNewBuild" AfterTargets="Build">

src/runtime/ClrBubbledException.cs

@@ -0,0 +1,62 @@
+using System;
+using System.Text;
+
+namespace Python.Runtime
+{
+    /// <summary>
+    /// Provides an abstraction to represent a .Net exception that is bubbled to Python and back to .Net
+    /// and includes the Python traceback.
+    /// </summary>
+    public class ClrBubbledException : Exception
+    {
+        /// <summary>
+        /// The Python traceback
+        /// </summary>
+        public string PythonTraceback { get; }
+
+        /// <summary>
+        /// Creates a new instance of <see cref="ClrBubbledException"/>
+        /// </summary>
+        /// <param name="sourceException">The original exception that was thrown in .Net</param>
+        /// <param name="pythonTraceback">The Python traceback</param>
+        public ClrBubbledException(Exception sourceException, string pythonTraceback)
+            : base(sourceException.Message, sourceException)
+        {
+            PythonTraceback = pythonTraceback;
+        }
+
+        /// <summary>
+        /// StackTrace Property
+        /// </summary>
+        /// <remarks>
+        /// A string representing the exception stack trace.
+        /// </remarks>
+        public override string StackTrace
+        {
+            get
+            {
+                return PythonTraceback + "Underlying exception stack trace:" + Environment.NewLine + InnerException.StackTrace;
+            }
+        }
+
+        public override string ToString()
+        {
+            StringBuilder description = new StringBuilder();
+            description.AppendFormat("{0}: {1}{2}", InnerException.GetType().Name, Message, Environment.NewLine);
+            description.AppendFormat(" --> {0}", PythonTraceback);
+            description.AppendFormat("   --- End of Python traceback ---{0}", Environment.NewLine);
+
+            if (InnerException.InnerException != null)
+            {
+                description.AppendFormat("   ---> {0}", InnerException.InnerException);
+                description.AppendFormat("{0}     --- End of inner exception stack trace ---{0}", Environment.NewLine);
+            }
+
+            description.Append(InnerException.StackTrace);
+            description.AppendFormat("{0}   --- End of underlying exception ---", Environment.NewLine);
+
+            var str = description.ToString();
+            return str;
+        }
+    }
+}

src/runtime/Properties/AssemblyInfo.cs

@@ -4,5 +4,5 @@
 [assembly: InternalsVisibleTo("Python.EmbeddingTest, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")]
 [assembly: InternalsVisibleTo("Python.Test, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")]
 
-[assembly: AssemblyVersion("2.0.22")]
-[assembly: AssemblyFileVersion("2.0.22")]
+[assembly: AssemblyVersion("2.0.23")]
+[assembly: AssemblyFileVersion("2.0.23")]

src/runtime/Python.Runtime.csproj

@@ -5,7 +5,7 @@
     <RootNamespace>Python.Runtime</RootNamespace>
     <AssemblyName>Python.Runtime</AssemblyName>
     <PackageId>QuantConnect.pythonnet</PackageId>
-    <Version>2.0.22</Version>
+    <Version>2.0.23</Version>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
     <RepositoryUrl>https://github.com/pythonnet/pythonnet</RepositoryUrl>

src/runtime/PythonException.cs

@@ -163,26 +163,32 @@ private static Exception FromPyErr(BorrowedReference typeRef, BorrowedReference
             var value = new PyObject(valRef);
             var traceback = PyObject.FromNullableReference(tbRef);
 
+            Exception exception = null;
+
             exceptionDispatchInfo = TryGetDispatchInfo(valRef);
             if (exceptionDispatchInfo != null)
             {
-                return exceptionDispatchInfo.SourceException;
+                exception = exceptionDispatchInfo.SourceException;
+                exceptionDispatchInfo = null;
             }
-
-            if (ManagedType.GetManagedObject(valRef) is CLRObject { inst: Exception e })
+            else if (ManagedType.GetManagedObject(valRef) is CLRObject { inst: Exception e })
             {
-                return e;
+                exception = e;
             }
-
-            if (TryDecodePyErr(typeRef, valRef, tbRef) is { } pyErr)
+            else if (TryDecodePyErr(typeRef, valRef, tbRef) is { } pyErr)
             {
-                return pyErr;
+                exception = pyErr;
             }
-
-            if (PyObjectConversions.TryDecode(valRef, typeRef, typeof(Exception), out object? decoded)
+            else if (PyObjectConversions.TryDecode(valRef, typeRef, typeof(Exception), out object? decoded)
                 && decoded is Exception decodedException)
             {
-                return decodedException;
+                exception = decodedException;
+            }
+
+            if (!(exception is null))
+            {
+                using var _ = new Py.GILState();
+                return new ClrBubbledException(exception, TracebackToString(traceback));
             }
 
             using var cause = Runtime.PyException_GetCause(nValRef);