Merge pull request #71 from Martin-Molinero/bug-remove-reference-tracking

Remove managed reference tracking

Author: Martin-Molinero

src/perf_tests/Python.PerformanceTests.csproj

@@ -13,7 +13,7 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.*" />
-    <PackageReference Include="quantconnect.pythonnet" Version="2.0.17" GeneratePathProperty="true">
+    <PackageReference Include="quantconnect.pythonnet" Version="2.0.18" GeneratePathProperty="true">
       <IncludeAssets>compile</IncludeAssets>
     </PackageReference>
   </ItemGroup>
@@ -25,7 +25,7 @@
   </Target>
 
   <Target Name="CopyBaseline" AfterTargets="Build">
-    <Copy SourceFiles="$(NuGetPackageRoot)quantconnect.pythonnet\2.0.17\lib\net5.0\Python.Runtime.dll" DestinationFolder="$(OutDir)baseline" />
+    <Copy SourceFiles="$(NuGetPackageRoot)quantconnect.pythonnet\2.0.18\lib\net5.0\Python.Runtime.dll" DestinationFolder="$(OutDir)baseline" />
   </Target>
 
   <Target Name="CopyNewBuild" AfterTargets="Build">

src/runtime/Properties/AssemblyInfo.cs

@@ -4,5 +4,5 @@
 [assembly: InternalsVisibleTo("Python.EmbeddingTest, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")]
 [assembly: InternalsVisibleTo("Python.Test, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")]
 
-[assembly: AssemblyVersion("2.0.17")]
-[assembly: AssemblyFileVersion("2.0.17")]
+[assembly: AssemblyVersion("2.0.18")]
+[assembly: AssemblyFileVersion("2.0.18")]

src/runtime/Python.Runtime.csproj

@@ -5,7 +5,7 @@
     <RootNamespace>Python.Runtime</RootNamespace>
     <AssemblyName>Python.Runtime</AssemblyName>
     <PackageId>QuantConnect.pythonnet</PackageId>
-    <Version>2.0.17</Version>
+    <Version>2.0.18</Version>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
     <RepositoryUrl>https://github.com/pythonnet/pythonnet</RepositoryUrl>

src/runtime/Runtime.cs

@@ -280,7 +280,6 @@ internal static void Shutdown()
             ClearClrModules();
             RemoveClrRootModule();
 
-            NullGCHandles(ExtensionType.loadedExtensions);
             ClassManager.RemoveClasses();
             TypeManager.RemoveTypes();
             _typesInitialized = false;
@@ -319,8 +318,6 @@ internal static void Shutdown()
                     PyEval_SaveThread();
                 }
 
-                ExtensionType.loadedExtensions.Clear();
-                CLRObject.reflectedObjects.Clear();
             }
             else
             {
@@ -349,11 +346,6 @@ static bool TryCollectingGarbage(int runs, bool forceBreakLoops)
                 {
                     if (attempt + 1 == runs) return true;
                 }
-                else if (forceBreakLoops)
-                {
-                    NullGCHandles(CLRObject.reflectedObjects);
-                    CLRObject.reflectedObjects.Clear();
-                }
             }
             return false;
         }

src/runtime/StateSerialization/RuntimeData.cs

@@ -140,57 +140,9 @@ static bool CheckSerializable (object o)
         private static SharedObjectsState SaveRuntimeDataObjects()
         {
             var contexts = new Dictionary<PyObject, Dictionary<string, object?>>(PythonReferenceComparer.Instance);
-            var extensionObjs = new Dictionary<PyObject, ExtensionType>(PythonReferenceComparer.Instance);
-            // make a copy with strongly typed references to avoid concurrent modification
-            var extensions = ExtensionType.loadedExtensions
-                                .Select(addr => new PyObject(
-                                    new BorrowedReference(addr),
-                                    // if we don't skip collect, finalizer might modify loadedExtensions
-                                    skipCollect: true))
-                                .ToArray();
-            foreach (var pyObj in extensions)
-            {
-                var extension = (ExtensionType)ManagedType.GetManagedObject(pyObj)!;
-                Debug.Assert(CheckSerializable(extension));
-                var context = extension.Save(pyObj);
-                if (context is not null)
-                {
-                    contexts[pyObj] = context;
-                }
-                extensionObjs.Add(pyObj, extension);
-            }
 
             var wrappers = new Dictionary<object, List<CLRObject>>();
             var userObjects = new CLRWrapperCollection();
-            // make a copy with strongly typed references to avoid concurrent modification
-            var reflectedObjects = CLRObject.reflectedObjects
-                                    .Select(addr => new PyObject(
-                                        new BorrowedReference(addr),
-                                        // if we don't skip collect, finalizer might modify reflectedObjects
-                                        skipCollect: true))
-                                    .ToList();
-            foreach (var pyObj in reflectedObjects)
-            {
-                // Wrapper must be the CLRObject
-                var clrObj = (CLRObject)ManagedType.GetManagedObject(pyObj)!;
-                object inst = clrObj.inst;
-                List<CLRObject> mappedObjs;
-                if (!userObjects.TryGetValue(inst, out var item))
-                {
-                    item = new CLRMappedItem(inst);
-                    userObjects.Add(item);
-
-                    Debug.Assert(!wrappers.ContainsKey(inst));
-                    mappedObjs = new List<CLRObject>();
-                    wrappers.Add(inst, mappedObjs);
-                }
-                else
-                {
-                    mappedObjs = wrappers[inst];
-                }
-                item.AddRef(pyObj);
-                mappedObjs.Add(clrObj);
-            }
 
             var wrapperStorage = new Dictionary<string, object?>();
             WrappersStorer?.Store(userObjects, wrapperStorage);
@@ -215,7 +167,6 @@ private static SharedObjectsState SaveRuntimeDataObjects()
             return new()
             {
                 InternalStores = internalStores,
-                Extensions = extensionObjs,
                 Wrappers = wrapperStorage,
                 Contexts = contexts,
             };

src/runtime/Types/ClassBase.cs

@@ -352,12 +352,7 @@ public static int tp_clear(BorrowedReference ob)
                 Runtime.PyObject_ClearWeakRefs(ob);
             }
 
-            if (TryFreeGCHandle(ob))
-            {
-                IntPtr addr = ob.DangerousGetAddress();
-                bool deleted = CLRObject.reflectedObjects.Remove(addr);
-                Debug.Assert(deleted);
-            }
+            TryFreeGCHandle(ob);
 
             int baseClearResult = BaseUnmanagedClear(ob);
             if (baseClearResult != 0)

src/runtime/Types/ClrObject.cs

@@ -11,8 +11,6 @@ internal sealed class CLRObject : ManagedType
     {
         internal readonly object inst;
 
-        // "borrowed" references
-        internal static readonly HashSet<IntPtr> reflectedObjects = new();
         static NewReference Create(object ob, BorrowedReference tp)
         {
             Debug.Assert(tp != null);
@@ -23,8 +21,6 @@ static NewReference Create(object ob, BorrowedReference tp)
             GCHandle gc = GCHandle.Alloc(self);
             InitGCHandle(py.Borrow(), type: tp, gc);
 
-            bool isNew = reflectedObjects.Add(py.DangerousGetAddress());
-            Debug.Assert(isNew);
 
             // Fix the BaseException args (and __cause__ in case of Python 3)
             // slot if wrapping a CLR exception
@@ -64,9 +60,6 @@ protected override void OnLoad(BorrowedReference ob, Dictionary<string, object?>
             base.OnLoad(ob, context);
             GCHandle gc = GCHandle.Alloc(this);
             SetGCHandle(ob, gc);
-
-            bool isNew = reflectedObjects.Add(ob.DangerousGetAddress());
-            Debug.Assert(isNew);
         }
     }
 }

src/runtime/Types/ExtensionType.cs

@@ -42,16 +42,11 @@ public virtual NewReference Alloc()
 
         public PyObject AllocObject() => new PyObject(Alloc().Steal());
 
-        // "borrowed" references
-        internal static readonly HashSet<IntPtr> loadedExtensions = new();
         void SetupGc (BorrowedReference ob, BorrowedReference tp)
         {
             GCHandle gc = GCHandle.Alloc(this);
             InitGCHandle(ob, tp, gc);
 
-            bool isNew = loadedExtensions.Add(ob.DangerousGetAddress());
-            Debug.Assert(isNew);
-
             // We have to support gc because the type machinery makes it very
             // hard not to - but we really don't have a need for it in most
             // concrete extension types, so untrack the object to save calls
@@ -92,11 +87,7 @@ public static int tp_clear(BorrowedReference ob)
                 Runtime.PyObject_ClearWeakRefs(ob);
             }
 
-            if (TryFreeGCHandle(ob))
-            {
-                bool deleted = loadedExtensions.Remove(ob.DangerousGetAddress());
-                Debug.Assert(deleted);
-            }
+            TryFreeGCHandle(ob);
 
             int res = ClassBase.BaseUnmanagedClear(ob);
             return res;