Oct 8, 2020 · Jun 13, 2019 · Jun 13, 2019 · Jun 13, 2019 · Jun 20, 2019 · Jul 1, 2019
diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs
                mode = ShutdownMode.Soft;
            }
            PythonEngine.Initialize(mode: mode);
 using (Py.GIL())
 try
            {
                try
                {
                    var pyScript = string.Format("import clr\n"
                        + "print('[{0} in python] imported clr')\n"
                        + "clr.AddReference('System')\n"
                        + "print('[{0} in python] allocated a clr object')\n"
                        + "import gc\n"
                        + "gc.collect()\n"
                        + "print('[{0} in python] collected garbage')\n",
                        name);
                    PythonEngine.Exec(pyScript);
                }
                catch (Exception e)
                using (Py.GIL())
                {
                    Console.WriteLine(string.Format("[{0} in .NET] Caught exception: {1}", name, e));
                    try
                    {
                        var pyScript = string.Format("import clr\n"
                            + "print('[{0} in python] imported clr')\n"
                            + "clr.AddReference('System')\n"
                            + "print('[{0} in python] allocated a clr object')\n"
                            + "import gc\n"
                            + "gc.collect()\n"
                            + "print('[{0} in python] collected garbage')\n",
                            name);
                        PythonEngine.Exec(pyScript);
                    }
                    catch (Exception e)
                    {
                        Console.WriteLine(string.Format("[{0} in .NET] Caught exception: {1}", name, e));
                        throw;
                    }
                }
            }
            PythonEngine.BeginAllowThreads();
            finally
            {
                PythonEngine.BeginAllowThreads();
            }
        }


diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs
            self.tpHandle = IntPtr.Zero;
            return 0;
        }

        protected override void OnLoad()
        {
            base.OnLoad();
            gcHandle = AllocGCHandle();
            Marshal.WriteIntPtr(pyHandle, TypeOffset.magic(), (IntPtr)gcHandle);
        }
    }
 }
diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs
                    cls.CallTypeTraverse(OnVisit, visitedPtr);
                    // XXX: Force release instance resources but not dealloc itself.
                    cls.CallTypeClear();
                    cls.DecrRefCount();
                }
            }
            finally
            {
                visitedHandle.Free();
            }
            cache.Clear();
        }

        private static int OnVisit(IntPtr ob, IntPtr arg)
            return 0;
        }


        internal static void StashPush(RuntimeDataStorage storage)
        {
            storage.PushValue(cache);
            foreach (var cls in cache.Values)
            {
                // This incref is for cache to hold the cls,
                // thus no need for decreasing it at StashPop.
                Runtime.XIncref(cls.pyHandle);
                cls.Save();
            }
        }

        internal static void StashPop(RuntimeDataStorage storage)
        {
            cache = storage.PopValue<Dictionary<Type, ClassBase>>();
            foreach (var cls in cache.Values)
            {
                cls.Load();
            }
        }

        /// <summary>
diff --git a/src/runtime/eventbinding.cs b/src/runtime/eventbinding.cs
            Runtime.Py_CLEAR(ref self.target);
            return 0;
        }

        protected override void OnSave()
        {
            base.OnSave();
            Runtime.XIncref(target);
        }
    }
 }
diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs
    /// <remarks>
    /// Readability of the Exceptions class improvements as we look toward version 2.7 ...
    /// </remarks>
    public class Exceptions
    publicstaticclass Exceptions
    {
        internal static IntPtr warnings_module;
        internal static IntPtr exceptions_module;

        private Exceptions()
        {
        }

        /// <summary>
        /// Initialization performed on startup of the Python runtime.
        /// </summary>
diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs
            self.Dealloc();
        }

        protected override void OnSave()
        {
            base.OnSave();
            Runtime.XIncref(pyHandle);
        }

        protected override void OnLoad()
        {
            base.OnLoad();
diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs
    /// <summary>
    /// Implements the "import hook" used to integrate Python with the CLR.
    /// </summary>
    internal class ImportHook
    internalstaticclass ImportHook
    {
        private static IntPtr py_import;
        private static CLRModule root;
            CLRModule.Reset();
        }

        internal static void StashPush(RuntimeDataStorage storage)
        {
            Runtime.XIncref(py_clr_module);
            Runtime.XIncref(root.pyHandle);
            storage.AddValue("py_clr_module", py_clr_module);
            storage.AddValue("root", root.pyHandle);
        }

        internal static void StashPop(RuntimeDataStorage storage)
        {
            InitImport();
            storage.GetValue("py_clr_module", out py_clr_module);
            var rootHandle = storage.GetValue<IntPtr>("root");
            root = (CLRModule)ManagedType.GetManagedObject(rootHandle);
        }

        /// <summary>
        /// Return the clr python module (new reference)
        /// </summary>
diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs

        internal void Save()
        {
            Runtime.XIncref(pyHandle);
            OnSave();
        }

        internal void Load()
        {
            OnLoad();
            Runtime.XDecref(pyHandle);
        }

        protected virtual void OnSave() { }
diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs
            {
                m = new ModuleObject(qname);
                StoreAttribute(name, m);
                m.DecrRefCount();
                return m;
            }

                }
                c = ClassManager.GetClass(type);
                StoreAttribute(name, c);
                c.DecrRefCount();
                return c;
            }

                {
                    m = new ModuleObject(qname);
                    StoreAttribute(name, m);
                    m.DecrRefCount();
                    return m;
                }

                    }
                    c = ClassManager.GetClass(type);
                    StoreAttribute(name, c);
                    c.DecrRefCount();
                    return c;
                }
            }
                        mi[0] = method;
                        var m = new ModuleFunctionObject(type, name, mi, allow_threads);
                        StoreAttribute(name, m);
                        m.DecrRefCount();
                    }
                }

                        string name = property.Name;
                        var p = new ModulePropertyObject(property);
                        StoreAttribute(name, p);
                        p.DecrRefCount();
                    }
                }
                type = type.BaseType;
diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
 #endif
            {
                PyCLRMetaType = MetaType.Initialize(); // Steal a reference
                ImportHook.Initialize();
            }
            Exceptions.Initialize();
            ImportHook.Initialize();

            // Need to add the runtime directory to sys.path so that we
            // can find built-in assemblies like System.Data, et. al.
                    obj.CallTypeClear();
                    // obj's tp_type will degenerate to a pure Python type after TypeManager.RemoveTypes(),
                    // thus just be safe to give it back to GC chain.
                    PyObject_GC_Track(obj.pyHandle);
                    if (!_PyObject_GC_IS_TRACKED(obj.pyHandle))
                    {
                        PyObject_GC_Track(obj.pyHandle);
                    }
                }
                if (obj.gcHandle.IsAllocated)
                {
                    obj.gcHandle.Free();
                }
                obj.gcHandle =new GCHandle();
                obj.gcHandle =default;
            }
            ManagedType.ClearTrackedObjects();
        }
diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs
            var metaStorage = new RuntimeDataStorage();
            MetaType.StashPush(metaStorage);

            var importStorage = new RuntimeDataStorage();
            ImportHook.StashPush(importStorage);

            var typeStorage = new RuntimeDataStorage();
            TypeManager.StashPush(typeStorage);


            var runtimeStorage = new RuntimeDataStorage();
            runtimeStorage.AddValue("meta", metaStorage);
            runtimeStorage.AddValue("import", importStorage);
            runtimeStorage.AddValue("types", typeStorage);
            runtimeStorage.AddValue("classes", clsStorage);
            runtimeStorage.AddValue("modules", moduleStorage);
            var formatter = CreateFormatter();
            var storage = (RuntimeDataStorage)formatter.Deserialize(ms);

            StashPopModules(storage.GetStorage("modules"));
            StashPopObjects(storage.GetStorage("objs"));
            StashPopModules(storage.GetStorage("modules"));
            ClassManager.StashPop(storage.GetStorage("classes"));
            TypeManager.StashPop(storage.GetStorage("types"));
            ImportHook.StashPop(storage.GetStorage("import"));
            PyCLRMetaType = MetaType.StashPop(storage.GetStorage("meta"));
        }

            return (T)GetValue(name);
        }

        public T GetValue<T>(string name, out T value)
        {
            value = GetValue<T>(name);
            return value;
        }

        public RuntimeDataStorage GetStorage(string name)
        {
            return GetValue<RuntimeDataStorage>(name);
        {
            return (T)PopValue();
        }

        public T PopValue<T>(out T value)
        {
            return value = (T)PopValue();
        }
    }


diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs
        private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static;
        private static Dictionary<Type, IntPtr> cache = new Dictionary<Type, IntPtr>();
        private static readonly Dictionary<IntPtr, SlotsHolder> _slotsHolders = new Dictionary<IntPtr, SlotsHolder>();
        private static Dictionary<Type, Type> _slotsImpls = new Dictionary<Type, Type>();

        // Slots which must be set
        private static readonly string[] _requiredSlots = new string[]
                Runtime.XDecref(tpHandle);
            }
            cache.Clear();
            _slotsImpls.Clear();
            _slotsHolders.Clear();
        }

            {
                Runtime.XIncref(tpHandle);
            }
            storage.PushValue(cache);
            storage.AddValue("cache", cache);
            storage.AddValue("slots", _slotsImpls);
        }

        internal static void StashPop(RuntimeDataStorage storage)
        {
            Debug.Assert(cache == null || cache.Count == 0);
            cache = storage.PopValue<Dictionary<Type, IntPtr>>();
            storage.GetValue("slots", out _slotsImpls);
            storage.GetValue("cache", out cache);
            foreach (var entry in cache)
            {
                Type type = entry.Key;
                IntPtr handle = entry.Value;
                SlotsHolder holder = CreateSolotsHolder(handle);
                InitializeSlots(handle, type, holder);
                InitializeSlots(handle,_slotsImpls[type], holder);
                // FIXME: mp_length_slot.CanAssgin(clrType)
            }
        }
            }
            handle = CreateType(type);
            cache[type] = handle;
            _slotsImpls.Add(type, type);
            return handle;
        }

            }
            handle = CreateType(obj, type);
            cache[type] = handle;
            _slotsImpls.Add(type, obj.GetType());
            return handle;
        }

                            mi[0] = method;
                            MethodObject m = new TypeMethod(type, method_name, mi);
                            Runtime.PyDict_SetItemString(dict, method_name, m.pyHandle);
                            //m.DecrRefCount();
                            addedMethods.Add(method_name);
                        }
                    }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -233,26 +233,33 @@ public static void RunPython()
		mode = ShutdownMode.Soft;
		}
		PythonEngine.Initialize(mode: mode);
		using (Py.GIL())
		try
		{
		try
		{
		var pyScript = string.Format("import clr\n"
		+ "print('[{0} in python] imported clr')\n"
		+ "clr.AddReference('System')\n"
		+ "print('[{0} in python] allocated a clr object')\n"
		+ "import gc\n"
		+ "gc.collect()\n"
		+ "print('[{0} in python] collected garbage')\n",
		name);
		PythonEngine.Exec(pyScript);
		}
		catch (Exception e)
		using (Py.GIL())
		{
		Console.WriteLine(string.Format("[{0} in .NET] Caught exception: {1}", name, e));
		try
		{
		var pyScript = string.Format("import clr\n"
		+ "print('[{0} in python] imported clr')\n"
		+ "clr.AddReference('System')\n"
		+ "print('[{0} in python] allocated a clr object')\n"
		+ "import gc\n"
		+ "gc.collect()\n"
		+ "print('[{0} in python] collected garbage')\n",
		name);
		PythonEngine.Exec(pyScript);
		}
		catch (Exception e)
		{
		Console.WriteLine(string.Format("[{0} in .NET] Caught exception: {1}", name, e));
		throw;
		}
		}
		}
		PythonEngine.BeginAllowThreads();
		finally
		{
		PythonEngine.BeginAllowThreads();
Copy link Member lostmsuMar 10, 2020 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. Huh? Why here? Where is the matching`EndAllowThreads`?
		}
		}


Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -303,5 +303,12 @@ public static int tp_clear(IntPtr ob)
		self.tpHandle = IntPtr.Zero;
		return 0;
		}

		protected override void OnLoad()
		{
		base.OnLoad();
		gcHandle = AllocGCHandle();
		Marshal.WriteIntPtr(pyHandle, TypeOffset.magic(), (IntPtr)gcHandle);
		}
		}
		}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -56,12 +56,14 @@ internal static void RemoveClasses()
		cls.CallTypeTraverse(OnVisit, visitedPtr);
		// XXX: Force release instance resources but not dealloc itself.
		cls.CallTypeClear();
Copy link Member lostmsuMar 10, 2020 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. NIT:`Dispose`? Copy link Contributor BadSingletonAug 26, 2020 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. Should this implement`IDisposable` ? Else I'd call it another name (like`DisposeType`) because it has meaning in C# Copy link MemberAuthor amos402Aug 27, 2020 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. `IDisposable` is not a fit option because it has no parameters, but it needs a callback and an argument for`tp_traverse` here. About`DisposeType`, do you mean to implement it as a method for`ClassBase`? But it still needs arguments from outside...... So I'm not sure what's the best for here too. btw. I updated the comments.
		cls.DecrRefCount();
		}
		}
		finally
		{
		visitedHandle.Free();
		}
		cache.Clear();
		}

		private static int OnVisit(IntPtr ob, IntPtr arg)
Copy link Member lostmsuMar 10, 2020 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. This should be named`DisposeRecursively` or`ClearRecursively`
Expand All		@@ -81,15 +83,25 @@ private static int OnVisit(IntPtr ob, IntPtr arg)
		return 0;
		}


		internal static void StashPush(RuntimeDataStorage storage)
Copy link Member lostmsuMar 10, 2020 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. `ClassManager.StashPush` - the name is not descriptive enough. It is completely unclear what this method should be doing. Same with`StashPop` below.
		{
		storage.PushValue(cache);
		foreach (var cls in cache.Values)
		{
		// This incref is for cache to hold the cls,
		// thus no need for decreasing it at StashPop.
		Runtime.XIncref(cls.pyHandle);
		cls.Save();
		}
		}

		internal static void StashPop(RuntimeDataStorage storage)
		{
		cache = storage.PopValue<Dictionary<Type, ClassBase>>();
		foreach (var cls in cache.Values)
		{
		cls.Load();
		}
		}

		/// <summary>
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -128,11 +128,5 @@ public static int tp_clear(IntPtr ob)
		Runtime.Py_CLEAR(ref self.target);
		return 0;
		}

		protected override void OnSave()
		{
		base.OnSave();
		Runtime.XIncref(target);
		}
		}
		}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -89,15 +89,11 @@ internal static Exception ToException(IntPtr ob)
		/// <remarks>
		/// Readability of the Exceptions class improvements as we look toward version 2.7 ...
		/// </remarks>
		public class Exceptions
		publicstaticclass Exceptions
		{
		internal static IntPtr warnings_module;
		internal static IntPtr exceptions_module;

		private Exceptions()
		{
		}

		/// <summary>
		/// Initialization performed on startup of the Python runtime.
		/// </summary>
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -98,12 +98,6 @@ public static void tp_dealloc(IntPtr ob)
		self.Dealloc();
		}

		protected override void OnSave()
		{
		base.OnSave();
		Runtime.XIncref(pyHandle);
		}

		protected override void OnLoad()
		{
		base.OnLoad();
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -6,7 +6,7 @@ namespace Python.Runtime
		/// <summary>
		/// Implements the "import hook" used to integrate Python with the CLR.
		/// </summary>
		internal class ImportHook
		internalstaticclass ImportHook
		{
		private static IntPtr py_import;
		private static CLRModule root;
Expand DownExpand Up		@@ -130,6 +130,22 @@ internal static void Shutdown()
		CLRModule.Reset();
		}

		internal static void StashPush(RuntimeDataStorage storage)
		{
		Runtime.XIncref(py_clr_module);
		Runtime.XIncref(root.pyHandle);
Comment on lines +129 to +130 Copy link Member lostmsuMar 11, 2020 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. Don't yo need a corresponding decref in`StashPop`? Copy link Contributor BadSingletonSep 8, 2020 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. I added a comment in the code for that one. The shutdown process decrefs these objects, so we need to incref them here to keep them alive through the domain reload.
		storage.AddValue("py_clr_module", py_clr_module);
		storage.AddValue("root", root.pyHandle);
		}

		internal static void StashPop(RuntimeDataStorage storage)
		{
		InitImport();
		storage.GetValue("py_clr_module", out py_clr_module);
		var rootHandle = storage.GetValue<IntPtr>("root");
		root = (CLRModule)ManagedType.GetManagedObject(rootHandle);
		}

		/// <summary>
		/// Return the clr python module (new reference)
		/// </summary>
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -202,12 +202,14 @@ protected void TypeClear()

		internal void Save()
		{
		Runtime.XIncref(pyHandle);
		OnSave();
		}

		internal void Load()
		{
		OnLoad();
		Runtime.XDecref(pyHandle);
		}

		protected virtual void OnSave() { }
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -103,6 +103,7 @@ public ManagedType GetAttribute(string name, bool guess)
		{
		m = new ModuleObject(qname);
		StoreAttribute(name, m);
		m.DecrRefCount();
Copy link Member lostmsuMar 11, 2020 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. It seems wasteful to incref in`StoreAttrbiute` since you are decref'ing after nearly ever`StoreAttribute` call. It is also hard to check where do you need vs not need to decref after`StoreAttribute`. Copy link MemberAuthor amos402Aug 27, 2020 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others.Learn more. It's due to the sole behavior of`StoreAttribute`, it's not only called by here, if we treat this interface alone, it should increase the reference.
		return m;
		}

Expand All		@@ -118,6 +119,7 @@ public ManagedType GetAttribute(string name, bool guess)
		}
		c = ClassManager.GetClass(type);
		StoreAttribute(name, c);
		c.DecrRefCount();
		return c;
		}

Expand All		@@ -132,6 +134,7 @@ public ManagedType GetAttribute(string name, bool guess)
		{
		m = new ModuleObject(qname);
		StoreAttribute(name, m);
		m.DecrRefCount();
		return m;
		}

Expand All		@@ -144,6 +147,7 @@ public ManagedType GetAttribute(string name, bool guess)
		}
		c = ClassManager.GetClass(type);
		StoreAttribute(name, c);
		c.DecrRefCount();
		return c;
		}
		}
Expand DownExpand Up		@@ -239,6 +243,7 @@ internal void InitializeModuleMembers()
		mi[0] = method;
		var m = new ModuleFunctionObject(type, name, mi, allow_threads);
		StoreAttribute(name, m);
		m.DecrRefCount();
		}
		}

Expand All		@@ -251,6 +256,7 @@ internal void InitializeModuleMembers()
		string name = property.Name;
		var p = new ModulePropertyObject(property);
		StoreAttribute(name, p);
		p.DecrRefCount();
		}
		}
		type = type.BaseType;
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -197,9 +197,9 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd
		#endif
		{
		PyCLRMetaType = MetaType.Initialize(); // Steal a reference
		ImportHook.Initialize();
		}
		Exceptions.Initialize();
		ImportHook.Initialize();

		// Need to add the runtime directory to sys.path so that we
		// can find built-in assemblies like System.Data, et. al.
Expand DownExpand Up		@@ -497,13 +497,16 @@ private static void MoveClrInstancesOnwershipToPython()
		obj.CallTypeClear();
		// obj's tp_type will degenerate to a pure Python type after TypeManager.RemoveTypes(),
		// thus just be safe to give it back to GC chain.
		PyObject_GC_Track(obj.pyHandle);
		if (!_PyObject_GC_IS_TRACKED(obj.pyHandle))
		{
		PyObject_GC_Track(obj.pyHandle);
		}
		}
		if (obj.gcHandle.IsAllocated)
		{
		obj.gcHandle.Free();
		}
		obj.gcHandle =new GCHandle();
		obj.gcHandle =default;
		}
		ManagedType.ClearTrackedObjects();
		}
Expand Down