- Notifications
You must be signed in to change notification settings - Fork750
Enable pythonnet to survive in the Unity3d editor environment.#752
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.
Already on GitHub?Sign in to your account
Merged
Uh oh!
There was an error while loading.Please reload this page.
Merged
Changes fromall commits
Commits
Show all changes
3 commits Select commitHold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Diff view
Diff view
There are no files selected for viewing
3 changes: 3 additions & 0 deletionsAUTHORS.md
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
2 changes: 2 additions & 0 deletionsCHANGELOG.md
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
1 change: 1 addition & 0 deletionssrc/embed_tests/Python.EmbeddingTest.15.csproj
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
4 changes: 3 additions & 1 deletionsrc/embed_tests/Python.EmbeddingTest.csproj
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
239 changes: 239 additions & 0 deletionssrc/embed_tests/TestDomainReload.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,239 @@ | ||
| using System; | ||
| using System.CodeDom.Compiler; | ||
| using System.Reflection; | ||
| using NUnit.Framework; | ||
| using Python.Runtime; | ||
| // | ||
| // This test case is disabled on .NET Standard because it doesn't have all the | ||
| // APIs we use. We could work around that, but .NET Core doesn't implement | ||
| // domain creation, so it's not worth it. | ||
| // | ||
| // Unfortunately this means no continuous integration testing for this case. | ||
| // | ||
| #if !NETSTANDARD && !NETCOREAPP | ||
| namespace Python.EmbeddingTest | ||
| { | ||
| class TestDomainReload | ||
| { | ||
| /// <summary> | ||
| /// Test that the python runtime can survive a C# domain reload without crashing. | ||
| /// | ||
| /// At the time this test was written, there was a very annoying | ||
| /// seemingly random crash bug when integrating pythonnet into Unity. | ||
| /// | ||
| /// The repro steps that David Lassonde, Viktoria Kovecses and | ||
| /// Benoit Hudson eventually worked out: | ||
| /// 1. Write a HelloWorld.cs script that uses Python.Runtime to access | ||
| /// some C# data from python: C# calls python, which calls C#. | ||
| /// 2. Execute the script (e.g. make it a MenuItem and click it). | ||
| /// 3. Touch HelloWorld.cs on disk, forcing Unity to recompile scripts. | ||
| /// 4. Wait several seconds for Unity to be done recompiling and | ||
| /// reloading the C# domain. | ||
| /// 5. Make python run the gc (e.g. by calling gc.collect()). | ||
| /// | ||
| /// The reason: | ||
| /// A. In step 2, Python.Runtime registers a bunch of new types with | ||
| /// their tp_traverse slot pointing to managed code, and allocates | ||
| /// some objects of those types. | ||
| /// B. In step 4, Unity unloads the C# domain. That frees the managed | ||
| /// code. But at the time of the crash investigation, pythonnet | ||
| /// leaked the python side of the objects allocated in step 1. | ||
| /// C. In step 5, python sees some pythonnet objects in its gc list of | ||
| /// potentially-leaked objects. It calls tp_traverse on those objects. | ||
| /// But tp_traverse was freed in step 3 => CRASH. | ||
| /// | ||
| /// This test distills what's going on without needing Unity around (we'd see | ||
| /// similar behaviour if we were using pythonnet on a .NET web server that did | ||
| /// a hot reload). | ||
| /// </summary> | ||
| [Test] | ||
| public static void DomainReloadAndGC() | ||
| { | ||
| // We're set up to run in the directory that includes the bin directory. | ||
| System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); | ||
| Assembly pythonRunner1 = BuildAssembly("test1"); | ||
| RunAssemblyAndUnload(pythonRunner1, "test1"); | ||
| // Verify that python is not initialized even though we ran it. | ||
| Assert.That(Runtime.Runtime.Py_IsInitialized(), Is.Zero); | ||
| // This caused a crash because objects allocated in pythonRunner1 | ||
| // still existed in memory, but the code to do python GC on those | ||
| // objects is gone. | ||
| Assembly pythonRunner2 = BuildAssembly("test2"); | ||
| RunAssemblyAndUnload(pythonRunner2, "test2"); | ||
| } | ||
| // | ||
| // The code we'll test. All that really matters is | ||
| // using GIL { Python.Exec(pyScript); } | ||
| // but the rest is useful for debugging. | ||
| // | ||
| // What matters in the python code is gc.collect and clr.AddReference. | ||
| // | ||
| // Note that the language version is 2.0, so no $"foo{bar}" syntax. | ||
| // | ||
| const string TestCode = @" | ||
| using Python.Runtime; | ||
| using System; | ||
| class PythonRunner { | ||
| public static void RunPython() { | ||
| AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; | ||
| string name = AppDomain.CurrentDomain.FriendlyName; | ||
| Console.WriteLine(string.Format(""[{0} in .NET] In PythonRunner.RunPython"", name)); | ||
| using (Py.GIL()) { | ||
| try { | ||
| var pyScript = string.Format(""import clr\n"" | ||
| + ""print('[{0} in python] imported clr')\n"" | ||
| + ""clr.AddReference('System')\n"" | ||
| + ""print('[{0} in python] allocated a clr object')\n"" | ||
| + ""import gc\n"" | ||
| + ""gc.collect()\n"" | ||
| + ""print('[{0} in python] collected garbage')\n"", | ||
| name); | ||
| PythonEngine.Exec(pyScript); | ||
| } catch(Exception e) { | ||
| Console.WriteLine(string.Format(""[{0} in .NET] Caught exception: {1}"", name, e)); | ||
| } | ||
| } | ||
| } | ||
| static void OnDomainUnload(object sender, EventArgs e) { | ||
| System.Console.WriteLine(string.Format(""[{0} in .NET] unloading"", AppDomain.CurrentDomain.FriendlyName)); | ||
| } | ||
| }"; | ||
| /// <summary> | ||
| /// Build an assembly out of the source code above. | ||
| /// | ||
| /// This creates a file <paramref name="assemblyName"/>.dll in order | ||
| /// to support the statement "proxy.theAssembly = assembly" below. | ||
| /// That statement needs a file, can't run via memory. | ||
| /// </summary> | ||
| static Assembly BuildAssembly(string assemblyName) | ||
| { | ||
| var provider = CodeDomProvider.CreateProvider("CSharp"); | ||
| var compilerparams = new CompilerParameters(); | ||
| compilerparams.ReferencedAssemblies.Add("Python.Runtime.dll"); | ||
| compilerparams.GenerateExecutable = false; | ||
| compilerparams.GenerateInMemory = false; | ||
| compilerparams.IncludeDebugInformation = false; | ||
| compilerparams.OutputAssembly = assemblyName; | ||
| var results = provider.CompileAssemblyFromSource(compilerparams, TestCode); | ||
| if (results.Errors.HasErrors) | ||
| { | ||
| var errors = new System.Text.StringBuilder("Compiler Errors:\n"); | ||
| foreach (CompilerError error in results.Errors) | ||
| { | ||
| errors.AppendFormat("Line {0},{1}\t: {2}\n", | ||
| error.Line, error.Column, error.ErrorText); | ||
| } | ||
| throw new Exception(errors.ToString()); | ||
| } | ||
| else | ||
| { | ||
| return results.CompiledAssembly; | ||
| } | ||
| } | ||
| /// <summary> | ||
| /// This is a magic incantation required to run code in an application | ||
| /// domain other than the current one. | ||
| /// </summary> | ||
| class Proxy : MarshalByRefObject | ||
| { | ||
| Assembly theAssembly = null; | ||
| public void InitAssembly(string assemblyPath) | ||
| { | ||
| theAssembly = Assembly.LoadFile(System.IO.Path.GetFullPath(assemblyPath)); | ||
| } | ||
| public void RunPython() | ||
| { | ||
| Console.WriteLine("[Proxy] Entering RunPython"); | ||
| // Call into the new assembly. Will execute Python code | ||
| var pythonrunner = theAssembly.GetType("PythonRunner"); | ||
| var runPythonMethod = pythonrunner.GetMethod("RunPython"); | ||
| runPythonMethod.Invoke(null, new object[] { }); | ||
| Console.WriteLine("[Proxy] Leaving RunPython"); | ||
| } | ||
| } | ||
| /// <summary> | ||
| /// Create a domain, run the assembly in it (the RunPython function), | ||
| /// and unload the domain. | ||
| /// </summary> | ||
| static void RunAssemblyAndUnload(Assembly assembly, string assemblyName) | ||
| { | ||
| Console.WriteLine($"[Program.Main] === creating domain for assembly {assembly.FullName}"); | ||
| // Create the domain. Make sure to set PrivateBinPath to a relative | ||
| // path from the CWD (namely, 'bin'). | ||
| // See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain | ||
| var currentDomain = AppDomain.CurrentDomain; | ||
| var domainsetup = new AppDomainSetup() | ||
| { | ||
| ApplicationBase = currentDomain.SetupInformation.ApplicationBase, | ||
| ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, | ||
| LoaderOptimization = LoaderOptimization.SingleDomain, | ||
| PrivateBinPath = "." | ||
| }; | ||
| var domain = AppDomain.CreateDomain( | ||
| $"My Domain {assemblyName}", | ||
| currentDomain.Evidence, | ||
| domainsetup); | ||
| // Create a Proxy object in the new domain, where we want the | ||
| // assembly (and Python .NET) to reside | ||
| Type type = typeof(Proxy); | ||
| System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); | ||
| var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( | ||
| type.Assembly.FullName, | ||
| type.FullName); | ||
| // From now on use the Proxy to call into the new assembly | ||
| theProxy.InitAssembly(assemblyName); | ||
| theProxy.RunPython(); | ||
| Console.WriteLine($"[Program.Main] Before Domain Unload on {assembly.FullName}"); | ||
| AppDomain.Unload(domain); | ||
| Console.WriteLine($"[Program.Main] After Domain Unload on {assembly.FullName}"); | ||
| // Validate that the assembly does not exist anymore | ||
| try | ||
| { | ||
| Console.WriteLine($"[Program.Main] The Proxy object is valid ({theProxy}). Unexpected domain unload behavior"); | ||
| } | ||
| catch (Exception) | ||
| { | ||
| Console.WriteLine("[Program.Main] The Proxy object is not valid anymore, domain unload complete."); | ||
| } | ||
| } | ||
| /// <summary> | ||
| /// Resolves the assembly. Why doesn't this just work normally? | ||
| /// </summary> | ||
| static Assembly ResolveAssembly(object sender, ResolveEventArgs args) | ||
| { | ||
| var loadedAssemblies = AppDomain.CurrentDomain.GetAssemblies(); | ||
| foreach (var assembly in loadedAssemblies) | ||
| { | ||
| if (assembly.FullName == args.Name) | ||
| { | ||
| return assembly; | ||
| } | ||
| } | ||
| return null; | ||
| } | ||
| } | ||
| } | ||
| #endif |
20 changes: 20 additions & 0 deletionssrc/embed_tests/TestRuntime.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
65 changes: 65 additions & 0 deletionssrc/embed_tests/TestTypeManager.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| using NUnit.Framework; | ||
| using Python.Runtime; | ||
| using System.Runtime.InteropServices; | ||
| namespace Python.EmbeddingTest | ||
| { | ||
| class TestTypeManager | ||
| { | ||
| [SetUp] | ||
| public static void Init() | ||
| { | ||
| Runtime.Runtime.Initialize(); | ||
| } | ||
| [TearDown] | ||
| public static void Fini() | ||
| { | ||
| // Don't shut down the runtime: if the python engine was initialized | ||
| // but not shut down by another test, we'd end up in a bad state. | ||
| } | ||
| [Test] | ||
| public static void TestNativeCode() | ||
| { | ||
| Assert.That(() => { var _ = TypeManager.NativeCode.Active; }, Throws.Nothing); | ||
| Assert.That(TypeManager.NativeCode.Active.Code.Length, Is.GreaterThan(0)); | ||
| } | ||
| [Test] | ||
| public static void TestMemoryMapping() | ||
| { | ||
| Assert.That(() => { var _ = TypeManager.CreateMemoryMapper(); }, Throws.Nothing); | ||
| var mapper = TypeManager.CreateMemoryMapper(); | ||
| // Allocate a read-write page. | ||
| int len = 12; | ||
| var page = mapper.MapWriteable(len); | ||
| Assert.That(() => { Marshal.WriteInt64(page, 17); }, Throws.Nothing); | ||
| Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); | ||
| // Mark it read-execute. We can still read, haven't changed any values. | ||
| mapper.SetReadExec(page, len); | ||
| Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); | ||
| // Test that we can't write to the protected page. | ||
| // | ||
| // We can't actually test access protection under Microsoft | ||
| // versions of .NET, because AccessViolationException is assumed to | ||
| // mean we're in a corrupted state: | ||
| // https://stackoverflow.com/questions/3469368/how-to-handle-accessviolationexception | ||
| // | ||
| // We can test under Mono but it throws NRE instead of AccessViolationException. | ||
| // | ||
| // We can't use compiler flags because we compile with MONO_LINUX | ||
| // while running on the Microsoft .NET Core during continuous | ||
| // integration tests. | ||
| if (System.Type.GetType ("Mono.Runtime") != null) | ||
| { | ||
| // Mono throws NRE instead of AccessViolationException for some reason. | ||
| Assert.That(() => { Marshal.WriteInt64(page, 73); }, Throws.TypeOf<System.NullReferenceException>()); | ||
| Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); | ||
| } | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.
Uh oh!
There was an error while loading.Please reload this page.
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.