Sep 23, 2021 · May 24, 2020 · Jun 3, 2020 · Jun 17, 2020 · Jun 17, 2020 · Sep 5, 2021
diff --git a/CHANGELOG.md b/CHANGELOG.md
 -    Improved exception handling:
 -   exceptions can now be converted with codecs
 -   `InnerException` and `__cause__` are propagated properly
 -   .NET collection types now implement standard Python collection interfaces from `collections.abc`.
 See [Mixins/collections.py](src/runtime/Mixins/collections.py).
 -   .NET arrays implement Python buffer protocol


diff --git a/src/runtime/InteropConfiguration.cs b/src/runtime/InteropConfiguration.cs
    using System;
    using System.Collections.Generic;

    using Python.Runtime.Mixins;

    public sealed class InteropConfiguration
    {
        internal readonly PythonBaseTypeProviderGroup pythonBaseTypeProviders
                PythonBaseTypeProviders =
                {
                    DefaultBaseTypeProvider.Instance,
                    new CollectionMixinsProvider(new Lazy<PyObject>(() => Py.Import("clr._extras.collections"))),
                },
            };
        }
diff --git a/src/runtime/Mixins/CollectionMixinsProvider.cs b/src/runtime/Mixins/CollectionMixinsProvider.cs
 using System;
 using System.Collections.Generic;
 using System.Linq;

 namespace Python.Runtime.Mixins
 {
    class CollectionMixinsProvider : IPythonBaseTypeProvider
    {
        readonly Lazy<PyObject> mixinsModule;
        public CollectionMixinsProvider(Lazy<PyObject> mixinsModule)
        {
            this.mixinsModule = mixinsModule ?? throw new ArgumentNullException(nameof(mixinsModule));
        }

        public PyObject Mixins => this.mixinsModule.Value;

        public IEnumerable<PyType> GetBaseTypes(Type type, IList<PyType> existingBases)
        {
            if (type is null)
                throw new ArgumentNullException(nameof(type));

            if (existingBases is null)
                throw new ArgumentNullException(nameof(existingBases));

            var interfaces = NewInterfaces(type).Select(GetDefinition).ToArray();

            var newBases = new List<PyType>();
            newBases.AddRange(existingBases);

            // dictionaries
            if (interfaces.Contains(typeof(IDictionary<,>)))
            {
                newBases.Add(new PyType(this.Mixins.GetAttr("MutableMappingMixin")));
            }
            else if (interfaces.Contains(typeof(IReadOnlyDictionary<,>)))
            {
                newBases.Add(new PyType(this.Mixins.GetAttr("MappingMixin")));
            }

            // item collections
            if (interfaces.Contains(typeof(IList<>))
                || interfaces.Contains(typeof(System.Collections.IList)))
            {
                newBases.Add(new PyType(this.Mixins.GetAttr("MutableSequenceMixin")));
            }
            else if (interfaces.Contains(typeof(IReadOnlyList<>)))
            {
                newBases.Add(new PyType(this.Mixins.GetAttr("SequenceMixin")));
            }
            else if (interfaces.Contains(typeof(ICollection<>))
                     || interfaces.Contains(typeof(System.Collections.ICollection)))
            {
                newBases.Add(new PyType(this.Mixins.GetAttr("CollectionMixin")));
            }
            else if (interfaces.Contains(typeof(System.Collections.IEnumerable)))
            {
                newBases.Add(new PyType(this.Mixins.GetAttr("IterableMixin")));
            }

            // enumerators
            if (interfaces.Contains(typeof(System.Collections.IEnumerator)))
            {
                newBases.Add(new PyType(this.Mixins.GetAttr("IteratorMixin")));
            }

            if (newBases.Count == existingBases.Count)
            {
                return existingBases;
            }

            if (type.IsInterface && type.BaseType is null)
            {
                newBases.RemoveAll(@base => @base.Handle == Runtime.PyBaseObjectType);
            }

            return newBases;
        }

        static Type[] NewInterfaces(Type type)
        {
            var result = type.GetInterfaces();
            return type.BaseType != null
                ? result.Except(type.BaseType.GetInterfaces()).ToArray()
                : result;
        }

        static Type GetDefinition(Type type)
            => type.IsGenericType ? type.GetGenericTypeDefinition() : type;
    }
 }
diff --git a/src/runtime/Mixins/collections.py b/src/runtime/Mixins/collections.py
 """
 Implements collections.abc for common .NET types
 https://docs.python.org/3.6/library/collections.abc.html
 """

 import collections.abc as col

 class IteratorMixin(col.Iterator):
    def close(self):
        self.Dispose()

 class IterableMixin(col.Iterable):
    pass

 class SizedMixin(col.Sized):
    def __len__(self): return self.Count

 class ContainerMixin(col.Container):
    def __contains__(self, item): return self.Contains(item)

 try:
    abc_Collection = col.Collection
 except AttributeError:
    # Python 3.5- does not have collections.abc.Collection
    abc_Collection = col.Container

 class CollectionMixin(SizedMixin, IterableMixin, ContainerMixin, abc_Collection):
    pass

 class SequenceMixin(CollectionMixin, col.Sequence):
    pass

 class MutableSequenceMixin(SequenceMixin, col.MutableSequence):
    pass

 class MappingMixin(CollectionMixin, col.Mapping):
    def __contains__(self, item): return self.ContainsKey(item)
    def keys(self): return self.Keys
    def items(self): return [(k,self.get(k)) for k in self.Keys]
    def values(self): return self.Values
    def __iter__(self): return self.Keys.__iter__()
    def get(self, key, default=None):
        existed, item = self.TryGetValue(key, None)
        return item if existed else default

 class MutableMappingMixin(MappingMixin, col.MutableMapping):
    _UNSET_ = object()

    def __delitem__(self, key):
        self.Remove(key)

    def clear(self):
        self.Clear()

    def pop(self, key, default=_UNSET_):
        existed, item = self.TryGetValue(key, None)
        if existed:
            self.Remove(key)
            return item
        elif default == self._UNSET_:
            raise KeyError(key)
        else:
            return default

    def setdefault(self, key, value=None):
        existed, item = self.TryGetValue(key, None)
        if existed:
            return item
        else:
            self[key] = value
            return value

    def update(self, items, **kwargs):
        if isinstance(items, col.Mapping):
            for key, value in items.items():
                self[key] = value
        else:
            for key, value in items:
                self[key] = value

        for key, value in kwargs.items():
            self[key] = value
diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj
  </ItemGroup>

  <ItemGroup>
    <None Remove="resources\clr.py" />
    <EmbeddedResource Include="resources\clr.py">
      <LogicalName>clr.py</LogicalName>
    </EmbeddedResource>
    <EmbeddedResource Include="resources\interop.py">
      <LogicalName>interop.py</LogicalName>
    </EmbeddedResource>
    <EmbeddedResource Include="Mixins\*.py" />
  </ItemGroup>

  <ItemGroup>
diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs
        internal const string MinimalPythonVersionRequired =
            "Only Python 3.5 or newer is supported";

        internal const string UseOverloadWithReferenceTypes =
            "This API is unsafe, and will be removed in the future. Use overloads working with *Reference types";

        internal static Int64 ReadCLong(IntPtr tp, int offset)
        {
            // On Windows, a C long is always 32 bits.
diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs

        public static int tp_clear(IntPtr ob)
        {
            ManagedType self = GetManagedObject(ob);
            if (GetManagedObject(ob) is { } self)
            {
                if (self.clearReentryGuard) return 0;

                // workaround for https://bugs.python.org/issue45266
                self.clearReentryGuard = true;

                try
                {
                    return ClearImpl(ob, self);
                }
                finally
                {
                    self.clearReentryGuard = false;
                }
            }
            else
            {
                return ClearImpl(ob, null);
            }
        }

        static int ClearImpl(IntPtr ob, ManagedType self)
        {
            bool isTypeObject = Runtime.PyObject_TYPE(ob) == Runtime.PyCLRMetaType;
            if (!isTypeObject)
            {
                ClearObjectDict(ob);

                int baseClearResult = BaseUnmanagedClear(ob);
                if (baseClearResult != 0)
                {
                    return baseClearResult;
                }

                ClearObjectDict(ob);
            }
            return 0;
        }
diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs

        internal CLRObject(object ob, IntPtr tp)
        {
 System.Diagnostics.Debug.Assert(tp != IntPtr.Zero);
            Debug.Assert(tp != IntPtr.Zero);
            IntPtr py = Runtime.PyType_GenericAlloc(tp, 0);

            tpHandle = tp;
diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs
        internal IntPtr pyHandle; // PyObject *
        internal IntPtr tpHandle; // PyType *

        internal bool clearReentryGuard;

        internal BorrowedReference ObjectReference
        {
            get

        internal static bool IsManagedType(BorrowedReference type)
        {
            var flags =(TypeFlags)Util.ReadCLong(type.DangerousGetAddress(), TypeOffset.tp_flags);
            var flags =PyType.GetFlags(type);
            return (flags & TypeFlags.HasClrInstance) != 0;
        }

diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs
            PythonException.ThrowIfIsNull(variables);

            int res = Runtime.PyDict_SetItem(
                VarsRef, PyIdentifier.__builtins__,
                VarsRef,new BorrowedReference(PyIdentifier.__builtins__),
                Runtime.PyEval_GetBuiltins()
            );
            PythonException.ThrowIfIsNotZero(res);
diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs
            var locals = new PyDict();
            try
            {
                BorrowedReference module =Runtime.PyImport_AddModule("clr._extras");
                BorrowedReference module =DefineModule("clr._extras");
                BorrowedReference module_globals = Runtime.PyModule_GetDict(module);
                BorrowedReference builtins = Runtime.PyEval_GetBuiltins();
                Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins);

                Assembly assembly = Assembly.GetExecutingAssembly();
                // add the contents of clr.py to the module

                LoadSubmodule(module_globals, "clr.interop", "interop.py");

                    LoadMixins(module_globals);

                // add the imported module to the clr module, and copy the API functions
                // and decorators into the main clr module.
                Runtime.PyDict_SetItemString(clr_dict, "_extras", module);
            Runtime.PyDict_SetItemString(targetModuleDict, memberName, module);
        }

        static void LoadMixins(BorrowedReference targetModuleDict)
        {
            foreach (string nested in new[] { "collections" })
            {
                LoadSubmodule(targetModuleDict,
                    fullName: "clr._extras." + nested,
                    resourceName: typeof(PythonEngine).Namespace + ".Mixins." + nested + ".py");
            }
        }

        static void OnDomainUnload(object _, EventArgs __)
        {
            Shutdown();
                {
                    globals = tempGlobals = NewReference.DangerousFromPointer(Runtime.PyDict_New());
                    Runtime.PyDict_SetItem(
                        globals, PyIdentifier.__builtins__,
                        globals,new BorrowedReference(PyIdentifier.__builtins__),
                        Runtime.PyEval_GetBuiltins()
                    );
                }
diff --git a/src/runtime/pytype.cs b/src/runtime/pytype.cs
            return Exceptions.ErrorCheckIfNull(result);
        }

        internal static TypeFlags GetFlags(BorrowedReference type)
        {
            Debug.Assert(TypeOffset.tp_flags > 0);
            return (TypeFlags)Util.ReadCLong(type.DangerousGetAddress(), TypeOffset.tp_flags);
        }

        internal static BorrowedReference GetBase(BorrowedReference type)
        {
            Debug.Assert(IsType(type));
diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
        /// <summary>
        /// Return 0 on success or -1 on failure.
        /// </summary>
        [Obsolete]
        internal static int PyDict_SetItem(BorrowedReference dict, IntPtr key, BorrowedReference value) => Delegates.PyDict_SetItem(dict, new BorrowedReference(key), value);
        /// <summary>
        /// Return 0 on success or -1 on failure.
        internal static NewReference PyType_FromSpecWithBases(in NativeTypeSpec spec, BorrowedReference bases) => Delegates.PyType_FromSpecWithBases(in spec, bases);

        /// <summary>
        /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error.
        /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type�s base class. Return 0 on success, or return -1 and sets an exception on error.
        /// </summary>

        internal static int PyType_Ready(IntPtr type) => Delegates.PyType_Ready(type);
diff --git a/tests/test_collection_mixins.py b/tests/test_collection_mixins.py
 import System.Collections.Generic as C

 def test_contains():
    l = C.List[int]()
    l.Add(42)
    assert 42 in l
    assert 43 not in l

 def test_dict_items():
    d = C.Dictionary[int, str]()
    d[42] = "a"
    items = d.items()
    assert len(items) == 1
    k,v = items[0]
    assert k == 42
    assert v == "a"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -17,6 +17,8 @@ This document follows the conventions laid out in [Keep a CHANGELOG][].
		- Improved exception handling:
		- exceptions can now be converted with codecs
		- `InnerException` and `__cause__` are propagated properly
		- .NET collection types now implement standard Python collection interfaces from `collections.abc`.
		See [Mixins/collections.py](src/runtime/Mixins/collections.py).
		- .NET arrays implement Python buffer protocol


Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -3,6 +3,8 @@ namespace Python.Runtime
		using System;
		using System.Collections.Generic;

		using Python.Runtime.Mixins;

		public sealed class InteropConfiguration
		{
		internal readonly PythonBaseTypeProviderGroup pythonBaseTypeProviders
Expand All		@@ -18,6 +20,7 @@ public static InteropConfiguration MakeDefault()
		PythonBaseTypeProviders =
		{
		DefaultBaseTypeProvider.Instance,
		new CollectionMixinsProvider(new Lazy<PyObject>(() => Py.Import("clr._extras.collections"))),
		},
		};
		}
Expand Down
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,90 @@
		using System;
		using System.Collections.Generic;
		using System.Linq;

		namespace Python.Runtime.Mixins
		{
		class CollectionMixinsProvider : IPythonBaseTypeProvider
		{
		readonly Lazy<PyObject> mixinsModule;
		public CollectionMixinsProvider(Lazy<PyObject> mixinsModule)
		{
		this.mixinsModule = mixinsModule ?? throw new ArgumentNullException(nameof(mixinsModule));
		}

		public PyObject Mixins => this.mixinsModule.Value;

		public IEnumerable<PyType> GetBaseTypes(Type type, IList<PyType> existingBases)
		{
		if (type is null)
		throw new ArgumentNullException(nameof(type));

		if (existingBases is null)
		throw new ArgumentNullException(nameof(existingBases));

		var interfaces = NewInterfaces(type).Select(GetDefinition).ToArray();

		var newBases = new List<PyType>();
		newBases.AddRange(existingBases);

		// dictionaries
		if (interfaces.Contains(typeof(IDictionary<,>)))
		{
		newBases.Add(new PyType(this.Mixins.GetAttr("MutableMappingMixin")));
		}
		else if (interfaces.Contains(typeof(IReadOnlyDictionary<,>)))
		{
		newBases.Add(new PyType(this.Mixins.GetAttr("MappingMixin")));
		}

		// item collections
		if (interfaces.Contains(typeof(IList<>))
		\|\| interfaces.Contains(typeof(System.Collections.IList)))
		{
		newBases.Add(new PyType(this.Mixins.GetAttr("MutableSequenceMixin")));
		}
		else if (interfaces.Contains(typeof(IReadOnlyList<>)))
		{
		newBases.Add(new PyType(this.Mixins.GetAttr("SequenceMixin")));
		}
		else if (interfaces.Contains(typeof(ICollection<>))
		\|\| interfaces.Contains(typeof(System.Collections.ICollection)))
		{
		newBases.Add(new PyType(this.Mixins.GetAttr("CollectionMixin")));
		}
		else if (interfaces.Contains(typeof(System.Collections.IEnumerable)))
		{
		newBases.Add(new PyType(this.Mixins.GetAttr("IterableMixin")));
		}

		// enumerators
		if (interfaces.Contains(typeof(System.Collections.IEnumerator)))
		{
		newBases.Add(new PyType(this.Mixins.GetAttr("IteratorMixin")));
		}

		if (newBases.Count == existingBases.Count)
		{
		return existingBases;
		}

		if (type.IsInterface && type.BaseType is null)
		{
		newBases.RemoveAll(@base => @base.Handle == Runtime.PyBaseObjectType);
		}

		return newBases;
		}

		static Type[] NewInterfaces(Type type)
		{
		var result = type.GetInterfaces();
		return type.BaseType != null
		? result.Except(type.BaseType.GetInterfaces()).ToArray()
		: result;
		}

		static Type GetDefinition(Type type)
		=> type.IsGenericType ? type.GetGenericTypeDefinition() : type;
		}
		}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,82 @@
		"""
		Implements collections.abc for common .NET types
		https://docs.python.org/3.6/library/collections.abc.html
		"""

		import collections.abc as col

		class IteratorMixin(col.Iterator):
		def close(self):
		self.Dispose()

		class IterableMixin(col.Iterable):
		pass

		class SizedMixin(col.Sized):
		def __len__(self): return self.Count

		class ContainerMixin(col.Container):
		def __contains__(self, item): return self.Contains(item)

		try:
		abc_Collection = col.Collection
		except AttributeError:
		# Python 3.5- does not have collections.abc.Collection
		abc_Collection = col.Container

		class CollectionMixin(SizedMixin, IterableMixin, ContainerMixin, abc_Collection):
		pass

		class SequenceMixin(CollectionMixin, col.Sequence):
		pass

		class MutableSequenceMixin(SequenceMixin, col.MutableSequence):
		pass

		class MappingMixin(CollectionMixin, col.Mapping):
		def __contains__(self, item): return self.ContainsKey(item)
		def keys(self): return self.Keys
		def items(self): return [(k,self.get(k)) for k in self.Keys]
		def values(self): return self.Values
		def __iter__(self): return self.Keys.__iter__()
		def get(self, key, default=None):
		existed, item = self.TryGetValue(key, None)
		return item if existed else default

		class MutableMappingMixin(MappingMixin, col.MutableMapping):
		_UNSET_ = object()

		def __delitem__(self, key):
		self.Remove(key)

		def clear(self):
		self.Clear()

		def pop(self, key, default=_UNSET_):
		existed, item = self.TryGetValue(key, None)
		if existed:
		self.Remove(key)
		return item
		elif default == self._UNSET_:
		raise KeyError(key)
		else:
		return default

		def setdefault(self, key, value=None):
		existed, item = self.TryGetValue(key, None)
		if existed:
		return item
		else:
		self[key] = value
		return value

		def update(self, items, **kwargs):
		if isinstance(items, col.Mapping):
		for key, value in items.items():
		self[key] = value
		else:
		for key, value in items:
		self[key] = value

		for key, value in kwargs.items():
		self[key] = value
Original file line number	Diff line number	Diff line change
Expand Up		@@ -39,13 +39,13 @@
		</ItemGroup>

		<ItemGroup>
		<None Remove="resources\clr.py" />
		<EmbeddedResource Include="resources\clr.py">
		<LogicalName>clr.py</LogicalName>
		</EmbeddedResource>
		<EmbeddedResource Include="resources\interop.py">
		<LogicalName>interop.py</LogicalName>
		</EmbeddedResource>
		<EmbeddedResource Include="Mixins\*.py" />
		</ItemGroup>

		<ItemGroup>
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,6 +12,9 @@ internal static class Util
		internal const string MinimalPythonVersionRequired =
		"Only Python 3.5 or newer is supported";

		internal const string UseOverloadWithReferenceTypes =
		"This API is unsafe, and will be removed in the future. Use overloads working with *Reference types";

		internal static Int64 ReadCLong(IntPtr tp, int offset)
		{
		// On Windows, a C long is always 32 bits.
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -360,18 +360,40 @@ public static void tp_dealloc(IntPtr ob)

		public static int tp_clear(IntPtr ob)
		{
		ManagedType self = GetManagedObject(ob);
		if (GetManagedObject(ob) is { } self)
		{
		if (self.clearReentryGuard) return 0;

		// workaround for https://bugs.python.org/issue45266
		self.clearReentryGuard = true;

		try
		{
		return ClearImpl(ob, self);
		}
		finally
		{
		self.clearReentryGuard = false;
		}
		}
		else
		{
		return ClearImpl(ob, null);
		}
		}

		static int ClearImpl(IntPtr ob, ManagedType self)
		{
		bool isTypeObject = Runtime.PyObject_TYPE(ob) == Runtime.PyCLRMetaType;
		if (!isTypeObject)
		{
		ClearObjectDict(ob);

		int baseClearResult = BaseUnmanagedClear(ob);
		if (baseClearResult != 0)
		{
		return baseClearResult;
		}

		ClearObjectDict(ob);
		}
		return 0;
		}
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,7 +12,7 @@ internal class CLRObject : ManagedType

		internal CLRObject(object ob, IntPtr tp)
		{
		System.Diagnostics.Debug.Assert(tp != IntPtr.Zero);
		Debug.Assert(tp != IntPtr.Zero);
		IntPtr py = Runtime.PyType_GenericAlloc(tp, 0);

		tpHandle = tp;
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -28,6 +28,8 @@ internal enum TrackTypes
		internal IntPtr pyHandle; // PyObject *
		internal IntPtr tpHandle; // PyType *

		internal bool clearReentryGuard;

		internal BorrowedReference ObjectReference
		{
		get
Expand DownExpand Up		@@ -160,7 +162,7 @@ internal static bool IsInstanceOfManagedType(IntPtr ob)

		internal static bool IsManagedType(BorrowedReference type)
		{
		var flags =(TypeFlags)Util.ReadCLong(type.DangerousGetAddress(), TypeOffset.tp_flags);
		var flags =PyType.GetFlags(type);
		return (flags & TypeFlags.HasClrInstance) != 0;
		}

Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -66,7 +66,7 @@ private PyScope(IntPtr ptr, PyScopeManager manager) : base(ptr)
		PythonException.ThrowIfIsNull(variables);

		int res = Runtime.PyDict_SetItem(
		VarsRef, PyIdentifier.__builtins__,
		VarsRef,new BorrowedReference(PyIdentifier.__builtins__),
		Runtime.PyEval_GetBuiltins()
		);
		PythonException.ThrowIfIsNotZero(res);
Expand Down