Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitf82aeea

Browse files
authored
Simplify UTF8 StrPtr usage (#2374)
* Use non-BOM encodings* Copy potential BOM to the output of PyString_FromStringThe documentation of the used `PyUnicode_DecodeUTF16` states that notpassing `*byteorder` or passing a 0 results in the first two bytes, ifthey are the BOM (U+FEFF, zero-width no-break space), to be interpretedand skipped, which is incorrect when we convert a known "non BOM"string, which all strings from C# are.* Default to UTF8 for StrPtr
1 parentb112885 commitf82aeea

File tree

4 files changed

+27
-23
lines changed

4 files changed

+27
-23
lines changed

‎src/embed_tests/TestPyType.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ public void CanCreateHeapType()
2828
conststringname="nÁmæ";
2929
conststringdocStr="dÁcæ";
3030

31-
usingvardoc=newStrPtr(docStr,Encodings.UTF8);
31+
usingvardoc=newStrPtr(docStr);
32+
3233
varspec=newTypeSpec(
3334
name:name,
3435
basicSize:Util.ReadInt32(Runtime.Runtime.PyBaseObjectType,TypeOffset.tp_basicsize),

‎src/runtime/Native/NativeTypeSpec.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ public NativeTypeSpec(TypeSpec spec)
1717
{
1818
if(specisnull)thrownewArgumentNullException(nameof(spec));
1919

20-
this.Name=newStrPtr(spec.Name,Encodings.UTF8);
20+
this.Name=newStrPtr(spec.Name);
2121
this.BasicSize=spec.BasicSize;
2222
this.ItemSize=spec.ItemSize;
2323
this.Flags=(int)spec.Flags;

‎src/runtime/Native/StrPtr.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ struct StrPtr : IDisposable
1010
publicIntPtrRawPointer{get;set;}
1111
unsafebyte*Bytes=>(byte*)this.RawPointer;
1212

13+
publicunsafeStrPtr(stringvalue):this(value,Encodings.UTF8){}
14+
1315
publicunsafeStrPtr(stringvalue,Encodingencoding)
1416
{
1517
if(valueisnull)thrownewArgumentNullException(nameof(value));

‎src/runtime/Runtime.cs

Lines changed: 22 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -795,13 +795,13 @@ public static int Py_Main(int argc, string[] argv)
795795

796796
internalstaticintPyRun_SimpleString(stringcode)
797797
{
798-
usingvarcodePtr=newStrPtr(code,Encodings.UTF8);
798+
usingvarcodePtr=newStrPtr(code);
799799
returnDelegates.PyRun_SimpleStringFlags(codePtr,Utf8String);
800800
}
801801

802802
internalstaticNewReferencePyRun_String(stringcode,RunFlagTypest,BorrowedReferenceglobals,BorrowedReferencelocals)
803803
{
804-
usingvarcodePtr=newStrPtr(code,Encodings.UTF8);
804+
usingvarcodePtr=newStrPtr(code);
805805
returnDelegates.PyRun_StringFlags(codePtr,st,globals,locals,Utf8String);
806806
}
807807

@@ -813,14 +813,15 @@ internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedR
813813
/// </summary>
814814
internalstaticNewReferencePy_CompileString(stringstr,stringfile,intstart)
815815
{
816-
usingvarstrPtr=newStrPtr(str,Encodings.UTF8);
816+
usingvarstrPtr=newStrPtr(str);
817+
817818
usingvarfileObj=newPyString(file);
818819
returnDelegates.Py_CompileStringObject(strPtr,fileObj,start,Utf8String,-1);
819820
}
820821

821822
internalstaticNewReferencePyImport_ExecCodeModule(stringname,BorrowedReferencecode)
822823
{
823-
usingvarnamePtr=newStrPtr(name,Encodings.UTF8);
824+
usingvarnamePtr=newStrPtr(name);
824825
returnDelegates.PyImport_ExecCodeModule(namePtr,code);
825826
}
826827

@@ -867,13 +868,13 @@ internal static bool PyObject_IsIterable(BorrowedReference ob)
867868

868869
internalstaticintPyObject_HasAttrString(BorrowedReferencepointer,stringname)
869870
{
870-
usingvarnamePtr=newStrPtr(name,Encodings.UTF8);
871+
usingvarnamePtr=newStrPtr(name);
871872
returnDelegates.PyObject_HasAttrString(pointer,namePtr);
872873
}
873874

874875
internalstaticNewReferencePyObject_GetAttrString(BorrowedReferencepointer,stringname)
875876
{
876-
usingvarnamePtr=newStrPtr(name,Encodings.UTF8);
877+
usingvarnamePtr=newStrPtr(name);
877878
returnDelegates.PyObject_GetAttrString(pointer,namePtr);
878879
}
879880

@@ -884,12 +885,12 @@ internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, S
884885
internalstaticintPyObject_DelAttr(BorrowedReference@object,BorrowedReferencename)=>Delegates.PyObject_SetAttr(@object,name,null);
885886
internalstaticintPyObject_DelAttrString(BorrowedReference@object,stringname)
886887
{
887-
usingvarnamePtr=newStrPtr(name,Encodings.UTF8);
888+
usingvarnamePtr=newStrPtr(name);
888889
returnDelegates.PyObject_SetAttrString(@object,namePtr,null);
889890
}
890891
internalstaticintPyObject_SetAttrString(BorrowedReference@object,stringname,BorrowedReferencevalue)
891892
{
892-
usingvarnamePtr=newStrPtr(name,Encodings.UTF8);
893+
usingvarnamePtr=newStrPtr(name);
893894
returnDelegates.PyObject_SetAttrString(@object,namePtr,value);
894895
}
895896

@@ -1071,7 +1072,7 @@ internal static bool PyBool_CheckExact(BorrowedReference ob)
10711072

10721073
internalstaticNewReferencePyLong_FromString(stringvalue,intradix)
10731074
{
1074-
usingvarvalPtr=newStrPtr(value,Encodings.UTF8);
1075+
usingvarvalPtr=newStrPtr(value);
10751076
returnDelegates.PyLong_FromString(valPtr,IntPtr.Zero,radix);
10761077
}
10771078

@@ -1274,7 +1275,7 @@ internal static NewReference EmptyPyBytes()
12741275
internalstaticNewReferencePyByteArray_FromStringAndSize(IntPtrstrPtr,nintlen)=>Delegates.PyByteArray_FromStringAndSize(strPtr,len);
12751276
internalstaticNewReferencePyByteArray_FromStringAndSize(strings)
12761277
{
1277-
usingvarptr=newStrPtr(s,Encodings.UTF8);
1278+
usingvarptr=newStrPtr(s);
12781279
returnPyByteArray_FromStringAndSize(ptr.RawPointer,checked((nint)ptr.ByteCount));
12791280
}
12801281

@@ -1302,7 +1303,7 @@ internal static IntPtr PyBytes_AsString(BorrowedReference ob)
13021303

13031304
internalstaticNewReferencePyUnicode_InternFromString(strings)
13041305
{
1305-
usingvarptr=newStrPtr(s,Encodings.UTF8);
1306+
usingvarptr=newStrPtr(s);
13061307
returnDelegates.PyUnicode_InternFromString(ptr);
13071308
}
13081309

@@ -1377,7 +1378,7 @@ internal static bool PyDict_Check(BorrowedReference ob)
13771378

13781379
internalstaticBorrowedReferencePyDict_GetItemString(BorrowedReferencepointer,stringkey)
13791380
{
1380-
usingvarkeyStr=newStrPtr(key,Encodings.UTF8);
1381+
usingvarkeyStr=newStrPtr(key);
13811382
returnDelegates.PyDict_GetItemString(pointer,keyStr);
13821383
}
13831384

@@ -1393,7 +1394,7 @@ internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer
13931394
/// </summary>
13941395
internalstaticintPyDict_SetItemString(BorrowedReferencedict,stringkey,BorrowedReferencevalue)
13951396
{
1396-
usingvarkeyPtr=newStrPtr(key,Encodings.UTF8);
1397+
usingvarkeyPtr=newStrPtr(key);
13971398
returnDelegates.PyDict_SetItemString(dict,keyPtr,value);
13981399
}
13991400

@@ -1402,7 +1403,7 @@ internal static int PyDict_SetItemString(BorrowedReference dict, string key, Bor
14021403

14031404
internalstaticintPyDict_DelItemString(BorrowedReferencepointer,stringkey)
14041405
{
1405-
usingvarkeyPtr=newStrPtr(key,Encodings.UTF8);
1406+
usingvarkeyPtr=newStrPtr(key);
14061407
returnDelegates.PyDict_DelItemString(pointer,keyPtr);
14071408
}
14081409

@@ -1517,7 +1518,7 @@ internal static bool PyIter_Check(BorrowedReference ob)
15171518

15181519
internalstaticNewReferencePyModule_New(stringname)
15191520
{
1520-
usingvarnamePtr=newStrPtr(name,Encodings.UTF8);
1521+
usingvarnamePtr=newStrPtr(name);
15211522
returnDelegates.PyModule_New(namePtr);
15221523
}
15231524

@@ -1531,7 +1532,7 @@ internal static NewReference PyModule_New(string name)
15311532
/// <returns>Return -1 on error, 0 on success.</returns>
15321533
internalstaticintPyModule_AddObject(BorrowedReferencemodule,stringname,StolenReferencevalue)
15331534
{
1534-
usingvarnamePtr=newStrPtr(name,Encodings.UTF8);
1535+
usingvarnamePtr=newStrPtr(name);
15351536
IntPtrvalueAddr=value.DangerousGetAddressOrNull();
15361537
intres=Delegates.PyModule_AddObject(module,namePtr,valueAddr);
15371538
// We can't just exit here because the reference is stolen only on success.
@@ -1549,7 +1550,7 @@ internal static int PyModule_AddObject(BorrowedReference module, string name, St
15491550

15501551
internalstaticNewReferencePyImport_ImportModule(stringname)
15511552
{
1552-
usingvarnamePtr=newStrPtr(name,Encodings.UTF8);
1553+
usingvarnamePtr=newStrPtr(name);
15531554
returnDelegates.PyImport_ImportModule(namePtr);
15541555
}
15551556

@@ -1558,7 +1559,7 @@ internal static NewReference PyImport_ImportModule(string name)
15581559

15591560
internalstaticBorrowedReferencePyImport_AddModule(stringname)
15601561
{
1561-
usingvarnamePtr=newStrPtr(name,Encodings.UTF8);
1562+
usingvarnamePtr=newStrPtr(name);
15621563
returnDelegates.PyImport_AddModule(namePtr);
15631564
}
15641565

@@ -1586,13 +1587,13 @@ internal static void PySys_SetArgvEx(int argc, string[] argv, int updatepath)
15861587

15871588
internalstaticBorrowedReferencePySys_GetObject(stringname)
15881589
{
1589-
usingvarnamePtr=newStrPtr(name,Encodings.UTF8);
1590+
usingvarnamePtr=newStrPtr(name);
15901591
returnDelegates.PySys_GetObject(namePtr);
15911592
}
15921593

15931594
internalstaticintPySys_SetObject(stringname,BorrowedReferenceob)
15941595
{
1595-
usingvarnamePtr=newStrPtr(name,Encodings.UTF8);
1596+
usingvarnamePtr=newStrPtr(name);
15961597
returnDelegates.PySys_SetObject(namePtr,ob);
15971598
}
15981599

@@ -1691,7 +1692,7 @@ internal static IntPtr PyMem_Malloc(long size)
16911692

16921693
internalstaticvoidPyErr_SetString(BorrowedReferenceob,stringmessage)
16931694
{
1694-
usingvarmsgPtr=newStrPtr(message,Encodings.UTF8);
1695+
usingvarmsgPtr=newStrPtr(message);
16951696
Delegates.PyErr_SetString(ob,msgPtr);
16961697
}
16971698

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp