Skip to content

Commit d0c588b

Browse files
amos402amos402
authored
Add intern string and PyIdentifier (#1254)
PyIdentifier will have pointers to common Python strings (e.g. `__class__`, `__doc__`, etc) to avoid allocation and conversion. `InternString.GetManagedString` provides cache for similar conversion from Python to .NET
1 parent 84e2735 commit d0c588b

18 files changed

+238
-34
lines changed

‎src/embed_tests/TestDomainReload.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -203,13 +203,13 @@ def test_obj_call():
203203
// Create a new module
204204
IntPtr module = PyRuntime.PyModule_New(name);
205205
Assert.That(module != IntPtr.Zero);
206-
IntPtr globals = PyRuntime.PyObject_GetAttrString(module, "__dict__");
206+
IntPtr globals = PyRuntime.PyObject_GetAttr(module, PyIdentifier.__dict__);
207207
Assert.That(globals != IntPtr.Zero);
208208
try
209209
{
210210
// import builtins
211211
// module.__dict__[__builtins__] = builtins
212-
int res = PyRuntime.PyDict_SetItemString(globals, "__builtins__",
212+
int res = PyRuntime.PyDict_SetItem(globals, PyIdentifier.__builtins__,
213213
PyRuntime.PyEval_GetBuiltins());
214214
PythonException.ThrowIfIsNotZero(res);
215215

‎src/runtime/Python.Runtime.15.csproj

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,22 @@
143143
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="All" />
144144
</ItemGroup>
145145

146+
<ItemGroup>
147+
<None Update="intern_.tt">
148+
<Generator>TextTemplatingFileGenerator</Generator>
149+
<LastGenOutput>intern_.cs</LastGenOutput>
150+
</None>
151+
</ItemGroup>
152+
153+
154+
<ItemGroup>
155+
<Compile Update="intern_.cs">
156+
<DesignTime>True</DesignTime>
157+
<AutoGen>True</AutoGen>
158+
<DependentUpon>intern_.tt</DependentUpon>
159+
</Compile>
160+
</ItemGroup>
161+
146162
<Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
147163

148164
<PropertyGroup>

‎src/runtime/Python.Runtime.csproj

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,14 +82,16 @@
8282
<Compile Include="Codecs\TupleCodecs.cs" />
8383
<Compile Include="converterextensions.cs" />
8484
<Compile Include="finalizer.cs" />
85+
<Compile Include="intern.cs" />
86+
<Compile Include="intern_.cs" />
8587
<Compile Include="Properties\AssemblyInfo.cs" />
8688
<Compile Include="..\SharedAssemblyInfo.cs">
8789
<Link>Properties\SharedAssemblyInfo.cs</Link>
8890
</Compile>
8991
<Compile Include="arrayobject.cs" />
9092
<Compile Include="assemblymanager.cs" />
9193
<Compile Include="BorrowedReference.cs" />
92-
<Compile Include="bufferinterface.cs" />
94+
<Compile Include="bufferinterface.cs" />
9395
<Compile Include="classderived.cs" />
9496
<Compile Include="classbase.cs" />
9597
<Compile Include="classmanager.cs" />
@@ -131,7 +133,7 @@
131133
<Compile Include="overload.cs" />
132134
<Compile Include="propertyobject.cs" />
133135
<Compile Include="pyansistring.cs" />
134-
<Compile Include="pybuffer.cs" />
136+
<Compile Include="pybuffer.cs" />
135137
<Compile Include="pydict.cs" />
136138
<Compile Include="PyExportAttribute.cs" />
137139
<Compile Include="pyfloat.cs" />
@@ -185,4 +187,4 @@
185187
<Copy SourceFiles="$(TargetAssembly)" DestinationFolder="$(PythonBuildDir)" />
186188
<!--Copy SourceFiles="$(TargetAssemblyPdb)" Condition="Exists('$(TargetAssemblyPdb)')" DestinationFolder="$(PythonBuildDir)" /-->
187189
</Target>
188-
</Project>
190+
</Project>

‎src/runtime/classbase.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,7 @@ public static IntPtr tp_repr(IntPtr ob)
279279
IntPtr args = Runtime.PyTuple_New(1);
280280
Runtime.XIncref(ob);
281281
Runtime.PyTuple_SetItem(args, 0, ob);
282-
IntPtr reprFunc = Runtime.PyObject_GetAttrString(Runtime.PyBaseObjectType, "__repr__");
282+
IntPtr reprFunc = Runtime.PyObject_GetAttr(Runtime.PyBaseObjectType, PyIdentifier.__repr__);
283283
var output = Runtime.PyObject_Call(reprFunc, args, IntPtr.Zero);
284284
Runtime.XDecref(args);
285285
Runtime.XDecref(reprFunc);

‎src/runtime/classmanager.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ private static void InitClassBase(Type type, ClassBase impl)
232232
var attr = (DocStringAttribute)attrs[0];
233233
string docStr = attr.DocString;
234234
doc = Runtime.PyString_FromString(docStr);
235-
Runtime.PyDict_SetItemString(dict, "__doc__", doc);
235+
Runtime.PyDict_SetItem(dict, PyIdentifier.__doc__, doc);
236236
Runtime.XDecref(doc);
237237
}
238238

@@ -249,16 +249,16 @@ private static void InitClassBase(Type type, ClassBase impl)
249249
var ctors = new ConstructorBinding(type, tp, co.binder);
250250
// ExtensionType types are untracked, so don't Incref() them.
251251
// TODO: deprecate __overloads__ soon...
252-
Runtime.PyDict_SetItemString(dict, "__overloads__", ctors.pyHandle);
253-
Runtime.PyDict_SetItemString(dict, "Overloads", ctors.pyHandle);
252+
Runtime.PyDict_SetItem(dict, PyIdentifier.__overloads__, ctors.pyHandle);
253+
Runtime.PyDict_SetItem(dict, PyIdentifier.Overloads, ctors.pyHandle);
254254
ctors.DecrRefCount();
255255
}
256256

257257
// don't generate the docstring if one was already set from a DocStringAttribute.
258258
if (!CLRModule._SuppressDocs && doc == IntPtr.Zero)
259259
{
260260
doc = co.GetDocString();
261-
Runtime.PyDict_SetItemString(dict, "__doc__", doc);
261+
Runtime.PyDict_SetItem(dict, PyIdentifier.__doc__, doc);
262262
Runtime.XDecref(doc);
263263
}
264264
}

‎src/runtime/exceptions.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,7 @@ public static void SetError(Exception e)
284284
}
285285

286286
IntPtr op = CLRObject.GetInstHandle(e);
287-
IntPtr etype = Runtime.PyObject_GetAttrString(op, "__class__");
287+
IntPtr etype = Runtime.PyObject_GetAttr(op, PyIdentifier.__class__);
288288
Runtime.PyErr_SetObject(new BorrowedReference(etype), new BorrowedReference(op));
289289
Runtime.XDecref(etype);
290290
Runtime.XDecref(op);

‎src/runtime/importhook.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,11 @@ static void InitImport()
4343
// look in CLR modules, then if we don't find any call the default
4444
// Python __import__.
4545
IntPtr builtins = Runtime.GetBuiltins();
46-
py_import = Runtime.PyObject_GetAttrString(builtins, "__import__");
46+
py_import = Runtime.PyObject_GetAttr(builtins, PyIdentifier.__import__);
4747
PythonException.ThrowIfIsNull(py_import);
4848

4949
hook = new MethodWrapper(typeof(ImportHook), "__import__", "TernaryFunc");
50-
int res = Runtime.PyObject_SetAttrString(builtins, "__import__", hook.ptr);
50+
int res = Runtime.PyObject_SetAttr(builtins, PyIdentifier.__import__, hook.ptr);
5151
PythonException.ThrowIfIsNotZero(res);
5252

5353
Runtime.XDecref(builtins);
@@ -60,7 +60,7 @@ static void RestoreImport()
6060
{
6161
IntPtr builtins = Runtime.GetBuiltins();
6262

63-
int res = Runtime.PyObject_SetAttrString(builtins, "__import__", py_import);
63+
int res = Runtime.PyObject_SetAttr(builtins, PyIdentifier.__import__, py_import);
6464
PythonException.ThrowIfIsNotZero(res);
6565
Runtime.XDecref(py_import);
6666
py_import = IntPtr.Zero;

‎src/runtime/intern.cs

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
5+
namespace Python.Runtime
6+
{
7+
static partial class InternString
8+
{
9+
private static Dictionary<string, IntPtr> _string2interns;
10+
private static Dictionary<IntPtr, string> _intern2strings;
11+
12+
static InternString()
13+
{
14+
var identifierNames = typeof(PyIdentifier).GetFields().Select(fi => fi.Name);
15+
var validNames = new HashSet<string>(identifierNames);
16+
if (validNames.Count != _builtinNames.Length)
17+
{
18+
throw new InvalidOperationException("Identifiers args not matching");
19+
}
20+
foreach (var name in _builtinNames)
21+
{
22+
if (!validNames.Contains(name))
23+
{
24+
throw new InvalidOperationException($"{name} is not declared");
25+
}
26+
}
27+
}
28+
29+
public static void Initialize()
30+
{
31+
_string2interns = new Dictionary<string, IntPtr>();
32+
_intern2strings = new Dictionary<IntPtr, string>();
33+
34+
Type type = typeof(PyIdentifier);
35+
foreach (string name in _builtinNames)
36+
{
37+
IntPtr op = Runtime.PyUnicode_InternFromString(name);
38+
SetIntern(name, op);
39+
type.GetField(name).SetValue(null, op);
40+
}
41+
}
42+
43+
public static void Shutdown()
44+
{
45+
foreach (var ptr in _intern2strings.Keys)
46+
{
47+
Runtime.XDecref(ptr);
48+
}
49+
_string2interns = null;
50+
_intern2strings = null;
51+
}
52+
53+
public static string GetManagedString(IntPtr op)
54+
{
55+
string s;
56+
if (TryGetInterned(op, out s))
57+
{
58+
return s;
59+
}
60+
return Runtime.GetManagedString(op);
61+
}
62+
63+
public static bool TryGetInterned(IntPtr op, out string s)
64+
{
65+
return _intern2strings.TryGetValue(op, out s);
66+
}
67+
68+
private static void SetIntern(string s, IntPtr op)
69+
{
70+
_string2interns.Add(s, op);
71+
_intern2strings.Add(op, s);
72+
}
73+
}
74+
}

‎src/runtime/intern_.cs

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
using System;
2+
3+
namespace Python.Runtime
4+
{
5+
static class PyIdentifier
6+
{
7+
public static IntPtr __name__;
8+
public static IntPtr __dict__;
9+
public static IntPtr __doc__;
10+
public static IntPtr __class__;
11+
public static IntPtr __module__;
12+
public static IntPtr __file__;
13+
public static IntPtr __slots__;
14+
public static IntPtr __self__;
15+
public static IntPtr __annotations__;
16+
public static IntPtr __init__;
17+
public static IntPtr __repr__;
18+
public static IntPtr __import__;
19+
public static IntPtr __builtins__;
20+
public static IntPtr builtins;
21+
public static IntPtr __overloads__;
22+
public static IntPtr Overloads;
23+
}
24+
25+
26+
static partial class InternString
27+
{
28+
private static readonly string[] _builtinNames = new string[]
29+
{
30+
"__name__",
31+
"__dict__",
32+
"__doc__",
33+
"__class__",
34+
"__module__",
35+
"__file__",
36+
"__slots__",
37+
"__self__",
38+
"__annotations__",
39+
"__init__",
40+
"__repr__",
41+
"__import__",
42+
"__builtins__",
43+
"builtins",
44+
"__overloads__",
45+
"Overloads",
46+
};
47+
}
48+
}

‎src/runtime/intern_.tt

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
<#@ template debug="true" hostSpecific="true" #>
2+
<#@ output extension=".cs" #>
3+
<#
4+
string[] internNames = new string[]
5+
{
6+
"__name__",
7+
"__dict__",
8+
"__doc__",
9+
"__class__",
10+
"__module__",
11+
"__file__",
12+
"__slots__",
13+
"__self__",
14+
"__annotations__",
15+
16+
"__init__",
17+
"__repr__",
18+
"__import__",
19+
"__builtins__",
20+
21+
"builtins",
22+
23+
"__overloads__",
24+
"Overloads",
25+
};
26+
#>
27+
using System;
28+
29+
namespace Python.Runtime
30+
{
31+
static class PyIdentifier
32+
{
33+
<#
34+
foreach (var name in internNames)
35+
{
36+
#>
37+
public static IntPtr <#= name #>;
38+
<#
39+
}
40+
#>
41+
}
42+
43+
44+
static partial class InternString
45+
{
46+
private static readonly string[] _builtinNames = new string[]
47+
{
48+
<#
49+
foreach (var name in internNames)
50+
{
51+
#>
52+
"<#= name #>",
53+
<#
54+
}
55+
#>
56+
};
57+
}
58+
}

‎src/runtime/metatype.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw)
109109
}
110110
}
111111

112-
IntPtr slots = Runtime.PyDict_GetItemString(dict, "__slots__");
112+
IntPtr slots = Runtime.PyDict_GetItem(dict, PyIdentifier.__slots__);
113113
if (slots != IntPtr.Zero)
114114
{
115115
return Exceptions.RaiseTypeError("subclasses of managed classes do not support __slots__");
@@ -197,7 +197,7 @@ public static IntPtr tp_call(IntPtr tp, IntPtr args, IntPtr kw)
197197
return IntPtr.Zero;
198198
}
199199

200-
var init = Runtime.PyObject_GetAttrString(obj, "__init__");
200+
var init = Runtime.PyObject_GetAttr(obj, PyIdentifier.__init__);
201201
Runtime.PyErr_Clear();
202202

203203
if (init != IntPtr.Zero)

‎src/runtime/methodbinding.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key)
8484
return IntPtr.Zero;
8585
}
8686

87-
string name = Runtime.GetManagedString(key);
87+
string name = InternString.GetManagedString(key);
8888
switch (name)
8989
{
9090
case "__doc__":

‎src/runtime/methodobject.cs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,8 +133,7 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key)
133133
return Exceptions.RaiseTypeError("string expected");
134134
}
135135

136-
string name = Runtime.GetManagedString(key);
137-
if (name == "__doc__")
136+
if (Runtime.PyUnicode_Compare(key, PyIdentifier.__doc__) == 0)
138137
{
139138
IntPtr doc = self.GetDocString();
140139
Runtime.XIncref(doc);

‎src/runtime/moduleobject.cs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,10 +48,10 @@ public ModuleObject(string name)
4848
IntPtr pyfilename = Runtime.PyString_FromString(filename);
4949
IntPtr pydocstring = Runtime.PyString_FromString(docstring);
5050
IntPtr pycls = TypeManager.GetTypeHandle(GetType());
51-
Runtime.PyDict_SetItemString(dict, "__name__", pyname);
52-
Runtime.PyDict_SetItemString(dict, "__file__", pyfilename);
53-
Runtime.PyDict_SetItemString(dict, "__doc__", pydocstring);
54-
Runtime.PyDict_SetItemString(dict, "__class__", pycls);
51+
Runtime.PyDict_SetItem(dict, PyIdentifier.__name__, pyname);
52+
Runtime.PyDict_SetItem(dict, PyIdentifier.__file__, pyfilename);
53+
Runtime.PyDict_SetItem(dict, PyIdentifier.__doc__, pydocstring);
54+
Runtime.PyDict_SetItem(dict, PyIdentifier.__class__, pycls);
5555
Runtime.XDecref(pyname);
5656
Runtime.XDecref(pyfilename);
5757
Runtime.XDecref(pydocstring);
@@ -282,7 +282,7 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key)
282282
return op;
283283
}
284284

285-
string name = Runtime.GetManagedString(key);
285+
string name = InternString.GetManagedString(key);
286286
if (name == "__dict__")
287287
{
288288
Runtime.XIncref(self.dict);

‎src/runtime/pyscope.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,8 @@ internal PyScope(IntPtr ptr, PyScopeManager manager)
6868
variables = Runtime.PyModule_GetDict(obj);
6969
PythonException.ThrowIfIsNull(variables);
7070

71-
int res = Runtime.PyDict_SetItemString(
72-
variables, "__builtins__",
71+
int res = Runtime.PyDict_SetItem(
72+
variables, PyIdentifier.__builtins__,
7373
Runtime.PyEval_GetBuiltins()
7474
);
7575
PythonException.ThrowIfIsNotZero(res);

0 commit comments

Comments
 (0)