Skip to content

Detect the size of wchar_t (aka Runtime.UCS) at runtime using PyUnicode_GetMax #1298

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 30, 2020
Merged

Detect the size of wchar_t (aka Runtime.UCS) at runtime using PyUnicode_GetMax #1298

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .editorconfig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ csharp_new_line_before_finally = true
[*.sln]
indent_style = tab

[*.csproj]
charset = utf-8
insert_final_newline = true

# bumpversion reformats itself after every bump
[.bumpversion.cfg]
trim_trailing_whitespace = false
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][].

### Changed
- Drop support for Python 2, 3.4, and 3.5
- `wchar_t` size aka `Runtime.UCS` is now determined at runtime
- `clr.AddReference` may now throw errors besides `FileNotFoundException`, that provide more
details about the cause of the failure
- `clr.AddReference` no longer adds ".dll" implicitly
Expand Down
6 changes: 0 additions & 6 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,14 +251,8 @@ def build_extension(self, ext):
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)

# Up to Python 3.2 sys.maxunicode is used to determine the size of
# Py_UNICODE, but from 3.3 onwards Py_UNICODE is a typedef of wchar_t.
import ctypes
unicode_width = ctypes.sizeof(ctypes.c_wchar)

defines = [
"PYTHON{0}{1}".format(PY_MAJOR, PY_MINOR),
"UCS{0}".format(unicode_width),
]

if CONFIG == "Debug":
Expand Down
25 changes: 12 additions & 13 deletions src/runtime/Python.Runtime.15.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,8 @@
<Python3Version>$(PYTHONNET_PY3_VERSION)</Python3Version>
<Python3Version Condition="'$(Python3Version)'==''">PYTHON38</Python3Version>
<PythonWinDefineConstants>$(PYTHONNET_WIN_DEFINE_CONSTANTS)</PythonWinDefineConstants>
<PythonWinDefineConstants Condition="'$(PythonWinDefineConstants)'==''">UCS2</PythonWinDefineConstants>
<PythonMonoDefineConstants>$(PYTHONNET_MONO_DEFINE_CONSTANTS)</PythonMonoDefineConstants>
<PythonMonoDefineConstants Condition="'$(PythonMonoDefineConstants)'==''">UCS4;MONO_LINUX;PYTHON_WITH_PYMALLOC</PythonMonoDefineConstants>
<PythonMonoDefineConstants Condition="'$(PythonMonoDefineConstants)'==''">MONO_LINUX;PYTHON_WITH_PYMALLOC</PythonMonoDefineConstants>
<PythonInteropFile Condition="'$(PythonInteropFile)'==''">$(PYTHONNET_INTEROP_FILE)</PythonInteropFile>
</PropertyGroup>
<PropertyGroup Condition="$(Configuration.Contains('Debug')) AND '$(TargetFramework)'=='net40'">
Expand Down Expand Up @@ -143,20 +142,20 @@
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="All" />
</ItemGroup>

<ItemGroup>
<None Update="intern_.tt">
<Generator>TextTemplatingFileGenerator</Generator>
<LastGenOutput>intern_.cs</LastGenOutput>
</None>
<ItemGroup>
<None Update="intern_.tt">
<Generator>TextTemplatingFileGenerator</Generator>
<LastGenOutput>intern_.cs</LastGenOutput>
</None>
</ItemGroup>


<ItemGroup>
<Compile Update="intern_.cs">
<DesignTime>True</DesignTime>
<AutoGen>True</AutoGen>
<DependentUpon>intern_.tt</DependentUpon>
</Compile>
<ItemGroup>
<Compile Update="intern_.cs">
<DesignTime>True</DesignTime>
<AutoGen>True</AutoGen>
<DependentUpon>intern_.tt</DependentUpon>
</Compile>
</ItemGroup>

<Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
Expand Down
16 changes: 8 additions & 8 deletions src/runtime/Python.Runtime.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,46 +29,46 @@
<PlatformTarget>x64</PlatformTarget>
</PropertyGroup>-->
<PropertyGroup Condition=" '$(Configuration)' == 'ReleaseMono'">
<DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON2;PYTHON27;UCS4</DefineConstants>
<DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON2;PYTHON27</DefineConstants>
<Optimize>true</Optimize>
<DebugType>pdbonly</DebugType>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)' == 'ReleaseMonoPY3'">
<DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON3;PYTHON38;UCS4</DefineConstants>
<DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON3;PYTHON38</DefineConstants>
<Optimize>true</Optimize>
<DebugType>pdbonly</DebugType>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)' == 'DebugMono'">
<DebugSymbols>true</DebugSymbols>
<DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON2;PYTHON27;UCS4;TRACE;DEBUG</DefineConstants>
<DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON2;PYTHON27;TRACE;DEBUG</DefineConstants>
<Optimize>false</Optimize>
<DebugType>full</DebugType>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)' == 'DebugMonoPY3'">
<DebugSymbols>true</DebugSymbols>
<DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON3;PYTHON38;UCS4;TRACE;DEBUG</DefineConstants>
<DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON3;PYTHON38;TRACE;DEBUG</DefineConstants>
<Optimize>false</Optimize>
<DebugType>full</DebugType>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)' == 'ReleaseWin'">
<DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON2;PYTHON27;UCS2</DefineConstants>
<DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON2;PYTHON27</DefineConstants>
<Optimize>true</Optimize>
<DebugType>pdbonly</DebugType>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)' == 'ReleaseWinPY3'">
<DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON3;PYTHON38;UCS2</DefineConstants>
<DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON3;PYTHON38</DefineConstants>
<Optimize>true</Optimize>
<DebugType>pdbonly</DebugType>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)' == 'DebugWin'">
<DebugSymbols>true</DebugSymbols>
<DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON2;PYTHON27;UCS2;TRACE;DEBUG</DefineConstants>
<DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON2;PYTHON27;TRACE;DEBUG</DefineConstants>
<Optimize>false</Optimize>
<DebugType>full</DebugType>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)' == 'DebugWinPY3'">
<DebugSymbols>true</DebugSymbols>
<DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON3;PYTHON38;UCS2;TRACE;DEBUG</DefineConstants>
<DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON3;PYTHON38;TRACE;DEBUG</DefineConstants>
<Optimize>false</Optimize>
<DebugType>full</DebugType>
</PropertyGroup>
Expand Down
39 changes: 4 additions & 35 deletions src/runtime/runtime.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,31 +18,8 @@ namespace Python.Runtime
/// </summary>
public class Runtime
{
// C# compiler copies constants to the assemblies that references this library.
// We needs to replace all public constants to static readonly fields to allow
// binary substitution of different Python.Runtime.dll builds in a target application.

public static int UCS => _UCS;

#if UCS4
internal const int _UCS = 4;

/// <summary>
/// EntryPoint to be used in DllImport to map to correct Unicode
/// methods prior to PEP393. Only used for PY27.
/// </summary>
private const string PyUnicodeEntryPoint = "PyUnicodeUCS4_";
#elif UCS2
internal const int _UCS = 2;

/// <summary>
/// EntryPoint to be used in DllImport to map to correct Unicode
/// methods prior to PEP393. Only used for PY27.
/// </summary>
private const string PyUnicodeEntryPoint = "PyUnicodeUCS2_";
#else
#error You must define either UCS2 or UCS4!
#endif
internal static readonly int _UCS = PyUnicode_GetMax() <= 0xFFFF ? 2 : 4;

#if PYTHON36
const string _minor = "6";
Expand Down Expand Up @@ -1537,17 +1514,6 @@ internal static IntPtr PyBytes_AS_STRING(IntPtr ob)
return ob + BytesOffset.ob_sval;
}

internal static IntPtr PyString_FromStringAndSize(string value, long size)
{
return _PyString_FromStringAndSize(value, new IntPtr(size));
}

[DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl,
EntryPoint = "PyUnicode_FromStringAndSize")]
internal static extern IntPtr _PyString_FromStringAndSize(
[MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string value,
IntPtr size
);

internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size)
{
Expand Down Expand Up @@ -1588,6 +1554,9 @@ internal static IntPtr PyUnicode_FromUnicode(string s, long size)
return PyUnicode_FromKindAndData(_UCS, s, size);
}

[DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)]
internal static extern int PyUnicode_GetMax();

internal static long PyUnicode_GetSize(IntPtr ob)
{
return (long)_PyUnicode_GetSize(ob);
Expand Down