WIP: loading TCL / Tk symbols dynamically

This is an attempt to load the symbols we need from the Tkinter.tkinter module at run time, rather than by linking at build time. It is one way of building a manylinux wheel that can build on a basic Manylinux docker image, and then run with the TCL / Tk library installed on the installing user's machine. It would also make the situation better on OSX, where we have to build against ActiveState TCL for compatibility with Python.org Python, but we would like to allow run-time TCL from e.g. homebrew. I have tested this on Debian Jessie Python 2.7 and 3.5, and on OSX 10.9 with Python 2.7. Questions: * Would y'all consider carrying something like this approach in the matplotlib source, but not enabled by default, to help building binary wheels? * Do you have any better suggestions about how to do this? * My C fu is weak; is there a way of collecting the typedefs I need from the TCL / Tk headers rather than copying them into the _tkagg.cpp source file (typdefs starting around line 52)? * My fu for Python C extension modules is also weak; did I configure exceptions and handle references correctly?
matplotlib · efiring · May 24, 2016 · May 17, 2016 · May 17, 2016 · May 18, 2016
commit edc80a38e76e6cf61a49a4311b0c5c3101835a44
diff --git a/src/_tkagg.cpp b/src/_tkagg.cpp
@@ -44,6 +44,38 @@ typedef struct
     Tcl_Interp *interp;
 } TkappObject;
 
+#define DYNAMIC_TKINTER
+
+#ifdef DYNAMIC_TKINTER
+// Load TCL / Tk symbols from tkinter extension module at run-time.
+// Typedefs, global vars for TCL / Tk library functions.
+typedef Tcl_Command (*tcl_cc)(Tcl_Interp *interp,
+        const char *cmdName, Tcl_CmdProc *proc,
+        ClientData clientData,
+        Tcl_CmdDeleteProc *deleteProc);
+static tcl_cc TCL_CREATE_COMMAND;
+typedef void (*tcl_app_res) (Tcl_Interp *interp, ...);
+static tcl_app_res TCL_APPEND_RESULT;
+typedef Tk_Window (*tk_mw) (Tcl_Interp *interp);
+static tk_mw TK_MAIN_WINDOW;
+typedef Tk_PhotoHandle (*tk_fp) (Tcl_Interp *interp, const char *imageName);
+static tk_fp TK_FIND_PHOTO;
+typedef void (*tk_ppb_nc) (Tk_PhotoHandle handle,
+        Tk_PhotoImageBlock *blockPtr, int x, int y,
+        int width, int height);
+static tk_ppb_nc TK_PHOTO_PUTBLOCK;
+typedef void (*tk_pb) (Tk_PhotoHandle handle);
+static tk_pb TK_PHOTO_BLANK;
+#else
+// Build-time linking against system TCL / Tk functions.
+#define TCL_CREATE_COMMAND Tcl_CreateCommand
+#define TCL_APPEND_RESULT Tcl_AppendResult
+#define TK_MAIN_WINDOW Tk_MainWindow
+#define TK_FIND_PHOTO Tk_FindPhoto
+#define TK_PHOTO_PUTBLOCK Tk_PhotoPutBlock
+#define TK_PHOTO_BLANK Tk_PhotoBlank
+#endif
+
 static int PyAggImagePhoto(ClientData clientdata, Tcl_Interp *interp, int argc, char **argv)
 {
     Tk_PhotoHandle photo;
@@ -61,25 +93,25 @@ static int PyAggImagePhoto(ClientData clientdata, Tcl_Interp *interp, int argc,
 
     long mode;
     long nval;
-    if (Tk_MainWindow(interp) == NULL) {
+    if (TK_MAIN_WINDOW(interp) == NULL) {
         // Will throw a _tkinter.TclError with "this isn't a Tk application"
         return TCL_ERROR;
     }
 
     if (argc != 5) {
-        Tcl_AppendResult(interp, "usage: ", argv[0], " destPhoto srcImage", (char *)NULL);
+        TCL_APPEND_RESULT(interp, "usage: ", argv[0], " destPhoto srcImage", (char *)NULL);
         return TCL_ERROR;
     }
 
     /* get Tcl PhotoImage handle */
-    photo = Tk_FindPhoto(interp, argv[1]);
+    photo = TK_FIND_PHOTO(interp, argv[1]);
     if (photo == NULL) {
-        Tcl_AppendResult(interp, "destination photo must exist", (char *)NULL);
+        TCL_APPEND_RESULT(interp, "destination photo must exist", (char *)NULL);
         return TCL_ERROR;
     }
     /* get array (or object that can be converted to array) pointer */
     if (sscanf(argv[2], SIZE_T_FORMAT, &aggl) != 1) {
-        Tcl_AppendResult(interp, "error casting pointer", (char *)NULL);
+        TCL_APPEND_RESULT(interp, "error casting pointer", (char *)NULL);
         return TCL_ERROR;
     }
     bufferobj = (PyObject *)aggl;
@@ -88,7 +120,7 @@ static int PyAggImagePhoto(ClientData clientdata, Tcl_Interp *interp, int argc,
     try {
         buffer = numpy::array_view<uint8_t, 3>(bufferobj);
     } catch (...) {
-        Tcl_AppendResult(interp, "buffer is of wrong type", (char *)NULL);
+        TCL_APPEND_RESULT(interp, "buffer is of wrong type", (char *)NULL);
         PyErr_Clear();
         return TCL_ERROR;
     }
@@ -99,13 +131,13 @@ static int PyAggImagePhoto(ClientData clientdata, Tcl_Interp *interp, int argc,
     /* get array mode (0=mono, 1=rgb, 2=rgba) */
     mode = atol(argv[3]);
     if ((mode != 0) && (mode != 1) && (mode != 2)) {
-        Tcl_AppendResult(interp, "illegal image mode", (char *)NULL);
+        TCL_APPEND_RESULT(interp, "illegal image mode", (char *)NULL);
         return TCL_ERROR;
     }
 
     /* check for bbox/blitting */
     if (sscanf(argv[4], SIZE_T_FORMAT, &bboxl) != 1) {
-        Tcl_AppendResult(interp, "error casting pointer", (char *)NULL);
+        TCL_APPEND_RESULT(interp, "error casting pointer", (char *)NULL);
         return TCL_ERROR;
     }
     bboxo = (PyObject *)bboxl;
@@ -126,7 +158,7 @@ static int PyAggImagePhoto(ClientData clientdata, Tcl_Interp *interp, int argc,
 
         destbuffer = new agg::int8u[deststride * destheight];
         if (destbuffer == NULL) {
-            Tcl_AppendResult(interp, "could not allocate memory", (char *)NULL);
+            TCL_APPEND_RESULT(interp, "could not allocate memory", (char *)NULL);
             return TCL_ERROR;
         }
 
@@ -167,7 +199,7 @@ static int PyAggImagePhoto(ClientData clientdata, Tcl_Interp *interp, int argc,
         block.pitch = deststride;
         block.pixelPtr = destbuffer;
 
-        Tk_PhotoPutBlock(photo, &block, destx, desty, destwidth, destheight);
+        TK_PHOTO_PUTBLOCK(photo, &block, destx, desty, destwidth, destheight);
         delete[] destbuffer;
 
     } else {
@@ -177,9 +209,9 @@ static int PyAggImagePhoto(ClientData clientdata, Tcl_Interp *interp, int argc,
         block.pixelPtr = buffer.data();
 
         /* Clear current contents */
-        Tk_PhotoBlank(photo);
+        TK_PHOTO_BLANK(photo);
         /* Copy opaque block to photo image, and leave the rest to TK */
-        Tk_PhotoPutBlock(photo, &block, 0, 0, block.width, block.height);
+        TK_PHOTO_PUTBLOCK(photo, &block, 0, 0, block.width, block.height);
     }
 
     return TCL_OK;
@@ -216,11 +248,11 @@ static PyObject *_tkinit(PyObject *self, PyObject *args)
 
     /* This will bomb if interp is invalid... */
 
-    Tcl_CreateCommand(interp,
-                      "PyAggImagePhoto",
-                      (Tcl_CmdProc *)PyAggImagePhoto,
-                      (ClientData)0,
-                      (Tcl_CmdDeleteProc *)NULL);
+    TCL_CREATE_COMMAND(interp,
+                       "PyAggImagePhoto",
+                       (Tcl_CmdProc *)PyAggImagePhoto,
+                       (ClientData)0,
+                       (Tcl_CmdDeleteProc *)NULL);
 
     Py_INCREF(Py_None);
     return Py_None;
@@ -232,6 +264,90 @@ static PyMethodDef functions[] = {
     { NULL, NULL } /* sentinel */
 };
 
+#ifdef DYNAMIC_TKINTER
+// Functions to fill global TCL / Tk function pointers from tkinter module.
+
+#include <dlfcn.h>
+
+#if PY3K
+#define TKINTER_PKG "tkinter"
+#define TKINTER_MOD "_tkinter"
+// From module __file__ attribute to char *string for dlopen.
+#define FNAME2CHAR(s) (PyBytes_AsString(PyUnicode_EncodeFSDefault(s)))
+#else
+#define TKINTER_PKG "Tkinter"
+#define TKINTER_MOD "tkinter"
+// From module __file__ attribute to char *string for dlopen
+#define FNAME2CHAR(s) (PyString_AsString(s))
+#endif
+
+void *_dfunc(void *lib_handle, const char *func_name)
+{
+    // Load function, unless there has been a previous error.  If so, then
+    // return NULL.  If there is an error loading the function, return NULL
+    // and set error flag.
+    static int have_error = 0;
+    void *func = NULL;
+    if (have_error == 0) {
+        // reset errors
+        dlerror();
+        func = dlsym(lib_handle, func_name);
+        const char *error = dlerror();
+        if (error != NULL) {
+            PyErr_SetString(PyExc_RuntimeError, error);
+            have_error = 1;
+        }
+    }
+    return func;
+}
+
+int _func_loader(void *tkinter_lib)
+{
+    // Fill global function pointers from dynamic lib.
+    // Return 0 fur success; 1 otherwise.
+    TCL_CREATE_COMMAND = (tcl_cc) _dfunc(tkinter_lib, "Tcl_CreateCommand");
+    TCL_APPEND_RESULT = (tcl_app_res) _dfunc(tkinter_lib, "Tcl_AppendResult");
+    TK_MAIN_WINDOW = (tk_mw) _dfunc(tkinter_lib, "Tk_MainWindow");
+    TK_FIND_PHOTO = (tk_fp) _dfunc(tkinter_lib, "Tk_FindPhoto");
+    TK_PHOTO_PUTBLOCK = (tk_ppb_nc) _dfunc(tkinter_lib, "Tk_PhotoPutBlock_NoComposite");
+    TK_PHOTO_BLANK = (tk_pb) _dfunc(tkinter_lib, "Tk_PhotoBlank");
+    return (TK_PHOTO_BLANK == NULL);
+}
+
+int load_tkinter_funcs(void)
+{
+    // Load tkinter global funcs from tkinter compiled module.
+    // Return 0 for success, non-zero for failure.
+    int ret = -1;
+    PyObject *pModule, *pSubmodule, *pString;
+
+    pModule = PyImport_ImportModule(TKINTER_PKG);
+    if (pModule != NULL) {
+        pSubmodule = PyObject_GetAttrString(pModule, TKINTER_MOD);
+        if (pSubmodule != NULL) {
+            pString = PyObject_GetAttrString(pSubmodule, "__file__");
+            if (pString != NULL) {
+                char *tkinter_libname = FNAME2CHAR(pString);
+                void *tkinter_lib = dlopen(tkinter_libname, RTLD_LAZY);
+                if (tkinter_lib == NULL) {
+                    PyErr_SetString(PyExc_RuntimeError,
+                            "Cannot dlopen tkinter module file");
+                } else {
+                    ret = _func_loader(tkinter_lib);
+                    // dlclose probably safe because tkinter has been
+                    // imported.
+                    dlclose(tkinter_lib);
+                }
+                Py_DECREF(pString);
+            }
+            Py_DECREF(pSubmodule);
+        }
+        Py_DECREF(pModule);
+    }
+    return ret;
+}
+#endif
+
 #if PY3K
 static PyModuleDef _tkagg_module = { PyModuleDef_HEAD_INIT, "_tkagg", "",   -1,  functions,
                                      NULL,                  NULL,     NULL, NULL };
@@ -244,13 +360,20 @@ PyMODINIT_FUNC PyInit__tkagg(void)
 
     import_array();
 
-    return m;
+#ifdef DYNAMIC_TKINTER
+    return (load_tkinter_funcs() == 0) ? m : NULL;
+#else
+    return m
+#endif
 }
 #else
 PyMODINIT_FUNC init_tkagg(void)
 {
     import_array();
 
     Py_InitModule("_tkagg", functions);
+#ifdef DYNAMIC_TKINTER
+    load_tkinter_funcs();
+#endif
 }
 #endif