DISPATCH-2144 Fatal Python error: _PyMem_DebugMalloc: Python memory allocator called without holding the GIL (#1495)

Author: jiridanek

src/python_embedded.c

@@ -57,15 +57,22 @@ void qd_python_initialize(qd_dispatch_t *qd, const char *python_pkgdir)
     if (python_pkgdir)
         dispatch_python_pkgdir = PyUnicode_FromString(python_pkgdir);
 
-    qd_python_lock_state_t ls = qd_python_lock();
     Py_Initialize();
+#if PY_VERSION_HEX < 0x03070000
+    PyEval_InitThreads(); // necessary for Python 3.6 and older versions
+#endif
     qd_python_setup();
-    qd_python_unlock(ls);
+    PyEval_SaveThread(); // drop the Python GIL; we will reacquire it in other threads as needed
+    qd_python_lock_state_t lk = qd_python_lock();
+    qd_python_setup();
+    qd_python_unlock(lk);
 }
 
 
 void qd_python_finalize(void)
 {
+    (void) PyGILState_Ensure(); // not qd_python_lock(), because that function has to be paired with an _unlock
+
     sys_mutex_free(ilock);
     Py_DECREF(dispatch_module);
     dispatch_module = 0;
@@ -900,11 +907,12 @@ qd_python_lock_state_t qd_python_lock(void)
 {
     sys_mutex_lock(ilock);
     lock_held = true;
-    return 0;
+    return PyGILState_Ensure();
 }
 
 void qd_python_unlock(qd_python_lock_state_t lock_state)
 {
+    PyGILState_Release(lock_state);
     lock_held = false;
     sys_mutex_unlock(ilock);
 }

src/server.c

@@ -1450,7 +1450,9 @@ void qd_server_free(qd_server_t *qd_server)
     sys_mutex_free(qd_server->lock);
     sys_mutex_free(qd_server->conn_activation_lock);
     sys_cond_free(qd_server->cond);
+    qd_python_lock_state_t ls = qd_python_lock();
     Py_XDECREF((PyObject *)qd_server->py_displayname_obj);
+    qd_python_unlock(ls);
     free(qd_server);
 }