Switching deprecated Thread Local Storage (TLS) usage in Python 3.7 to Thread Specific Storage (TSS) (#1454)

* Switching deprecated Thread Local Storage (TLS) usage in Python 3.7 to Thread Specific Storage (TSS)

* Changing Python version from 3.6 to 3.7 for Travis CI, to match brew's version of Python 3

* Introducing PYBIND11_ macros to switch between TLS and TSS API

Author: YannickJadoul

.travis.yml

@@ -68,7 +68,7 @@ matrix:
     env: PYTHON=2.7 CPP=14 CLANG CMAKE=1
   - os: osx
     osx_image: xcode9
-    env: PYTHON=3.6 CPP=14 CLANG DEBUG=1
+    env: PYTHON=3.7 CPP=14 CLANG DEBUG=1
   # Test a PyPy 2.7 build
   - os: linux
     env: PYPY=5.8 PYTHON=2.7 CPP=11 GCC=4.8

include/pybind11/detail/internals.h

@@ -18,6 +18,25 @@ inline PyTypeObject *make_static_property_type();
 inline PyTypeObject *make_default_metaclass();
 inline PyObject *make_object_base_type(PyTypeObject *metaclass);
 
+// The old Python Thread Local Storage (TLS) API is deprecated in Python 3.7 in favor of the new
+// Thread Specific Storage (TSS) API.
+#if PY_VERSION_HEX >= 0x03070000
+    #define PYBIND11_TLS_KEY_INIT(var) Py_tss_t *var = nullptr
+    #define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get((key))
+    #define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set((key), (tstate))
+    #define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set((key), nullptr)
+#else
+    // Usually an int but a long on Cygwin64 with Python 3.x
+    #define PYBIND11_TLS_KEY_INIT(var) decltype(PyThread_create_key()) var = 0
+    #define PYBIND11_TLS_GET_VALUE(key) PyThread_get_key_value((key))
+    #if PY_MAJOR_VERSION < 3
+        #define PYBIND11_TLS_REPLACE_VALUE(key, value) do { PyThread_delete_key_value((key)); PyThread_set_key_value((key), (value)); } while (false)
+    #else
+        #define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_set_key_value((key), (value))
+    #endif
+    #define PYBIND11_TLS_DELETE_VALUE(key) PyThread_set_key_value((key), nullptr)
+#endif
+
 // Python loads modules by default with dlopen with the RTLD_LOCAL flag; under libc++ and possibly
 // other STLs, this means `typeid(A)` from one module won't equal `typeid(A)` from another module
 // even when `A` is the same, non-hidden-visibility type (e.g. from a common include).  Under
@@ -79,7 +98,7 @@ struct internals {
     PyTypeObject *default_metaclass;
     PyObject *instance_base;
 #if defined(WITH_THREAD)
-    decltype(PyThread_create_key()) tstate = 0; // Usually an int but a long on Cygwin64 with Python 3.x
+    PYBIND11_TLS_KEY_INIT(tstate);
     PyInterpreterState *istate = nullptr;
 #endif
 };
@@ -111,7 +130,7 @@ struct type_info {
 };
 
 /// Tracks the `internals` and `type_info` ABI version independent of the main library version
-#define PYBIND11_INTERNALS_VERSION 1
+#define PYBIND11_INTERNALS_VERSION 2
 
 #if defined(WITH_THREAD)
 #  define PYBIND11_INTERNALS_KIND ""
@@ -166,8 +185,17 @@ PYBIND11_NOINLINE inline internals &get_internals() {
 #if defined(WITH_THREAD)
         PyEval_InitThreads();
         PyThreadState *tstate = PyThreadState_Get();
-        internals_ptr->tstate = PyThread_create_key();
-        PyThread_set_key_value(internals_ptr->tstate, tstate);
+        #if PY_VERSION_HEX >= 0x03070000
+            internals_ptr->tstate = PyThread_tss_alloc();
+            if (!internals_ptr->tstate || PyThread_tss_create(internals_ptr->tstate))
+                pybind11_fail("get_internals: could not successfully initialize the TSS key!");
+            PyThread_tss_set(internals_ptr->tstate, tstate);
+        #else
+            internals_ptr->tstate = PyThread_create_key();
+            if (internals_ptr->tstate == -1)
+                pybind11_fail("get_internals: could not successfully initialize the TLS key!");
+            PyThread_set_key_value(internals_ptr->tstate, tstate);
+        #endif
         internals_ptr->istate = tstate->interp;
 #endif
         builtins[id] = capsule(internals_pp);

include/pybind11/pybind11.h

@@ -1774,7 +1774,7 @@ class gil_scoped_acquire {
 public:
     PYBIND11_NOINLINE gil_scoped_acquire() {
         auto const &internals = detail::get_internals();
-        tstate = (PyThreadState *) PyThread_get_key_value(internals.tstate);
+        tstate = (PyThreadState *) PYBIND11_TLS_GET_VALUE(internals.tstate);
 
         if (!tstate) {
             tstate = PyThreadState_New(internals.istate);
@@ -1783,10 +1783,7 @@ class gil_scoped_acquire {
                     pybind11_fail("scoped_acquire: could not create thread state!");
             #endif
             tstate->gilstate_counter = 0;
-            #if PY_MAJOR_VERSION < 3
-                PyThread_delete_key_value(internals.tstate);
-            #endif
-            PyThread_set_key_value(internals.tstate, tstate);
+            PYBIND11_TLS_REPLACE_VALUE(internals.tstate, tstate);
         } else {
             release = detail::get_thread_state_unchecked() != tstate;
         }
@@ -1825,7 +1822,7 @@ class gil_scoped_acquire {
             #endif
             PyThreadState_Clear(tstate);
             PyThreadState_DeleteCurrent();
-            PyThread_delete_key_value(detail::get_internals().tstate);
+            PYBIND11_TLS_DELETE_VALUE(detail::get_internals().tstate);
             release = false;
         }
     }
@@ -1850,11 +1847,7 @@ class gil_scoped_release {
         tstate = PyEval_SaveThread();
         if (disassoc) {
             auto key = internals.tstate;
-            #if PY_MAJOR_VERSION < 3
-                PyThread_delete_key_value(key);
-            #else
-                PyThread_set_key_value(key, nullptr);
-            #endif
+            PYBIND11_TLS_DELETE_VALUE(key);
         }
     }
     ~gil_scoped_release() {
@@ -1863,10 +1856,7 @@ class gil_scoped_release {
         PyEval_RestoreThread(tstate);
         if (disassoc) {
             auto key = detail::get_internals().tstate;
-            #if PY_MAJOR_VERSION < 3
-                PyThread_delete_key_value(key);
-            #endif
-            PyThread_set_key_value(key, tstate);
+            PYBIND11_TLS_REPLACE_VALUE(key, tstate);
         }
     }
 private: