Skip to content

Commit 8167594

Browse files
[3.14] gh-133885: Use locks instead of critical sections for _zstd (gh-134289) (gh-134560)
Move from using critical sections to locks for the (de)compression methods. Since the methods allow other threads to run, we should use a lock rather than a critical section. (cherry picked from commit 8dbc119) Co-authored-by: Emma Smith <[email protected]>
1 parent 896b745 commit 8167594

File tree

7 files changed

+229
-182
lines changed

7 files changed

+229
-182
lines changed

Lib/test/test_zstd.py

Lines changed: 53 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2430,10 +2430,8 @@ def test_buffer_protocol(self):
24302430
self.assertEqual(f.write(arr), LENGTH)
24312431
self.assertEqual(f.tell(), LENGTH)
24322432

2433-
@unittest.skip("it fails for now, see gh-133885")
24342433
class FreeThreadingMethodTests(unittest.TestCase):
24352434

2436-
@unittest.skipUnless(Py_GIL_DISABLED, 'this test can only possibly fail with GIL disabled')
24372435
@threading_helper.reap_threads
24382436
@threading_helper.requires_working_threading()
24392437
def test_compress_locking(self):
@@ -2470,7 +2468,6 @@ def run_method(method, input_data, output_data):
24702468
actual = b''.join(output) + rest2
24712469
self.assertEqual(expected, actual)
24722470

2473-
@unittest.skipUnless(Py_GIL_DISABLED, 'this test can only possibly fail with GIL disabled')
24742471
@threading_helper.reap_threads
24752472
@threading_helper.requires_working_threading()
24762473
def test_decompress_locking(self):
@@ -2506,6 +2503,59 @@ def run_method(method, input_data, output_data):
25062503
actual = b''.join(output)
25072504
self.assertEqual(expected, actual)
25082505

2506+
@threading_helper.reap_threads
2507+
@threading_helper.requires_working_threading()
2508+
def test_compress_shared_dict(self):
2509+
num_threads = 8
2510+
2511+
def run_method(b):
2512+
level = threading.get_ident() % 4
2513+
# sync threads to increase chance of contention on
2514+
# capsule storing dictionary levels
2515+
b.wait()
2516+
ZstdCompressor(level=level,
2517+
zstd_dict=TRAINED_DICT.as_digested_dict)
2518+
b.wait()
2519+
ZstdCompressor(level=level,
2520+
zstd_dict=TRAINED_DICT.as_undigested_dict)
2521+
b.wait()
2522+
ZstdCompressor(level=level,
2523+
zstd_dict=TRAINED_DICT.as_prefix)
2524+
threads = []
2525+
2526+
b = threading.Barrier(num_threads)
2527+
for i in range(num_threads):
2528+
thread = threading.Thread(target=run_method, args=(b,))
2529+
2530+
threads.append(thread)
2531+
2532+
with threading_helper.start_threads(threads):
2533+
pass
2534+
2535+
@threading_helper.reap_threads
2536+
@threading_helper.requires_working_threading()
2537+
def test_decompress_shared_dict(self):
2538+
num_threads = 8
2539+
2540+
def run_method(b):
2541+
# sync threads to increase chance of contention on
2542+
# decompression dictionary
2543+
b.wait()
2544+
ZstdDecompressor(zstd_dict=TRAINED_DICT.as_digested_dict)
2545+
b.wait()
2546+
ZstdDecompressor(zstd_dict=TRAINED_DICT.as_undigested_dict)
2547+
b.wait()
2548+
ZstdDecompressor(zstd_dict=TRAINED_DICT.as_prefix)
2549+
threads = []
2550+
2551+
b = threading.Barrier(num_threads)
2552+
for i in range(num_threads):
2553+
thread = threading.Thread(target=run_method, args=(b,))
2554+
2555+
threads.append(thread)
2556+
2557+
with threading_helper.start_threads(threads):
2558+
pass
25092559

25102560

25112561
if __name__ == "__main__":

Modules/_zstd/clinic/decompressor.c.h

Lines changed: 2 additions & 9 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Modules/_zstd/clinic/zstddict.c.h

Lines changed: 4 additions & 23 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)