Python `array` module is not free-thread safe.

[tom-pytel]

Bug report

Bug description:

The python array module is not currently free-thread safe. The error generator below can generate segfaults of aborts for most of the array object methods. I found #116738 which lists this module as in need of an audit, an apparenly closed start on this in #120103 and no owner for this module in the Experts Index, so decided to give it a go (see attached PR).

Reproducer, any single check(...) call below will generate some kind of segfault or abort fairly quickly.

from array import array
from io import BytesIO
from random import randint
import threading


def pop1(b, a): # MODIFIES!
 b.wait()
 try: a.pop()
 except IndexError: pass

def append1(b, a): # MODIFIES!
 b.wait()
 a.append(2)

def insert1(b, a): # MODIFIES!
 b.wait()
 a.insert(0, 2)

def extend(b, a): # MODIFIES!
 c = array('i', [2])
 b.wait()
 a.extend(c)
def extend2(b, a, c): # MODIFIES!
 b.wait()
 a.extend(c)

def inplace_concat(b, a): # MODIFIES!
 c = array('i', [2])
 b.wait()
 a += c
def inplace_concat2(b, a, c): # MODIFIES!
 b.wait()
 a += c

def inplace_repeat2(b, a): # MODIFIES!
 b.wait()
 a *= 2

def clear(b, a, *args): # MODIFIES!
 b.wait()
 a.clear()
def clear2(b, a, c): # MODIFIES c!
 b.wait()
 try: c.clear()
 except BufferError: pass

def remove1(b, a): # MODIFIES!
 b.wait()
 try: a.remove(1)
 except ValueError: pass

def fromunicode(b, a): # MODIFIES!
 b.wait()
 a.fromunicode('test')

def frombytes(b, a): # MODIFIES!
 b.wait()
 a.frombytes(b'0000')
def frombytes2(b, a, c): # MODIFIES!
 b.wait()
 a.frombytes(c)

def fromlist(b, a): # MODIFIES!
 n = randint(0, 100)
 b.wait()
 a.fromlist([2] * n)

def ass_subscr2(b, a, c): # MODIFIES!
 b.wait()
 a[:] = c

def ass0(b, a): # modifies inplace
 b.wait()
 try: a[0] = 0
 except IndexError: pass

def byteswap(b, a): # modifies inplace
 b.wait()
 a.byteswap()

def tounicode(b, a):
 b.wait()
 a.tounicode()

def tobytes(b, a):
 b.wait()
 a.tobytes()

def tolist(b, a):
 b.wait()
 a.tolist()

def tofile(b, a):
 f = BytesIO()
 b.wait()
 a.tofile(f)

def reduce_ex2(b, a):
 b.wait()
 a.__reduce_ex__(2)

def reduce_ex3(b, a):
 b.wait()
 c = a.__reduce_ex__(3)
 assert not c[1] or b'\xdd' not in c[1][3]

def copy(b, a):
 b.wait()
 c = a.__copy__()
 assert not c or 0xdd not in c

def repr1(b, a):
 b.wait()
 repr(a)

def repeat2(b, a):
 b.wait()
 a * 2

def count1(b, a):
 b.wait()
 a.count(1)

def index1(b, a):
 b.wait()
 try: a.index(1)
 except ValueError: pass

def contains1(b, a):
 b.wait()
 try: 1 in a
 except ValueError: pass

def subscr0(b, a):
 b.wait()
 try: a[0]
 except IndexError: pass

def concat(b, a):
 b.wait()
 a + a
def concat2(b, a, c):
 b.wait()
 a + c

def richcmplhs(b, a):
 c = a[:]
 b.wait()
 a == c

def richcmprhs(b, a):
 c = a[:]
 b.wait()
 c == a

def new(b, a):
 tc = a.typecode
 b.wait()
 array(tc, a)

def repr_(b, a):
 b.wait()
 repr(a)

def irepeat(b, a): # MODIFIES!
 b.wait()
 a *= 2


def newi(b, l):
 b.wait()
 array('i', l)

def fromlistl(b, a, l): # MODIFIES!
 b.wait()
 a.fromlist(l)
def fromlistlclear(b, a, l): # MODIFIES LIST!
 b.wait()
 l.clear()


def iter_next(b, a, it): # MODIFIES ITERATOR!
 b.wait()
 list(it)

def iter_reduce(b, a, it):
 b.wait()
 c = it.__reduce__()
 assert not c[1] or b'\xdd' not in c[1][0]


def check(funcs, a=None, *args):
 if a is None:
 a = array('i', [1])

 barrier = threading.Barrier(len(funcs))
 thrds = []

 for func in funcs:
 thrd = threading.Thread(target=func, args=(barrier, a, *args))

 thrds.append(thrd)
 thrd.start()

 for thrd in thrds:
 thrd.join()


if __name__ == "__main__":
 while True:
 check([pop1] * 10)
 check([pop1] + [subscr0] * 10)
 check([append1] * 10)
 check([insert1] * 10)
 check([pop1] + [index1] * 10)
 check([pop1] + [contains1] * 10)
 check([insert1] + [repeat2] * 10)
 check([pop1] + [repr1] * 10)
 check([inplace_repeat2] * 10)
 check([byteswap] * 10)
 check([insert1] + [clear] * 10)
 check([pop1] + [count1] * 10)
 check([remove1] * 10)
 check([clear] + [copy] * 10, array('B', b'0' * 0x400000))
 check([pop1] + [reduce_ex2] * 10)
 check([clear] + [reduce_ex3] * 10, array('B', b'0' * 0x400000))
 check([pop1] + [tobytes] * 10)
 check([pop1] + [tolist] * 10)
 check([clear, tounicode] * 10, array('w', 'a'*10000))
 check([clear, tofile] * 10, array('w', 'a'*10000))
 check([clear] + [extend] * 10)
 check([clear] + [inplace_concat] * 10)
 check([clear] + [concat] * 10, array('w', 'a'*10000))
 check([fromunicode] * 10, array('w', 'a'))
 check([frombytes] * 10)
 check([fromlist] * 10)
 check([clear] + [richcmplhs] * 10, array('i', [1]*10000))
 check([clear] + [richcmprhs] * 10, array('i', [1]*10000))
 check([clear, ass0] * 10, array('i', [1]*10000)) # to test array_ass_item must disable Py_mp_ass_subscript
 check([clear] + [new] * 10, array('w', 'a'*10000))
 check([clear] + [repr_] * 10, array('B', b'0' * 0x40000))
 check([clear] + [repr_] * 10, array('B', b'0' * 0x40000))
 check([clear] + [irepeat] * 10, array('B', b'0' * 0x40000))
 check([clear] + [iter_reduce] * 10, a := array('B', b'0' * 0x400), iter(a))

 # make sure we handle non-self objects correctly
 check([clear] + [newi] * 10, [2] * randint(0, 100))
 check([fromlistlclear] + [fromlistl] * 10, array('i', [1]), [2] * randint(0, 100))
 check([clear2] + [concat2] * 10, array('w', 'a'*10000), array('w', 'a'*10000))
 check([clear2] + [inplace_concat2] * 10, array('w', 'a'*10000), array('w', 'a'*10000))
 check([clear2] + [extend2] * 10, array('w', 'a'*10000), array('w', 'a'*10000))
 check([clear2] + [ass_subscr2] * 10, array('w', 'a'*10000), array('w', 'a'*10000))
 check([clear2] + [frombytes2] * 10, array('w', 'a'*10000), array('B', b'a'*10000))

 # iterator stuff
 check([clear] + [iter_next] * 10, a := array('i', [1] * 10), iter(a))

CPython versions tested on:

CPython main branch

Operating systems tested on:

Linux

Linked PRs

• gh-128942: make arraymodule.c free-thread safe #128943
• Revert "gh-128942: make array module thread safe (#128943)" #130707
• gh-128942: make arraymodule.c free-thread safe (lock-free) #130771

[kumaraditya303]

Thanks!

[colesbury]

Re-opening due to performance regression on pyperformance.

Let's try to benchmark future PRs that may be performance sensitive. I usually use https://github.com/colesbury/fastbench because it's quick to run and well correlated with the full pyperformance run. There's also the runners for pyperformance at:

https://github.com/facebookexperimental/free-threading-benchmarking
https://github.com/faster-cpython/benchmarking-public