How can I limit the maximum of open threads to 20 in the following code? I'm aware that there have been some similar questions asked in the past, but I specifically want to know how this is best done with a Queue and with a working example if possible.
# b is a list with 10000 items
threads = [threading.Thread(target=targetFunction, args=(ptf,anotherarg)) for ptf in b]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
The simple way to do this is with a queue.Queue for the work and starting the threads with for _ in range(MAXTHREADS): threading.Thread(target=f, args=(the_queue,)).start(). I find this easier to read by subclassing Thread, however. Your mileage may vary.
import threading
import queue
class Worker(threading.Thread):
def __init__(self, q, other_arg, *args, **kwargs):
self.q = q
self.other_arg = other_arg
super().__init__(*args, **kwargs)
def run(self):
while True:
try:
work = self.q.get(timeout=3) # 3s timeout
except queue.Empty:
return
# do whatever work you have to do on work
self.q.task_done()
q = queue.Queue()
for ptf in b:
q.put_nowait(ptf)
for _ in range(20):
Worker(q, otherarg).start()
q.join() # blocks until the queue is empty.
If you're insistent about using a function, I'd suggest wrapping your targetFunction with something that knows how to get from the queue.
def wrapper_targetFunc(f, q, somearg):
while True:
try:
work = q.get(timeout=3) # or whatever
except queue.Empty:
return
f(work, somearg)
q.task_done()
q = queue.Queue()
for ptf in b:
q.put_nowait(ptf)
for _ in range(20):
threading.Thread(target=wrapper_targetFunc,
args=(targetFunction, q, otherarg)).start()
q.join()
