Windows Python 2.7 multiprocessing.Manager.Queue deadlocks on put from child process -


i'm trying code similar following example working correctly:

from multiprocessing import process, queue, manager, pool import time datetime import datetime  def results_producer(the_work, num_procs):     results = manager().queue()     ppool = pool(num_procs)     multiplier = 3     #step = len(the_work)/(num_procs*multiplier)     step = 100     in xrange(0,len(the_work), step):         batch = the_work[i:i+step]         ppool.apply_async(do_work1, args=(i,batch,results))#,callback=results.put_nowait)     return (ppool, results)  def results_consumer(results, total_work, num_procs, pool=none):     current = 0     batch_size=10     total = total_work     est_remaining = 0     while current < total_work:         size = results.qsize()         est_remaining = total_work - (current + size)         if current % 1000 == 0:             print 'attempting retrieve item queue empty? %s, size: %d , remaining work: %d' % (results.empty(), size, est_remaining)         item = results.get()         results.task_done()         current += 1         if current % batch_size == 0 or total_work - current < batch_size:             if pool not none , est_remaining == 0 , size/num_procs > batch_size:                 pool.apply_async(do_work2, args=(current, item, true))             else:                 do_work2(current,item, false)             if current % 1000 == 0:                 print 'queue size: %d , remaining work: %d' % (size, est_remaining)  def do_work1(i, w, results):     time.sleep(.05)     if % 1000 == 0:        print 'did work %d: %d %d' % (i,w[0], w[-1])     j in w:         #create increasing amount of work on queue         results.put_nowait(range(j*2))  def do_work2(index, item, in_parallel):     time.sleep(1)     if index % 50 == 0:         print 'processed result %d length %d in parallel %s' % (index, len(item), in_parallel)  if __name__ == "__main__":     num_workers = 2     start = datetime.now()     print 'start: %s' % start     amount_work = 4000     the_work = [i in xrange(amount_work)]     ppool, results = results_producer(the_work, num_workers)      results_consumer(results, len(the_work), num_workers, ppool)      if ppool not none:         ppool.close()         ppool.join()     print 'took: %s time' % (datetime.now() - start) 

and deadlocks on results.put_nowait call do_work1 though queue empty! code able put work on queue results.get call results_consumer blocks since apparently empty though work has not been consumed yet.

additionally, checked programming guidelines: https://docs.python.org/2/library/multiprocessing.html , believe above code conforms it. lastly problem in post: python multiprocessing.queue deadlocks on put , get seems similar , claims solved on windows (i'm running on windows 8.1) above code doesn't block due parent process attempting join child process since logic similar suggested answer. suggestions cause of deadlock , how fix it? in general, best way enable multiple producers provide results consumer process in python?


Comments

Popular posts from this blog

php - Zend Framework / Skeleton-Application / Composer install issue -

c# - Better 64-bit byte array hash -

python - PyCharm Type error Message -