In [1]:

    

import multiprocessing
from contextlib import closing

def parallel(worker, data):
    with closing(multiprocessing.Pool(multiprocessing.cpu_count())) as pool:
        for result in pool.map(worker, data):
            yield result


    

In [2]:

    

import time
import random

def simple_worker(data):
    wait_seconds = random.randint(1, 10)
    print("Sleep %d seconds" % wait_seconds)
    time.sleep(wait_seconds)
    return data
    
list(parallel(simple_worker, list(range(10))))


    

    




Sleep 9 seconds
Sleep 1 seconds
Sleep 10 seconds
Sleep 7 seconds
Sleep 9 seconds
Sleep 3 seconds
Sleep 3 seconds
Sleep 10 seconds
Sleep 2 seconds
Sleep 8 seconds






    
Out[2]:





[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]

In [13]:

    

class Consumer(multiprocessing.Process):
    def __init__(self, result_queue):
        multiprocessing.Process.__init__(self)
        self.result_queue = result_queue
        self.keep_running = True

    def run(self):
        while self.keep_running:
            result = self.result_queue.get()
            print("Consumer receives %s" % result)
        print("consumer stops now")

class Producer(multiprocessing.Process):
    def __init__(self, result_queue, value):
        multiprocessing.Process.__init__(self)
        self.result_queue = result_queue
        self.value = value
        self.start()

    def run(self):
        wait_seconds = random.randint(1, 10)
        print("Sleep %d seconds" % wait_seconds)
        time.sleep(wait_seconds)
        self.result_queue.put(self.value)


# Establish communication queues
result_queue = multiprocessing.Queue()
consumer = Consumer(result_queue)
consumer.start()

tasks = [Producer(result_queue, idx) for idx in range(10)]
for task in tasks:
    task.join()

print("Waiting for consumer")
consumer.keep_running = False
result_queue.close()
consumer.terminate()
print("done")


    

    




Sleep 4 seconds
Sleep 7 seconds
Sleep 8 seconds
Sleep 8 seconds
Sleep 5 seconds
Sleep 2 seconds
Sleep 10 seconds
Sleep 4 seconds
Sleep 9 seconds
Sleep 9 seconds
Consumer receives 9
Consumer receives 1
Consumer receives 5
Consumer receives 7
Consumer receives 0
Consumer receives 2
Consumer receives 3
Consumer receives 8
Consumer receives 6
Consumer receives 4
Waiting for consumer
done

In [ ]: