In [1]:

    

import os, sys
import numpy as np
import time
from multiprocessing import Pool
from multiprocessing.dummy import Pool as ThreadPool


    

In [2]:

    

def get_a_thing(seed=None):
    """Returns a number after a random amount of time
    """
    np.random.seed()
    t = np.random.random()*3
    time.sleep(t)
    return t

def get_n_things(seed=None):
    """Returns a random number of random things
    """
    n = int(np.random.random()*5)
    return [get_a_thing() for i in range(n)]
    
    
def build_list(N=6, msg='hello.'):
    start = time.time()
    N = int(N)
    l = []
    while len(l) < N:
        l += (get_n_things())
    end = time.time()
    print('Built list of {} things on pid {}.  Took {:.3f}s. {}'.format(N, os.getpid(),
                                                                       end-start,
                                                                       msg))
    sys.stdout.flush()
    return l[:N]


    

In [3]:

    

build_list(4)


    

    




Built list of 4 things on pid 10286.  Took 6.679s. hello.






    
Out[3]:





[1.5205096328923748,
 2.0517572256985392,
 0.31739338650658444,
 2.745190905068869]

In [31]:

    

from functools import partial
import itertools

def build_list_wrapper(args):
    return build_list(*args)

def run_parallel(N=8, threads=False):
    start = time.time()
    if threads:
        pool = ThreadPool(4)
    else:
        pool = Pool(4)
    
        start = time.time()
    
    f = partial(build_list, msg='yo')
        
    results = pool.map(f, range(N))
    
    end = time.time()
    print('{:.3} s'.format(end-start))
    pool.terminate()
    return results


    

In [32]:

    

run_parallel(threads=True)


    

    




Built list of 0 things on pid 10286.  Took 0.000s. yo
Built list of 2 things on pid 10286.  Took 2.726s. yo
Built list of 1 things on pid 10286.  Took 4.988s. yo
Built list of 3 things on pid 10286.  Took 6.790s. yo
Built list of 4 things on pid 10286.  Took 12.516s. yo
Built list of 5 things on pid 10286.  Took 10.345s. yo
Built list of 6 things on pid 10286.  Took 10.499s. yo
Built list of 7 things on pid 10286.  Took 8.717s. yo
15.5 s






    
Out[32]:





[[],
 [1.018741015833875],
 [1.9315918121362787, 0.7738834922977176],
 [0.19909977346885221, 2.076437143666374, 0.5430238775263693],
 [2.0729002796832754,
  0.6952739799216717,
  2.336141478402969,
  1.5493230954663746],
 [1.1602512753493142,
  0.11331577302782869,
  2.5784789658830567,
  2.7134176293599843,
  1.2200994995631036],
 [2.9224428976370476,
  0.3093354837648403,
  1.9917850268367165,
  1.2128619813519776,
  1.0363784234829319,
  2.9468843988107456],
 [1.917363475517086,
  1.0519048018307924,
  1.1278417548520963,
  1.157408055067608,
  0.8098359177273987,
  0.6734979808260134,
  0.9118311367568643]]

In [ ]: