In [1]:

    

import joblib
from joblib import Parallel, delayed


    

In [48]:

    

a = range(10000000)


    

In [49]:

    

def foo(var):
    return (((var / 2.0) * 1.5) + 23.42) * (0.00000005 * var)


    

In [50]:

    

def batch_foo(vars):
    results = []
    for var in vars:
        results.append(foo(var))


    

In [51]:

    

# %%time # Wall time: 5.12 s
# results = batch_foo(a)


    

    




CPU times: user 4.96 s, sys: 320 ms, total: 5.28 s
Wall time: 5.12 s

In [52]:

    

p = Parallel(n_jobs=3, verbose=10, batch_size=len(a)/10)


    

In [42]:

    

def partition(lst, n):
    division = len(lst) / float(n)
    return [ lst[int(round(division * i)): int(round(division * (i + 1)))] for i in xrange(n) ]


    

In [54]:

    

def yield_partition(lst, n):
    division = len(lst) / float(n)
    for i in xrange(n):
        yield lst[int(round(division * i)): int(round(division * (i + 1)))]


    

In [53]:

    

# results = p(delayed(batch_foo)(chunk) for chunk in partition(a, 3))
# [Parallel(n_jobs=3)]: Done   3 out of   3 | elapsed:   28.3s finished
results = p(delayed(batch_foo)(chunk) for chunk in partition(a, 3))


    

    




len(lst) / float(n) = division = 3333333.33333
for i in xrange(3)
	0	lst[0: 3333333]
	1	lst[3333333: 6666667]
	2	lst[6666667: 10000000]






    




[Parallel(n_jobs=3)]: Done   3 tasks      | elapsed:   28.3s
[Parallel(n_jobs=3)]: Done   3 out of   3 | elapsed:   28.3s finished

In [58]:

    

res = Parallel(n_jobs=3,
               verbose=10)(delayed(batch_foo)(chunk) for chunk in partition(a, 3))


    

    




len(lst) / float(n) = division = 3333333.33333
for i in xrange(3)
	0	lst[0: 3333333]
	1	lst[3333333: 6666667]
	2	lst[6666667: 10000000]






    




[Parallel(n_jobs=3)]: Done   4 out of   3 | elapsed:   21.9s remaining:   -5.5s
[Parallel(n_jobs=3)]: Done   4 out of   3 | elapsed:   29.7s remaining:   -7.4s
[Parallel(n_jobs=3)]: Done   3 out of   3 | elapsed:   29.7s finished

In [59]:

    

res = Parallel(n_jobs=3,
               verbose=10,
               backend='threading')(delayed(batch_foo)(chunk) for chunk in partition(a, 3))


    

    




[Parallel(n_jobs=3)]: Done   4 out of   3 | elapsed:   10.5s remaining:   -2.6s
[Parallel(n_jobs=3)]: Done   4 out of   3 | elapsed:   10.6s remaining:   -2.6s
[Parallel(n_jobs=3)]: Done   3 out of   3 | elapsed:   10.6s finished






    




len(lst) / float(n) = division = 3333333.33333
for i in xrange(3)
	0	lst[0: 3333333]
	1	lst[3333333: 6666667]
	2	lst[6666667: 10000000]

In [ ]: