USE PARALLEL COMPUTATIONS

Values to compute



In [1]:

    
one_val = 10
n_rpt = 20

def do_something(dummy):
    return one_val * dummy

MAP



In [2]:

    
output = map(do_something, range(n_rpt))
print(list(output))









    



[0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160, 170, 180, 190]

MULTIPROCESSING



In [3]:

    
from multiprocessing import Pool
p = Pool(12)
print(p.map(do_something, range(n_rpt)))









    



[0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160, 170, 180, 190]

JOBLIB



In [4]:

    
from joblib import Parallel, delayed

output = Parallel(n_jobs=12, verbose=20)(delayed(do_something)(i) for i in range(n_rpt))
print(output)









    



[0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160, 170, 180, 190]






    



[Parallel(n_jobs=12)]: Done   1 out of  20 | elapsed:    0.0s remaining:    0.0s
[Parallel(n_jobs=12)]: Done   2 out of  20 | elapsed:    0.0s remaining:    0.0s
[Parallel(n_jobs=12)]: Done   4 out of  20 | elapsed:    0.0s remaining:    0.0s
[Parallel(n_jobs=12)]: Done   6 out of  20 | elapsed:    0.0s remaining:    0.0s
[Parallel(n_jobs=12)]: Done   8 out of  20 | elapsed:    0.0s remaining:    0.0s
[Parallel(n_jobs=12)]: Done  10 out of  20 | elapsed:    0.0s remaining:    0.0s
[Parallel(n_jobs=12)]: Done  12 out of  20 | elapsed:    0.0s remaining:    0.0s
[Parallel(n_jobs=12)]: Done  14 out of  20 | elapsed:    0.0s remaining:    0.0s
[Parallel(n_jobs=12)]: Done  16 out of  20 | elapsed:    0.0s remaining:    0.0s
[Parallel(n_jobs=12)]: Done  18 out of  20 | elapsed:    0.0s remaining:    0.0s
[Parallel(n_jobs=12)]: Done  20 out of  20 | elapsed:    0.0s finished

IPYTHON PARALLEL

In ipython 2.0, rememberto start the clusters in the main interface. If not, it will fail.



In [5]:

    
from IPython.parallel import Client
c = Client()

v = c[:]
v.push({'one_val': one_val})

a = v.map(do_something, range(n_rpt))
print(list(a))









    



[0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160, 170, 180, 190]

SUBMIT JOBS TO LSF



In [6]:

    
from lsf import map_lsf

from logging import getLogger, INFO
lg = getLogger('lsf')
lg.setLevel(INFO)

output = map_lsf(do_something, range(n_rpt), queue='vshort', variables={'one_val': one_val}, imports={'math': 'sqrt'})
print(output)









    



INFO:lsf:funct_3413034_p40326_b000000_do_something_j000000 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000002 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000001 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000004 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000006 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000008 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000010 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000012 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000014 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000016 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000018 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000003 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000007 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000011 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000015 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000019 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000005 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000013 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000009 has finished
INFO:lsf:funct_3413034_p40326_b000000_do_something_j000017 has finished
WARNING:lsf:Wall Time: 14.51s, CPU time: 1.26s, speed-up: 0.09X






    



[0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160, 170, 180, 190]