In [11]:

    

%autosave 20

import pickle
from functools import partial
from multiprocessing import cpu_count
from multiprocessing import Pool as ProcessPool
from multiprocessing.pool import ThreadPool
from threading import Thread
from queue import Queue

import numpy as np


    

    















    




Autosaving every 20 seconds

In [2]:

    

cpus = cpu_count()
repeat = 8*cpus


    

In [3]:

    

A = np.random.rand(64, 64, 5, 5)
def f(x):
    return np.linalg.eigvals(A + x)

# def f(x):
#     return np.log(np.random.rand(100_000)**2 + x*500 + 1)**np.pi

# A = np.random.rand(100_000, 2)
# b = np.random.rand(100_000)
# def f(x):
#     return np.linalg.lstsq(A, b, rcond=None)

# a = list(range(1_000_000))

# def f(x, y):
#     return sum(a) * x + y


    

In [4]:

    

%timeit f(0)


    

    




41 ms ± 1.18 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)

In [5]:

    

%%timeit
for i in range(repeat):
    f(i)


    

    




2.34 s ± 5.6 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [6]:

    

%%timeit
threads = tuple(Thread(target=f, daemon=False, args=(i,))
                for i in range(repeat))
for t in threads:
    t.start()
for t in threads:
    t.join()


    

    




331 ms ± 7.87 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [7]:

    

%%timeit
with ThreadPool(processes=cpus) as pool:
    result = pool.map(f, range(repeat))


    

    




389 ms ± 34.7 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [8]:

    

%%timeit
with ProcessPool(processes=cpus) as pool:
#     result = pool.map(partial(f, 2), range(repeat))
    result = pool.map(f, range(repeat))


    

    




437 ms ± 1.65 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [9]:

    

b = pickle.dumps([1,2])
pickle.loads(b)


    

    
Out[9]:





[1, 2]

In [16]:

    

q = Queue()
q.put(1)
q.get(timeout=1)
q.get()


    

    




---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-16-9a144bc03be6> in <module>()
      2 q.put(1)
      3 q.get(timeout=1)
----> 4 q.get()

/opt/conda/lib/python3.6/queue.py in get(self, block, timeout)
    162             elif timeout is None:
    163                 while not self._qsize():
--> 164                     self.not_empty.wait()
    165             elif timeout < 0:
    166                 raise ValueError("'timeout' must be a non-negative number")

/opt/conda/lib/python3.6/threading.py in wait(self, timeout)
    293         try:    # restore state no matter what (e.g., KeyboardInterrupt)
    294             if timeout is None:
--> 295                 waiter.acquire()
    296                 gotit = True
    297             else:

KeyboardInterrupt: 

In [ ]: