notebook.community

Edit and run

Score multi-session events using the replay score from Davidson et al.





In [1]:



    


import numpy as np
import os
import pandas as pd
import warnings

import nelpy as nel

warnings.filterwarnings("ignore")

Load experimental data





In [2]:



    


datadirs = ['/Users/ckemere/Development/Data/Buzsaki']

fileroot = next( (dir for dir in datadirs if os.path.isdir(dir)), None)
# conda install pandas=0.19.2
if fileroot is None:
    raise FileNotFoundError('datadir not found')

load_from_nel = True

# load from nel file:
if load_from_nel:
    jar = nel.load_pkl(os.path.join(fileroot,'gor01vvp01_processed_speed.nel'))
    exp_data = jar.exp_data
    aux_data = jar.aux_data
    del jar
    
    with pd.HDFStore(os.path.join(fileroot,'DibaMetadata.h5')) as store:
        df = store.get('Session_Metadata')
        df2 = store.get('Subset_Metadata')



    











In [ ]:

Define subset of sessions to score





In [3]:



    


# restrict sessions to explore to a smaller subset
min_n_placecells = 16
min_n_PBEs = 27 # 27 total events ==> minimum 21 events in training set

df2_subset = df2[(df2.n_PBEs >= min_n_PBEs) & (df2.n_placecells >= min_n_placecells)]

sessions = df2_subset['time'].values.tolist()
segments = df2_subset['segment'].values.tolist()

print('Evaluating subset of {} sessions'.format(len(sessions)))

df2_subset.sort_values(by=['n_PBEs', 'n_placecells'], ascending=[0,0])



    


















    






Evaluating subset of 40 sessions










    
Out[3]:











  
    

      
      animal
      month
      day
      time
      track
      segment
      duration
      n_cells
      n_placecells
      n_PBEs
      Notes
      prescreen_z
    

  
  
    

      58
      gor01
      6
      9
      22-24-40
      two
      short
      1620.0
      203
      61
      301
      NaN
      NaN
    

    

      58
      gor01
      6
      9
      22-24-40
      two
      short
      1620.0
      203
      61
      301
      NaN
      NaN
    

    

      35
      gor01
      6
      7
      16-40-19
      two
      short
      1330.0
      117
      46
      277
      NaN
      NaN
    

    

      35
      gor01
      6
      7
      16-40-19
      two
      short
      1330.0
      117
      46
      277
      NaN
      NaN
    

    

      36
      gor01
      6
      7
      16-40-19
      two
      long
      1180.0
      117
      43
      150
      NaN
      NaN
    

    

      36
      gor01
      6
      7
      16-40-19
      two
      long
      1180.0
      117
      43
      150
      NaN
      NaN
    

    

      73
      gor01
      6
      9
      1-22-43
      one
      long
      1012.0
      203
      62
      117
      NaN
      NaN
    

    

      73
      gor01
      6
      9
      1-22-43
      one
      long
      1012.0
      203
      62
      117
      NaN
      NaN
    

    

      59
      gor01
      6
      9
      22-24-40
      two
      long
      912.0
      203
      66
      103
      NaN
      NaN
    

    

      59
      gor01
      6
      9
      22-24-40
      two
      long
      912.0
      203
      66
      103
      NaN
      NaN
    

    

      72
      gor01
      6
      9
      1-22-43
      one
      short
      617.0
      203
      71
      91
      NaN
      NaN
    

    

      72
      gor01
      6
      9
      1-22-43
      one
      short
      617.0
      203
      71
      91
      NaN
      NaN
    

    

      6
      gor01
      6
      8
      21-16-25
      two
      long
      720.0
      171
      82
      57
      NaN
      NaN
    

    

      6
      gor01
      6
      8
      21-16-25
      two
      long
      720.0
      171
      82
      57
      NaN
      NaN
    

    

      0
      vvp01
      4
      9
      17-29-30
      one
      short
      490.0
      68
      32
      46
      NaN
      NaN
    

    

      0
      vvp01
      4
      9
      17-29-30
      one
      short
      490.0
      68
      32
      46
      NaN
      NaN
    

    

      83
      vvp01
      4
      9
      16-40-54
      two
      long
      861.0
      41
      25
      42
      NaN
      NaN
    

    

      83
      vvp01
      4
      9
      16-40-54
      two
      long
      861.0
      41
      25
      42
      NaN
      NaN
    

    

      5
      gor01
      6
      8
      21-16-25
      two
      short
      457.0
      171
      64
      37
      NaN
      NaN
    

    

      5
      gor01
      6
      8
      21-16-25
      two
      short
      457.0
      171
      64
      37
      NaN
      NaN
    

    

      82
      vvp01
      4
      9
      16-40-54
      two
      short
      485.0
      41
      16
      37
      NaN
      NaN
    

    

      82
      vvp01
      4
      9
      16-40-54
      two
      short
      485.0
      41
      16
      37
      NaN
      NaN
    

    

      14
      gor01
      6
      12
      16-53-46
      two
      long
      470.0
      81
      36
      36
      NaN
      NaN
    

    

      14
      gor01
      6
      12
      16-53-46
      two
      long
      470.0
      81
      36
      36
      NaN
      NaN
    

    

      28
      vvp01
      4
      25
      14-28-51
      one
      long
      385.0
      80
      32
      36
      NaN
      NaN
    

    

      28
      vvp01
      4
      25
      14-28-51
      one
      long
      385.0
      80
      32
      36
      NaN
      NaN
    

    

      51
      vvp01
      4
      16
      15-12-23
      one
      short
      347.0
      74
      24
      34
      NaN
      NaN
    

    

      51
      vvp01
      4
      16
      15-12-23
      one
      short
      347.0
      74
      24
      34
      NaN
      NaN
    

    

      1
      vvp01
      4
      9
      17-29-30
      one
      long
      800.0
      68
      34
      33
      NaN
      NaN
    

    

      1
      vvp01
      4
      9
      17-29-30
      one
      long
      800.0
      68
      34
      33
      NaN
      NaN
    

    

      61
      gor01
      6
      12
      15-55-31
      one
      long
      660.0
      97
      40
      32
      NaN
      NaN
    

    

      61
      gor01
      6
      12
      15-55-31
      one
      long
      660.0
      97
      40
      32
      NaN
      NaN
    

    

      75
      gor01
      6
      13
      15-22-3
      two
      long
      530.0
      82
      37
      31
      NaN
      NaN
    

    

      75
      gor01
      6
      13
      15-22-3
      two
      long
      530.0
      82
      37
      31
      NaN
      NaN
    

    

      50
      gor01
      6
      13
      14-42-6
      one
      long
      520.0
      109
      36
      29
      NaN
      NaN
    

    

      50
      gor01
      6
      13
      14-42-6
      one
      long
      520.0
      109
      36
      29
      NaN
      NaN
    

    

      41
      vvp01
      4
      17
      12-52-15
      two
      short
      328.0
      59
      26
      28
      NaN
      NaN
    

    

      41
      vvp01
      4
      17
      12-52-15
      two
      short
      328.0
      59
      26
      28
      NaN
      NaN
    

    

      60
      gor01
      6
      12
      15-55-31
      one
      short
      410.0
      97
      44
      27
      NaN
      NaN
    

    

      60
      gor01
      6
      12
      15-55-31
      one
      short
      410.0
      97
      44
      27
      NaN
      NaN

Parallel scoring

NOTE: it is relatively easy (syntax-wise) to score each session as a parallel task, but since the Bayesian scoring takes such a long time to compute, we can be more efficient (higher % utilization) by further parallelizing over events, and not just over sessions. This further level of parallelization makes the bookkeeping a little ugly, so I provide the code for both approaches here.





In [4]:



    


n_jobs = 20 # set this equal to number of cores
n_shuffles = 100 # 5000
n_samples = 35000 # 35000
w=3 # single sided bandwidth (0 means only include bin who's center is under line, 3 means a total of 7 bins)



    











In [5]:



    


import matplotlib.pyplot as plt
%matplotlib inline



    











In [6]:



    


# Parallelize by EVENT
import dask
import distributed.joblib
from joblib import Parallel, delayed 
from joblib import parallel_backend

# A function that can be called to do work:
def work_events(arg):    

    # Split the list to individual variables:
    session, segment, ii, bst, tc = arg
    scores, shuffled_scores, percentiles = nel.analysis.replay.score_Davidson_final_bst_fast(bst=bst,
                                                                                        tuningcurve=tc,
                                                                                        w=w,
                                                                                        n_shuffles=n_shuffles,
                                                                                        n_samples=n_samples)

    return (session, segment, ii, scores, shuffled_scores, percentiles)

# List of instances to pass to work():
# unroll all events:
parallel_events = []
for session, segment in zip(sessions, segments):
    for nn in range(aux_data[session][segment]['PBEs'].n_epochs):
        parallel_events.append((session, segment, nn, aux_data[session][segment]['PBEs'][nn], 
                                aux_data[session][segment]['tc']))

#parallel_results = list(map(work_events, parallel_events))

with parallel_backend('dask.distributed', scheduler_host='35.184.42.12:8786'):
    # Anything returned by work() can be stored:
    parallel_results = Parallel(n_jobs=n_jobs, verbose=1)(map(delayed(work_events), parallel_events))



    


















    






distributed.client - WARNING - Client report stream closed to scheduler
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c665c5bf8>, <tornado.concurrent.Future object at 0x1c665c1b00>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-6407a9716f68431588a9af8b84f20215']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c665c8048>, <tornado.concurrent.Future object at 0x1c665c1cc0>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-26da7ce53ab044509e59182ee3938f89']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c665c8400>, <tornado.concurrent.Future object at 0x1c665c1e80>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-ddac9995d7e54cdca15818d8a136f560']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c665d3378>, <tornado.concurrent.Future object at 0x1c665ca5c0>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-d9d69834d4294a2a85e4cac8f277f93c']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c665d3730>, <tornado.concurrent.Future object at 0x1c665ca780>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-813964dd1fb74a4aa1b6ab2bb3513628']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c665d3ae8>, <tornado.concurrent.Future object at 0x1c665ca940>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-d08512594b1a4edd9cc985af54da6a1a']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c665e36a8>, <tornado.concurrent.Future object at 0x1c665caf98>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-67fc4ea64ce749788fc3b7ea25445668']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c665e3a60>, <tornado.concurrent.Future object at 0x1c8b899208>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-748a59748cd14f23b5bc02b6332e28f6']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c665e3e18>, <tornado.concurrent.Future object at 0x1c8b899438>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-a6e6dd3837c74cbea6047e5b444bfe82']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c8b89dd90>, <tornado.concurrent.Future object at 0x1c8b899cf8>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-c4b043b3de264c159820fafe0b531c09']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c8b8a51e0>, <tornado.concurrent.Future object at 0x1c8b899f28>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-1898167894c9431c96db172454602ff1']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c8b8a5598>, <tornado.concurrent.Future object at 0x1c8b8a1198>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-1bdd41c97f33460f8cf0610c95df7809']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c8b8afb70>, <tornado.concurrent.Future object at 0x1c8b8d1940>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-d83d51c8f73f488e82b2cdebc340fc46']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c8b8aff28>, <tornado.concurrent.Future object at 0x1c8b8d1b00>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-941e5055b1bb46f99d568ba53da80007']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1caca3d378>, <tornado.concurrent.Future object at 0x1c8b8d1cc0>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-e99cfbbc14d04172963adae93ad9ee63']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1caca3dea0>, <tornado.concurrent.Future object at 0x1caca3f240>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-a291416f50234eb5ba627a45da48694b']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1caca422f0>, <tornado.concurrent.Future object at 0x1caca3f400>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-87fb012e2bfa46b9851ea426dfbfbee0']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1caca426a8>, <tornado.concurrent.Future object at 0x1caca3f5c0>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-cfd465e6ce7f4891946654cb5f61f624']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c96202e18>, <tornado.concurrent.Future object at 0x1c96206748>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-b5673a0c21f845a5b708af3d933771d1']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c96202510>, <tornado.concurrent.Future object at 0x1c961f9d30>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-dea4305e14444e4a917bc7a6ba91feda']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c8b8d2bf8>, <tornado.concurrent.Future object at 0x1c1d82f470>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-e3b663a420c444c889406754456a0975']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c1d8339d8>, <tornado.concurrent.Future object at 0x1c1d82f630>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-33c6ab5feecf4af596dcb04e2b27335f']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c1d8428c8>, <tornado.concurrent.Future object at 0x1c665c17b8>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-2e832b4246cc475fa32c23ffe5793d6d']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c665c5620>, <tornado.concurrent.Future object at 0x1c665c1978>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-6440fa3eafdf4be3a1f3b9548b833aca']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c665b1488>, <tornado.concurrent.Future object at 0x1c1d8254e0>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-829131006e7a4784803d77f21e7d857e']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c9621b400>, <tornado.concurrent.Future object at 0x1c961f9630>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-d37ed62f20c842c190530c19a2073328']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c1d82cd90>, <tornado.concurrent.Future object at 0x1c1d825cf8>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-b7752b6a0de44d63bebe08f6495fb492']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c1d82c7b8>, <tornado.concurrent.Future object at 0x1c1d825b70>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-18555c569b6547719329a8372d444421']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c7160e378>, <tornado.concurrent.Future object at 0x1c1d82f5f8>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-7bf32d7bb35c4881a379a2796073b0f2']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c9621bf28>, <tornado.concurrent.Future object at 0x1c96213ac8>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-4b9bc1db0b634d24aa42a7022fb0b7e7']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c1d82c378>, <tornado.concurrent.Future object at 0x1c1d8259b0>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-dca11a517c3b4acda2309690cf69d276']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c961fd0d0>, <tornado.concurrent.Future object at 0x1caf97b898>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-b65d8fdf16374118a97eda72a6a6dc26']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c961fd048>, <tornado.concurrent.Future object at 0x1c665c11d0>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-d93b759350c740f9a61c4495e3797095']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c9620e268>, <tornado.concurrent.Future object at 0x1c96213fd0>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-74029b057a084d15a2122b91fcee1818']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c665bd7b8>, <tornado.concurrent.Future object at 0x1c665ae390>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-99f79e21b623420cb5b81e1ebd914fd2']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c71616bf8>, <tornado.concurrent.Future object at 0x1c961f9b70>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-2ea1fdb3b0854cb3a24f9ac842c71f42']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c1d8426a8>, <tornado.concurrent.Future object at 0x1c1d82fc50>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-37c2cf884e9847ccaaf427a14ea240ec']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c9621b7b8>, <tornado.concurrent.Future object at 0x1c96206208>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-d0504b86914d4eb0948d08ec7bdb2ed7']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c9620e950>, <tornado.concurrent.Future object at 0x1c665c1588>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-a6ca03df81754fd0a6f4eb8c7e4672e2']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c8b8d2ae8>, <tornado.concurrent.Future object at 0x1c1d82f048>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-0fa3c317be1f420fb7046415caed3be5']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c96216d08>, <tornado.concurrent.Future object at 0x1c961f97f0>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-7c360b253392409496d96dd59a3b0209']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c96230268>, <tornado.concurrent.Future object at 0x1c71602358>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-016afbd8dcb446868b8893fab469b165']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c1d828c80>, <tornado.concurrent.Future object at 0x1c1d825630>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-f4a571d4648e49df80dbd068cf509d7c']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c715e0d08>, <tornado.concurrent.Future object at 0x1c1d82f2b0>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-2c0cd671b771471fb42588c481f2b4b8']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c715e0bf8>, <tornado.concurrent.Future object at 0x1caf97bc88>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-303b0ed7475e418f99e49ef3ce51212a']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1caca45488>, <tornado.concurrent.Future object at 0x1c1d825160>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-7fd950ed94b84282b87996e8805484f6']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c9621b378>, <tornado.concurrent.Future object at 0x1c96206c18>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-a73a73e65df647b191614e9967783ea4']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c1d8332f0>, <tornado.concurrent.Future object at 0x1c1d82f0f0>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-ac5ae1219d8646ccb7244b10e579a051']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c1d828f28>, <tornado.concurrent.Future object at 0x1c1d8257f0>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-0ea38ad87928479ca2ade34134e36373']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c9621b730>, <tornado.concurrent.Future object at 0x1c96213668>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-803a075599f949fc934ceefac1d7d943']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c1d833950>, <tornado.concurrent.Future object at 0x1c715ec6d8>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-4b29ce28a85f40f7b7e58e8213527811']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c962162f0>, <tornado.concurrent.Future object at 0x1c71602d30>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-573eafef27174563a793bc127cbea4ad']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c7160ed90>, <tornado.concurrent.Future object at 0x1caca3f0b8>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-9bc070bc1e3741fe9da096dd84527831']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1caca3d510>, <tornado.concurrent.Future object at 0x1c8b8d1eb8>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-02ff4cd66c014a55af2930413728b5ee']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c665d3c80>, <tornado.concurrent.Future object at 0x1c665cab70>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-a3b50ab5f1a24c36a51090019986e49a']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c665e3378>, <tornado.concurrent.Future object at 0x1c665cada0>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-0b66d906109d4799b4bf5adc58e8ea8c']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c8b89d400>, <tornado.concurrent.Future object at 0x1c8b8998d0>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-021b15cddf2d43eca3611a97eac931a4']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c8b8d22f0>, <tornado.concurrent.Future object at 0x1c8b899b00>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-648f34f2cc83473580f75e2b587e8d11']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c9620e8c8>, <tornado.concurrent.Future object at 0x1caca3fb00>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-ee9553bc36bd469cac8378b7e4f797ca']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c8b8af598>, <tornado.concurrent.Future object at 0x1c8b8d13c8>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-68ead2f3c6ab406c95cb8b687e1433ee']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c665ab598>, <tornado.concurrent.Future object at 0x1c8b8996a0>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-59299ef9202d45ef9b2dadcf380888c3']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c8b8af1e0>, <tornado.concurrent.Future object at 0x1c8b8a1f60>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-c6d97994bcd942e1ac1db4883379c23f']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c665c8950>, <tornado.concurrent.Future object at 0x1c665ca278>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-1faba7a288bb4f7d81a53a0cbf3d346e']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c665bda60>, <tornado.concurrent.Future object at 0x1caca3f978>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-620458dbd711491f9a90375bc1f4b56c']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c962302f0>, <tornado.concurrent.Future object at 0x1c1d85b080>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-ccbe4c232a6747c7b8c49ca08915a2b4']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c1d856048>, <tornado.concurrent.Future object at 0x1c1d8472b0>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-62995942315941c6b119340a74567de0']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c1d856950>, <tornado.concurrent.Future object at 0x1c8b8a1400>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-b846d4a46046400fa9fc22053014da58']
tornado.application - ERROR - Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x1c1d8330d0>, <tornado.concurrent.Future object at 0x1c8b8ac198>)
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/joblib.py", line 128, in callback_wrapper
    result = yield _wait([future])
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 2906, in _wait
    raise CancelledError(cancelled)
concurrent.futures._base.CancelledError: ['work_events-batch-d2edfb4506d34e689a551bff06f32967']
distributed.utils - ERROR - in <closed TCP>: TimeoutError: [Errno 60] Operation timed out: while trying to call remote method 'cancel'
Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/comm/tcp.py", line 170, in read
    n_frames = yield stream.read_bytes(8)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
tornado.iostream.StreamClosedError: Stream is closed

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/core.py", line 454, in send_recv_from_rpc
    result = yield send_recv(comm=comm, op=key, **kwargs)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/core.py", line 340, in send_recv
    response = yield comm.read()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/comm/tcp.py", line 184, in read
    convert_stream_closed_error(self, e)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/comm/tcp.py", line 120, in convert_stream_closed_error
    raise CommClosedError("in %s: %s: %s" % (obj, exc.__class__.__name__, exc))
distributed.comm.core.CommClosedError: in <closed TCP>: TimeoutError: [Errno 60] Operation timed out

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/utils.py", line 229, in f
    result[0] = yield make_coro()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/client.py", line 1602, in _cancel
    yield self.scheduler.cancel(keys=keys, client=self.id, force=force)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/Users/ckemere/anaconda3/lib/python3.6/site-packages/distributed/core.py", line 457, in send_recv_from_rpc
    % (e, key,))
distributed.comm.core.CommClosedError: in <closed TCP>: TimeoutError: [Errno 60] Operation timed out: while trying to call remote method 'cancel'










    






---------------------------------------------------------------------------
CancelledError                            Traceback (most recent call last)
~/anaconda3/lib/python3.6/site-packages/joblib/parallel.py in retrieve(self)
    700                 else:
--> 701                     self._output.extend(job.get())
    702 

~/anaconda3/lib/python3.6/site-packages/distributed/client.py in result(self, timeout)
    167         elif self.status == 'cancelled':
--> 168             raise result
    169         else:

CancelledError: work_events-batch-6407a9716f68431588a9af8b84f20215

During handling of the above exception, another exception occurred:

CommClosedError                           Traceback (most recent call last)
<ipython-input-6-b5f8ff536de8> in <module>()
     30 with parallel_backend('dask.distributed', scheduler_host='35.184.42.12:8786'):
     31     # Anything returned by work() can be stored:
---> 32     parallel_results = Parallel(n_jobs=n_jobs, verbose=1)(map(delayed(work_events), parallel_events))
     33 

~/anaconda3/lib/python3.6/site-packages/joblib/parallel.py in __call__(self, iterable)
    787                 # consumption.
    788                 self._iterating = False
--> 789             self.retrieve()
    790             # Make sure that we get a last message telling us we are done
    791             elapsed_time = time.time() - self._start_time

~/anaconda3/lib/python3.6/site-packages/joblib/parallel.py in retrieve(self)
    719                     # scheduling.
    720                     ensure_ready = self._managed_backend
--> 721                     backend.abort_everything(ensure_ready=ensure_ready)
    722 
    723                 if not isinstance(exception, TransportableException):

~/anaconda3/lib/python3.6/site-packages/distributed/joblib.py in abort_everything(self, ensure_ready)
    139         # Tell the client to cancel any task submitted via this instance
    140         # as joblib.Parallel will never access those results.
--> 141         self.client.cancel(self.futures)
    142         self.futures.clear()
    143 

~/anaconda3/lib/python3.6/site-packages/distributed/client.py in cancel(self, futures, asynchronous, force)
   1621         """
   1622         return self.sync(self._cancel, futures, asynchronous=asynchronous,
-> 1623                          force=force)
   1624 
   1625     @gen.coroutine

~/anaconda3/lib/python3.6/site-packages/distributed/client.py in sync(self, func, *args, **kwargs)
    559             return future
    560         else:
--> 561             return sync(self.loop, func, *args, **kwargs)
    562 
    563     def __str__(self):

~/anaconda3/lib/python3.6/site-packages/distributed/utils.py in sync(loop, func, *args, **kwargs)
    239         e.wait(1000000)
    240     if error[0]:
--> 241         six.reraise(*error[0])
    242     else:
    243         return result[0]

~/anaconda3/lib/python3.6/site-packages/six.py in reraise(tp, value, tb)
    684         if value.__traceback__ is not tb:
    685             raise value.with_traceback(tb)
--> 686         raise value
    687 
    688 else:

~/anaconda3/lib/python3.6/site-packages/distributed/utils.py in f()
    227             yield gen.moment
    228             thread_state.asynchronous = True
--> 229             result[0] = yield make_coro()
    230         except Exception as exc:
    231             logger.exception(exc)

~/anaconda3/lib/python3.6/site-packages/tornado/gen.py in run(self)
   1053 
   1054                     try:
-> 1055                         value = future.result()
   1056                     except Exception:
   1057                         self.had_exception = True

~/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py in result(self, timeout)
    236         if self._exc_info is not None:
    237             try:
--> 238                 raise_exc_info(self._exc_info)
    239             finally:
    240                 self = None

~/anaconda3/lib/python3.6/site-packages/tornado/util.py in raise_exc_info(exc_info)

~/anaconda3/lib/python3.6/site-packages/tornado/gen.py in run(self)
   1061                     if exc_info is not None:
   1062                         try:
-> 1063                             yielded = self.gen.throw(*exc_info)
   1064                         finally:
   1065                             # Break up a reference to itself

~/anaconda3/lib/python3.6/site-packages/distributed/client.py in _cancel(self, futures, force)
   1600     def _cancel(self, futures, force=False):
   1601         keys = list({tokey(f.key) for f in futures_of(futures)})
-> 1602         yield self.scheduler.cancel(keys=keys, client=self.id, force=force)
   1603         for k in keys:
   1604             st = self.futures.pop(k, None)

~/anaconda3/lib/python3.6/site-packages/tornado/gen.py in run(self)
   1053 
   1054                     try:
-> 1055                         value = future.result()
   1056                     except Exception:
   1057                         self.had_exception = True

~/anaconda3/lib/python3.6/site-packages/tornado/concurrent.py in result(self, timeout)
    236         if self._exc_info is not None:
    237             try:
--> 238                 raise_exc_info(self._exc_info)
    239             finally:
    240                 self = None

~/anaconda3/lib/python3.6/site-packages/tornado/util.py in raise_exc_info(exc_info)

~/anaconda3/lib/python3.6/site-packages/tornado/gen.py in run(self)
   1061                     if exc_info is not None:
   1062                         try:
-> 1063                             yielded = self.gen.throw(*exc_info)
   1064                         finally:
   1065                             # Break up a reference to itself

~/anaconda3/lib/python3.6/site-packages/distributed/core.py in send_recv_from_rpc(**kwargs)
    455             except (RPCClosed, CommClosedError) as e:
    456                 raise e.__class__("%s: while trying to call remote method %r"
--> 457                                   % (e, key,))
    458 
    459             self.comms[comm] = True  # mark as open

CommClosedError: in <closed TCP>: TimeoutError: [Errno 60] Operation timed out: while trying to call remote method 'cancel'
















In [ ]:



    


# standardize parallel results
bdries_ = [aux_data[session][segment]['PBEs'].n_epochs for session, segment in zip(sessions, segments) ]
bdries = np.cumsum(np.insert(bdries_,0,0))
bdries

sessions_ = np.array([result[0] for result in parallel_results])
segments_ = np.array([result[1] for result in parallel_results])
idx = [result[2] for result in parallel_results]

scores_bayes_evt = np.array([float(result[3]) for result in parallel_results])
scores_bayes_shuffled_evt = np.array([result[4].squeeze() for result in parallel_results])
scores_bayes_percentile_evt = np.array([float(result[5]) for result in parallel_results])

results = {}
for nn in range(len(bdries)-1):
    session = np.unique(sessions_[bdries[nn]:bdries[nn+1]])
    if len(session) > 1:
        raise ValueError("parallel results in different format / order than expected!")
    session = session[0]
    segment = np.unique(segments_[bdries[nn]:bdries[nn+1]])
    if len(segment) > 1:
        raise ValueError("parallel results in different format / order than expected!")
    segment = segment[0]
    try:
        results[session][segment]['scores_bayes'] = scores_bayes_evt[bdries[nn]:bdries[nn+1]]
    except KeyError:
        try:
            results[session][segment] = dict()
            results[session][segment]['scores_bayes'] = scores_bayes_evt[bdries[nn]:bdries[nn+1]]
        except KeyError:
            results[session] = dict()
            results[session][segment] = dict()
            results[session][segment]['scores_bayes'] = scores_bayes_evt[bdries[nn]:bdries[nn+1]]

    results[session][segment]['scores_bayes_shuffled'] = scores_bayes_shuffled_evt[bdries[nn]:bdries[nn+1]]
    results[session][segment]['scores_bayes_percentile'] = scores_bayes_percentile_evt[bdries[nn]:bdries[nn+1]]

print('done packing results')

Save results to disk





In [ ]:



    


jar = nel.ResultsContainer(results=results, description='gor01 and vvp01 speed restricted results for best 20 candidate sessions')
jar.save_pkl('score_bayes_all_sessions.nel')

Content source: ckemere/CloudShuffles

Similar notebooks:

notebook.community | gallery | about