In [1]:
INPUT_DIR='../../input/nodules/'
OUTPUT_DIR='../../output/lung-cancer/04/'
MODEL_FILE='../../output/lung-cancer/03-ok/tf-checkpoint-best9615'
IMAGE_DIMS = (50,50,50,1)
In [2]:
%matplotlib inline
import numpy as np
import pandas as pd
import h5py
import matplotlib.pyplot as plt
import sklearn
import os
import glob
from modules.logging import logger
import modules.utils as utils
from modules.utils import Timer
import modules.logging
import modules.cnn as cnn
import modules.ctscan as ctscan
In [3]:
logger.info('Prepare CNN for training')
network = cnn.net_nodule3d_swethasubramanian(IMAGE_DIMS)
model = cnn.prepare_cnn_model(network, OUTPUT_DIR, model_file=MODEL_FILE)
2017-03-28 00:36:58,355 INFO Prepare CNN for training
2017-03-28 00:36:58,575 INFO Prepare CNN
2017-03-28 00:36:58,588 INFO Preparing output dir
2017-03-28 00:36:58,589 INFO Initializing network...
---------------------------------------------------------------------------
InternalError Traceback (most recent call last)
/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1021 try:
-> 1022 return fn(*args)
1023 except errors.OpError as e:
/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata)
1003 feed_dict, fetch_list, target_list,
-> 1004 status, run_metadata)
1005
/usr/lib/python3.5/contextlib.py in __exit__(self, type, value, traceback)
65 try:
---> 66 next(self.gen)
67 except StopIteration:
/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/errors_impl.py in raise_exception_on_not_ok_status()
468 compat.as_text(pywrap_tensorflow.TF_Message(status)),
--> 469 pywrap_tensorflow.TF_GetCode(status))
470 finally:
InternalError: Dst tensor is not initialized.
[[Node: Adam/zeros_13 = Const[dtype=DT_FLOAT, value=Tensor<type: float shape: [1500000,64] values: [0 0 0]...>, _device="/job:localhost/replica:0/task:0/gpu:0"]()]]
During handling of the above exception, another exception occurred:
InternalError Traceback (most recent call last)
<ipython-input-3-d2edbf3f5522> in <module>()
1 logger.info('Prepare CNN for training')
2 network = cnn.net_nodule3d_swethasubramanian(IMAGE_DIMS)
----> 3 model = cnn.prepare_cnn_model(network, OUTPUT_DIR, model_file=MODEL_FILE)
/notebooks/datascience-snippets/kaggle-lung-cancer/modules/cnn.py in prepare_cnn_model(network, output_dir, model_file)
125 tensorboard_dir=dir_tflogs,
126 checkpoint_path=dir_checkpoints,
--> 127 best_checkpoint_path=dir_checkpoint_best)
128
129 if(model_file!=None):
/usr/local/lib/python3.5/dist-packages/tflearn/models/dnn.py in __init__(self, network, clip_gradients, tensorboard_verbose, tensorboard_dir, checkpoint_path, best_checkpoint_path, max_checkpoints, session, best_val_accuracy)
62 max_checkpoints=max_checkpoints,
63 session=session,
---> 64 best_val_accuracy=best_val_accuracy)
65 self.session = self.trainer.session
66
/usr/local/lib/python3.5/dist-packages/tflearn/helpers/trainer.py in __init__(self, train_ops, graph, clip_gradients, tensorboard_dir, tensorboard_verbose, checkpoint_path, best_checkpoint_path, max_checkpoints, keep_checkpoint_every_n_hours, random_seed, session, best_val_accuracy)
165 except Exception as e:
166 init = tf.initialize_all_variables()
--> 167 self.session.run(init)
168
169 def fit(self, feed_dicts, n_epoch=10, val_feed_dicts=None, show_metric=False,
/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata)
765 try:
766 result = self._run(None, fetches, feed_dict, options_ptr,
--> 767 run_metadata_ptr)
768 if run_metadata:
769 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
963 if final_fetches or final_targets:
964 results = self._do_run(handle, final_targets, final_fetches,
--> 965 feed_dict_string, options, run_metadata)
966 else:
967 results = []
/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
1013 if handle is None:
1014 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
-> 1015 target_list, options, run_metadata)
1016 else:
1017 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1033 except KeyError:
1034 pass
-> 1035 raise type(e)(node_def, op, message)
1036
1037 def _extend_graph(self):
InternalError: Dst tensor is not initialized.
[[Node: Adam/zeros_13 = Const[dtype=DT_FLOAT, value=Tensor<type: float shape: [1500000,64] values: [0 0 0]...>, _device="/job:localhost/replica:0/task:0/gpu:0"]()]]
Caused by op 'Adam/zeros_13', defined at:
File "/usr/lib/python3.5/runpy.py", line 184, in _run_module_as_main
"__main__", mod_spec)
File "/usr/lib/python3.5/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.5/dist-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/usr/local/lib/python3.5/dist-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/usr/local/lib/python3.5/dist-packages/ipykernel/kernelapp.py", line 474, in start
ioloop.IOLoop.instance().start()
File "/usr/local/lib/python3.5/dist-packages/zmq/eventloop/ioloop.py", line 177, in start
super(ZMQIOLoop, self).start()
File "/usr/local/lib/python3.5/dist-packages/tornado/ioloop.py", line 887, in start
handler_func(fd_obj, events)
File "/usr/local/lib/python3.5/dist-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/usr/local/lib/python3.5/dist-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/usr/local/lib/python3.5/dist-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/usr/local/lib/python3.5/dist-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/usr/local/lib/python3.5/dist-packages/ipykernel/kernelbase.py", line 390, in execute_request
user_expressions, allow_stdin)
File "/usr/local/lib/python3.5/dist-packages/ipykernel/ipkernel.py", line 196, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/usr/local/lib/python3.5/dist-packages/ipykernel/zmqshell.py", line 501, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/IPython/core/interactiveshell.py", line 2717, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/usr/local/lib/python3.5/dist-packages/IPython/core/interactiveshell.py", line 2821, in run_ast_nodes
if self.run_code(code, result):
File "/usr/local/lib/python3.5/dist-packages/IPython/core/interactiveshell.py", line 2881, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-3-d2edbf3f5522>", line 3, in <module>
model = cnn.prepare_cnn_model(network, OUTPUT_DIR, model_file=MODEL_FILE)
File "/notebooks/datascience-snippets/kaggle-lung-cancer/modules/cnn.py", line 127, in prepare_cnn_model
best_checkpoint_path=dir_checkpoint_best)
File "/usr/local/lib/python3.5/dist-packages/tflearn/models/dnn.py", line 64, in __init__
best_val_accuracy=best_val_accuracy)
File "/usr/local/lib/python3.5/dist-packages/tflearn/helpers/trainer.py", line 131, in __init__
clip_gradients)
File "/usr/local/lib/python3.5/dist-packages/tflearn/helpers/trainer.py", line 664, in initialize_training_ops
name="apply_grad_op_" + str(i))
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/optimizer.py", line 412, in apply_gradients
self._create_slots(var_list)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/adam.py", line 119, in _create_slots
self._zeros_slot(v, "m", self._name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/optimizer.py", line 656, in _zeros_slot
named_slots[var] = slot_creator.create_zeros_slot(var, op_name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/slot_creator.py", line 121, in create_zeros_slot
val = array_ops.zeros(primary.get_shape().as_list(), dtype=dtype)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/array_ops.py", line 1370, in zeros
output = constant(zero, shape=shape, dtype=dtype, name=name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/constant_op.py", line 169, in constant
attrs={"value": tensor_value, "dtype": dtype_value}, name=name).outputs[0]
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/ops.py", line 2395, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/ops.py", line 1264, in __init__
self._traceback = _extract_stack()
InternalError (see above for traceback): Dst tensor is not initialized.
[[Node: Adam/zeros_13 = Const[dtype=DT_FLOAT, value=Tensor<type: float shape: [1500000,64] values: [0 0 0]...>, _device="/job:localhost/replica:0/task:0/gpu:0"]()]]
In [ ]:
logger.info('Evaluate dataset')
cnn.evaluate_dataset(INPUT_DIR + 'nodules-test.h5', model, batch_size=12, confusion_matrix=True)
In [ ]:
Content source: flaviostutz/datascience-snippets
Similar notebooks: