In [ ]:
%ls
In [ ]:
# run single train
layer1 = 4
layer2 = 4
fully = 512
%run ./convolutional.py --config=conv4.cfg --conv1_depth=$layer1 --conv2_depth=$layer2 --fc_depth=$fully
In [1]:
# run all trains with filter size 4
neurons = [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
fully = [512, 1024]
for i in neurons:
for j in neurons:
for k in fully:
print "training: " + str(i) + ", " + str(j) + ", " + str(k)
%run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k
training: 2, 2, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 2, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 4, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 4, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 8, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 8, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 16, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 16, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 32, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 32, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 64, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 64, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 128, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 128, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 256, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 256, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 512, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 512, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 1024, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 2, 1024, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 2, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 2, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 4, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 4, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 8, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 8, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 16, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 16, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 32, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 32, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 64, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 64, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 128, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 128, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 256, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 256, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 512, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 512, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 1024, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 4, 1024, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 2, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 2, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 4, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 4, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 8, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 8, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 16, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 16, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 32, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 32, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 64, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 64, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 128, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 128, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 256, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 256, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 512, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 512, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 1024, 512
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 8, 1024, 1024
Using settings in ./conv4.cfg
train with /gpu:0, evaluate with /gpu:3
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 2, 512
Using settings in ./conv4.cfg
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 2, 1024
Using settings in ./conv4.cfg
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 4, 512
Using settings in ./conv4.cfg
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 4, 1024
Using settings in ./conv4.cfg
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 8, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 8, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 16, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 16, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 32, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 32, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 64, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 64, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 128, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 128, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 256, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 256, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 512, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 512, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 1024, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 16, 1024, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 2, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 2, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 4, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 4, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 8, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 8, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 16, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 16, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 32, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 32, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 64, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 64, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 128, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 128, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 256, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 256, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 512, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 512, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 1024, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 32, 1024, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 2, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 2, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 4, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 4, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 8, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 8, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 16, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 16, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 32, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 32, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 64, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 64, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 128, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 128, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 256, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 256, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 512, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 512, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 1024, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 64, 1024, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 2, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 2, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 4, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 4, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 8, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 8, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 16, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 16, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 32, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 32, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 64, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 64, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 128, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 128, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 256, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 256, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 512, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 512, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 1024, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 128, 1024, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 2, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 2, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 4, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 4, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 8, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 8, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 16, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 16, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 32, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 32, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 64, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 64, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 128, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 128, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 256, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 256, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 512, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 512, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 1024, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 256, 1024, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 512, 2, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 512, 2, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 512, 4, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 512, 4, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 512, 8, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 512, 8, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 512, 16, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 512, 16, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 512, 32, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 512, 32, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 512, 64, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 512, 64, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 512, 128, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 512, 128, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 512, 256, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
training: 512, 256, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
360 _, l, lr, predictions = sess.run(
361 [optimizer, loss, learning_rate, train_prediction],
--> 362 feed_dict=feed_dict)
363
364 if step % EVAL_FREQUENCY == 0:
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[64,512,14,14]
[[Node: gradients_175/MaxPool_700_grad/MaxPoolGrad = MaxPoolGrad[T=DT_FLOAT, data_format="NHWC", ksize=[1, 2, 2, 1], padding="SAME", strides=[1, 2, 2, 1], _device="/job:localhost/replica:0/task:0/gpu:0"](Relu_1050, MaxPool_700, gradients_175/Conv2D_701_grad/tuple/control_dependency)]]
Caused by op u'gradients_175/MaxPool_700_grad/MaxPoolGrad', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 409, in <module>
tf.app.run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 321, in main
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(loss, global_step=batch)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 193, in minimize
grad_loss=grad_loss)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 250, in compute_gradients
colocate_gradients_with_ops=colocate_gradients_with_ops)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gradients.py", line 484, in gradients
in_grads = _AsList(grad_fn(op, *out_grads))
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/nn_grad.py", line 361, in _MaxPoolGrad
data_format=op.get_attr("data_format"))
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_nn_ops.py", line 1222, in _max_pool_grad
data_format=data_format, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
...which was originally created as op u'MaxPool_700', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
[elided 28 identical lines from previous traceback]
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 299, in main
logits = model(vars, train_data_node, True)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 175, in model
padding='SAME')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/nn_ops.py", line 649, in max_pool
name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_nn_ops.py", line 1123, in _max_pool
data_format=data_format, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 512, 512, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
InternalErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
InternalError: Dst tensor is not initialized.
[[Node: zeros_1076 = Const[dtype=DT_FLOAT, value=Tensor<type: float shape: [50176,1024] values: 0 0 0...>, _device="/job:localhost/replica:0/task:0/gpu:0"]()]]
Caused by op u'zeros_1076', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 409, in <module>
tf.app.run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 321, in main
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(loss, global_step=batch)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 195, in minimize
name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 297, in apply_gradients
self._create_slots(var_list)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/momentum.py", line 51, in _create_slots
self._zeros_slot(v, "momentum", self._name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 491, in _zeros_slot
named_slots[var] = slot_creator.create_zeros_slot(var, op_name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 106, in create_zeros_slot
val = array_ops.zeros(primary.get_shape().as_list(), dtype=dtype)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/array_ops.py", line 1122, in zeros
output = constant(0, shape=shape, dtype=dtype, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/constant_op.py", line 167, in constant
attrs={"value": tensor_value, "dtype": dtype_value}, name=name).outputs[0]
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 512, 512, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[50176,1024]
[[Node: truncated_normal_478/mul = Mul[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](truncated_normal_478/TruncatedNormal, truncated_normal_478/stddev)]]
Caused by op u'truncated_normal_478/mul', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 409, in <module>
tf.app.run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 296, in main
vars = init_vars(FLAGS.filter_size, FLAGS.conv1_depth, FLAGS.conv2_depth, FLAGS.fc_depth)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 133, in init_vars
seed=SEED))
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/random_ops.py", line 179, in truncated_normal
mul = rnd * stddev_tensor
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/math_ops.py", line 754, in binary_op_wrapper
return func(x, y, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/math_ops.py", line 903, in _mul_dispatch
return gen_math_ops.mul(x, y, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_math_ops.py", line 1427, in mul
result = _op_def_lib.apply_op("Mul", x=x, y=y, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 512, 1024, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[4,4,1,4]
[[Node: Variable_270/Momentum/Assign = Assign[T=DT_FLOAT, _class=["loc:@Variable_270"], use_locking=true, validate_shape=true, _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_270/Momentum, zeros_271)]]
Caused by op u'Variable_270/Momentum/Assign', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 406, in <module>
print(args)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 321, in main
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(loss, global_step=batch)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 195, in minimize
name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 297, in apply_gradients
self._create_slots(var_list)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/momentum.py", line 51, in _create_slots
self._zeros_slot(v, "momentum", self._name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 491, in _zeros_slot
named_slots[var] = slot_creator.create_zeros_slot(var, op_name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 108, in create_zeros_slot
colocate_with_primary=colocate_with_primary)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 86, in create_slot
return _create_slot_var(primary, val, scope)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 50, in _create_slot_var
slot = variables.Variable(val, name=scope, trainable=False)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 211, in __init__
dtype=dtype)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 313, in _init_from_args
validate_shape=validate_shape).op
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_state_ops.py", line 45, in assign
use_locking=use_locking, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 512, 1024, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[50176,1024]
[[Node: Variable_355/Momentum/Assign = Assign[T=DT_FLOAT, _class=["loc:@Variable_355"], use_locking=true, validate_shape=true, _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_355/Momentum, zeros_356)]]
Caused by op u'Variable_355/Momentum/Assign', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 406, in <module>
print(args)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 321, in main
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(loss, global_step=batch)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 195, in minimize
name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 297, in apply_gradients
self._create_slots(var_list)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/momentum.py", line 51, in _create_slots
self._zeros_slot(v, "momentum", self._name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 491, in _zeros_slot
named_slots[var] = slot_creator.create_zeros_slot(var, op_name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 108, in create_zeros_slot
colocate_with_primary=colocate_with_primary)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 86, in create_slot
return _create_slot_var(primary, val, scope)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 50, in _create_slot_var
slot = variables.Variable(val, name=scope, trainable=False)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 211, in __init__
dtype=dtype)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 313, in _init_from_args
validate_shape=validate_shape).op
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_state_ops.py", line 45, in assign
use_locking=use_locking, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 2, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[25088,1024]
[[Node: truncated_normal_70/mul = Mul[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](truncated_normal_70/TruncatedNormal, truncated_normal_70/stddev)]]
Caused by op u'truncated_normal_70/mul', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 406, in <module>
print(args)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 296, in main
vars = init_vars(FLAGS.filter_size, FLAGS.conv1_depth, FLAGS.conv2_depth, FLAGS.fc_depth)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 133, in init_vars
seed=SEED))
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/random_ops.py", line 179, in truncated_normal
mul = rnd * stddev_tensor
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/math_ops.py", line 754, in binary_op_wrapper
return func(x, y, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/math_ops.py", line 903, in _mul_dispatch
return gen_math_ops.mul(x, y, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_math_ops.py", line 1427, in mul
result = _op_def_lib.apply_op("Mul", x=x, y=y, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 2, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[50176,512]
[[Node: Variable_1426/Momentum/Assign = Assign[T=DT_FLOAT, _class=["loc:@Variable_1426"], use_locking=true, validate_shape=true, _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_1426/Momentum, zeros_1427)]]
Caused by op u'Variable_1426/Momentum/Assign', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 409, in <module>
tf.app.run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 321, in main
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(loss, global_step=batch)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 195, in minimize
name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 297, in apply_gradients
self._create_slots(var_list)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/momentum.py", line 51, in _create_slots
self._zeros_slot(v, "momentum", self._name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 491, in _zeros_slot
named_slots[var] = slot_creator.create_zeros_slot(var, op_name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 108, in create_zeros_slot
colocate_with_primary=colocate_with_primary)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 86, in create_slot
return _create_slot_var(primary, val, scope)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 50, in _create_slot_var
slot = variables.Variable(val, name=scope, trainable=False)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 211, in __init__
dtype=dtype)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 313, in _init_from_args
validate_shape=validate_shape).op
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_state_ops.py", line 45, in assign
use_locking=use_locking, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 4, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[6272,1024]
[[Node: truncated_normal_694/TruncatedNormal = TruncatedNormal[T=DT_INT32, dtype=DT_FLOAT, seed=87654321, seed2=66478, _device="/job:localhost/replica:0/task:0/gpu:0"](truncated_normal_694/shape)]]
Caused by op u'truncated_normal_694/TruncatedNormal', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 409, in <module>
tf.app.run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 296, in main
vars = init_vars(FLAGS.filter_size, FLAGS.conv1_depth, FLAGS.conv2_depth, FLAGS.fc_depth)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 133, in init_vars
seed=SEED))
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/random_ops.py", line 178, in truncated_normal
seed2=seed2)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_random_ops.py", line 291, in _truncated_normal
seed=seed, seed2=seed2, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 4, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[50176,1024]
[[Node: Variable_715/Momentum/Assign = Assign[T=DT_FLOAT, _class=["loc:@Variable_715"], use_locking=true, validate_shape=true, _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_715/Momentum, zeros_716)]]
Caused by op u'Variable_715/Momentum/Assign', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 407, in <module>
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 321, in main
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(loss, global_step=batch)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 195, in minimize
name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 297, in apply_gradients
self._create_slots(var_list)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/momentum.py", line 51, in _create_slots
self._zeros_slot(v, "momentum", self._name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 491, in _zeros_slot
named_slots[var] = slot_creator.create_zeros_slot(var, op_name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 108, in create_zeros_slot
colocate_with_primary=colocate_with_primary)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 86, in create_slot
return _create_slot_var(primary, val, scope)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 50, in _create_slot_var
slot = variables.Variable(val, name=scope, trainable=False)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 211, in __init__
dtype=dtype)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 313, in _init_from_args
validate_shape=validate_shape).op
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_state_ops.py", line 45, in assign
use_locking=use_locking, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 8, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[50176,1024]
[[Node: Variable_895/Momentum/Assign = Assign[T=DT_FLOAT, _class=["loc:@Variable_895"], use_locking=true, validate_shape=true, _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_895/Momentum, zeros_896)]]
Caused by op u'Variable_895/Momentum/Assign', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 407, in <module>
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 321, in main
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(loss, global_step=batch)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 195, in minimize
name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 297, in apply_gradients
self._create_slots(var_list)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/momentum.py", line 51, in _create_slots
self._zeros_slot(v, "momentum", self._name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 491, in _zeros_slot
named_slots[var] = slot_creator.create_zeros_slot(var, op_name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 108, in create_zeros_slot
colocate_with_primary=colocate_with_primary)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 86, in create_slot
return _create_slot_var(primary, val, scope)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 50, in _create_slot_var
slot = variables.Variable(val, name=scope, trainable=False)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 211, in __init__
dtype=dtype)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 313, in _init_from_args
validate_shape=validate_shape).op
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_state_ops.py", line 45, in assign
use_locking=use_locking, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 8, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[50176,1024]
[[Node: Variable_1435/Momentum/Assign = Assign[T=DT_FLOAT, _class=["loc:@Variable_1435"], use_locking=true, validate_shape=true, _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_1435/Momentum, zeros_1436)]]
Caused by op u'Variable_1435/Momentum/Assign', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 409, in <module>
tf.app.run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 321, in main
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(loss, global_step=batch)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 195, in minimize
name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 297, in apply_gradients
self._create_slots(var_list)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/momentum.py", line 51, in _create_slots
self._zeros_slot(v, "momentum", self._name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 491, in _zeros_slot
named_slots[var] = slot_creator.create_zeros_slot(var, op_name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 108, in create_zeros_slot
colocate_with_primary=colocate_with_primary)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 86, in create_slot
return _create_slot_var(primary, val, scope)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 50, in _create_slot_var
slot = variables.Variable(val, name=scope, trainable=False)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 211, in __init__
dtype=dtype)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 313, in _init_from_args
validate_shape=validate_shape).op
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_state_ops.py", line 45, in assign
use_locking=use_locking, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 16, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[50176,1024]
[[Node: truncated_normal_478/mul = Mul[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](truncated_normal_478/TruncatedNormal, truncated_normal_478/stddev)]]
Caused by op u'truncated_normal_478/mul', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 409, in <module>
tf.app.run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 296, in main
vars = init_vars(FLAGS.filter_size, FLAGS.conv1_depth, FLAGS.conv2_depth, FLAGS.fc_depth)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 133, in init_vars
seed=SEED))
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/random_ops.py", line 179, in truncated_normal
mul = rnd * stddev_tensor
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/math_ops.py", line 754, in binary_op_wrapper
return func(x, y, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/math_ops.py", line 903, in _mul_dispatch
return gen_math_ops.mul(x, y, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_math_ops.py", line 1427, in mul
result = _op_def_lib.apply_op("Mul", x=x, y=y, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 16, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[50176,1024]
[[Node: truncated_normal_78/TruncatedNormal = TruncatedNormal[T=DT_INT32, dtype=DT_FLOAT, seed=87654321, seed2=66478, _device="/job:localhost/replica:0/task:0/gpu:0"](truncated_normal_78/shape)]]
Caused by op u'truncated_normal_78/TruncatedNormal', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 406, in <module>
print(args)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 296, in main
vars = init_vars(FLAGS.filter_size, FLAGS.conv1_depth, FLAGS.conv2_depth, FLAGS.fc_depth)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 133, in init_vars
seed=SEED))
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/random_ops.py", line 178, in truncated_normal
seed2=seed2)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_random_ops.py", line 291, in _truncated_normal
seed=seed, seed2=seed2, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 32, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[512,10]
[[Node: Variable_474/Momentum/Assign = Assign[T=DT_FLOAT, _class=["loc:@Variable_474"], use_locking=true, validate_shape=true, _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_474/Momentum, zeros_475)]]
Caused by op u'Variable_474/Momentum/Assign', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 406, in <module>
print(args)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 321, in main
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(loss, global_step=batch)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 195, in minimize
name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 297, in apply_gradients
self._create_slots(var_list)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/momentum.py", line 51, in _create_slots
self._zeros_slot(v, "momentum", self._name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 491, in _zeros_slot
named_slots[var] = slot_creator.create_zeros_slot(var, op_name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 108, in create_zeros_slot
colocate_with_primary=colocate_with_primary)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 86, in create_slot
return _create_slot_var(primary, val, scope)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 50, in _create_slot_var
slot = variables.Variable(val, name=scope, trainable=False)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 211, in __init__
dtype=dtype)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 313, in _init_from_args
validate_shape=validate_shape).op
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_state_ops.py", line 45, in assign
use_locking=use_locking, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 32, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[50176,1024]
[[Node: truncated_normal_158/TruncatedNormal = TruncatedNormal[T=DT_INT32, dtype=DT_FLOAT, seed=87654321, seed2=66478, _device="/job:localhost/replica:0/task:0/gpu:0"](truncated_normal_158/shape)]]
Caused by op u'truncated_normal_158/TruncatedNormal', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 406, in <module>
print(args)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 296, in main
vars = init_vars(FLAGS.filter_size, FLAGS.conv1_depth, FLAGS.conv2_depth, FLAGS.fc_depth)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 133, in init_vars
seed=SEED))
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/random_ops.py", line 178, in truncated_normal
seed2=seed2)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_random_ops.py", line 291, in _truncated_normal
seed=seed, seed2=seed2, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 64, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[50176,1024]
[[Node: Variable_895/Momentum/Assign = Assign[T=DT_FLOAT, _class=["loc:@Variable_895"], use_locking=true, validate_shape=true, _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_895/Momentum, zeros_896)]]
Caused by op u'Variable_895/Momentum/Assign', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 407, in <module>
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 321, in main
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(loss, global_step=batch)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 195, in minimize
name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 297, in apply_gradients
self._create_slots(var_list)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/momentum.py", line 51, in _create_slots
self._zeros_slot(v, "momentum", self._name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 491, in _zeros_slot
named_slots[var] = slot_creator.create_zeros_slot(var, op_name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 108, in create_zeros_slot
colocate_with_primary=colocate_with_primary)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 86, in create_slot
return _create_slot_var(primary, val, scope)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 50, in _create_slot_var
slot = variables.Variable(val, name=scope, trainable=False)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 211, in __init__
dtype=dtype)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 313, in _init_from_args
validate_shape=validate_shape).op
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_state_ops.py", line 45, in assign
use_locking=use_locking, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 64, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[12544,1024]
[[Node: Variable_319/Momentum/Assign = Assign[T=DT_FLOAT, _class=["loc:@Variable_319"], use_locking=true, validate_shape=true, _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_319/Momentum, zeros_320)]]
Caused by op u'Variable_319/Momentum/Assign', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 406, in <module>
print(args)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 321, in main
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(loss, global_step=batch)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 195, in minimize
name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 297, in apply_gradients
self._create_slots(var_list)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/momentum.py", line 51, in _create_slots
self._zeros_slot(v, "momentum", self._name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 491, in _zeros_slot
named_slots[var] = slot_creator.create_zeros_slot(var, op_name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 108, in create_zeros_slot
colocate_with_primary=colocate_with_primary)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 86, in create_slot
return _create_slot_var(primary, val, scope)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 50, in _create_slot_var
slot = variables.Variable(val, name=scope, trainable=False)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 211, in __init__
dtype=dtype)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 313, in _init_from_args
validate_shape=validate_shape).op
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_state_ops.py", line 45, in assign
use_locking=use_locking, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 128, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[784,512]
[[Node: truncated_normal_346/TruncatedNormal = TruncatedNormal[T=DT_INT32, dtype=DT_FLOAT, seed=87654321, seed2=66478, _device="/job:localhost/replica:0/task:0/gpu:0"](truncated_normal_346/shape)]]
Caused by op u'truncated_normal_346/TruncatedNormal', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 407, in <module>
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 296, in main
vars = init_vars(FLAGS.filter_size, FLAGS.conv1_depth, FLAGS.conv2_depth, FLAGS.fc_depth)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 133, in init_vars
seed=SEED))
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/random_ops.py", line 178, in truncated_normal
seed2=seed2)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_random_ops.py", line 291, in _truncated_normal
seed=seed, seed2=seed2, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 128, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[50176,1024]
[[Node: truncated_normal_718/TruncatedNormal = TruncatedNormal[T=DT_INT32, dtype=DT_FLOAT, seed=87654321, seed2=66478, _device="/job:localhost/replica:0/task:0/gpu:0"](truncated_normal_718/shape)]]
Caused by op u'truncated_normal_718/TruncatedNormal', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 409, in <module>
tf.app.run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 296, in main
vars = init_vars(FLAGS.filter_size, FLAGS.conv1_depth, FLAGS.conv2_depth, FLAGS.fc_depth)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 133, in init_vars
seed=SEED))
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/random_ops.py", line 178, in truncated_normal
seed2=seed2)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_random_ops.py", line 291, in _truncated_normal
seed=seed, seed2=seed2, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 256, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[50176,512]
[[Node: Variable_346/Momentum/Assign = Assign[T=DT_FLOAT, _class=["loc:@Variable_346"], use_locking=true, validate_shape=true, _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_346/Momentum, zeros_347)]]
Caused by op u'Variable_346/Momentum/Assign', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 406, in <module>
print(args)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 321, in main
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(loss, global_step=batch)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 195, in minimize
name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 297, in apply_gradients
self._create_slots(var_list)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/momentum.py", line 51, in _create_slots
self._zeros_slot(v, "momentum", self._name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 491, in _zeros_slot
named_slots[var] = slot_creator.create_zeros_slot(var, op_name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 108, in create_zeros_slot
colocate_with_primary=colocate_with_primary)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 86, in create_slot
return _create_slot_var(primary, val, scope)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 50, in _create_slot_var
slot = variables.Variable(val, name=scope, trainable=False)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 211, in __init__
dtype=dtype)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 313, in _init_from_args
validate_shape=validate_shape).op
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_state_ops.py", line 45, in assign
use_locking=use_locking, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 256, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[25088,512]
[[Node: Variable_508/Momentum/Assign = Assign[T=DT_FLOAT, _class=["loc:@Variable_508"], use_locking=true, validate_shape=true, _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_508/Momentum, zeros_509)]]
Caused by op u'Variable_508/Momentum/Assign', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 406, in <module>
print(args)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 321, in main
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(loss, global_step=batch)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 195, in minimize
name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 297, in apply_gradients
self._create_slots(var_list)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/momentum.py", line 51, in _create_slots
self._zeros_slot(v, "momentum", self._name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 491, in _zeros_slot
named_slots[var] = slot_creator.create_zeros_slot(var, op_name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 108, in create_zeros_slot
colocate_with_primary=colocate_with_primary)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 86, in create_slot
return _create_slot_var(primary, val, scope)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 50, in _create_slot_var
slot = variables.Variable(val, name=scope, trainable=False)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 211, in __init__
dtype=dtype)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 313, in _init_from_args
validate_shape=validate_shape).op
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_state_ops.py", line 45, in assign
use_locking=use_locking, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 512, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[50176,1024]
[[Node: Variable_1435/Momentum/Assign = Assign[T=DT_FLOAT, _class=["loc:@Variable_1435"], use_locking=true, validate_shape=true, _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_1435/Momentum, zeros_1436)]]
Caused by op u'Variable_1435/Momentum/Assign', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 409, in <module>
tf.app.run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 321, in main
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(loss, global_step=batch)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 195, in minimize
name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 297, in apply_gradients
self._create_slots(var_list)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/momentum.py", line 51, in _create_slots
self._zeros_slot(v, "momentum", self._name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 491, in _zeros_slot
named_slots[var] = slot_creator.create_zeros_slot(var, op_name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 108, in create_zeros_slot
colocate_with_primary=colocate_with_primary)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 86, in create_slot
return _create_slot_var(primary, val, scope)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 50, in _create_slot_var
slot = variables.Variable(val, name=scope, trainable=False)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 211, in __init__
dtype=dtype)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 313, in _init_from_args
validate_shape=validate_shape).op
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_state_ops.py", line 45, in assign
use_locking=use_locking, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 512, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[50176,1024]
[[Node: truncated_normal_318/TruncatedNormal = TruncatedNormal[T=DT_INT32, dtype=DT_FLOAT, seed=87654321, seed2=66478, _device="/job:localhost/replica:0/task:0/gpu:0"](truncated_normal_318/shape)]]
Caused by op u'truncated_normal_318/TruncatedNormal', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 407, in <module>
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 296, in main
vars = init_vars(FLAGS.filter_size, FLAGS.conv1_depth, FLAGS.conv2_depth, FLAGS.fc_depth)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 133, in init_vars
seed=SEED))
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/random_ops.py", line 178, in truncated_normal
seed2=seed2)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_random_ops.py", line 291, in _truncated_normal
seed=seed, seed2=seed2, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 1024, 512
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[50176,1024]
[[Node: Variable_175/Momentum/Assign = Assign[T=DT_FLOAT, _class=["loc:@Variable_175"], use_locking=true, validate_shape=true, _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_175/Momentum, zeros_176)]]
Caused by op u'Variable_175/Momentum/Assign', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 406, in <module>
print(args)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 321, in main
optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(loss, global_step=batch)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 195, in minimize
name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 297, in apply_gradients
self._create_slots(var_list)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/momentum.py", line 51, in _create_slots
self._zeros_slot(v, "momentum", self._name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 491, in _zeros_slot
named_slots[var] = slot_creator.create_zeros_slot(var, op_name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 108, in create_zeros_slot
colocate_with_primary=colocate_with_primary)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 86, in create_slot
return _create_slot_var(primary, val, scope)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/slot_creator.py", line 50, in _create_slot_var
slot = variables.Variable(val, name=scope, trainable=False)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 211, in __init__
dtype=dtype)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 313, in _init_from_args
validate_shape=validate_shape).op
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_state_ops.py", line 45, in assign
use_locking=use_locking, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
training: 1024, 1024, 1024
Logging test accuracy at ../log/mnist-conv_fiter4.csv
ResourceExhaustedErrorTraceback (most recent call last)
/home/webofthink/tf-hpo/src/convolutional.py in <module>()
407
408 if __name__ == '__main__':
--> 409 tf.app.run()
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.pyc in run(main)
28 f._parse_flags()
29 main = main or sys.modules['__main__'].main
---> 30 sys.exit(main(sys.argv))
/home/webofthink/tf-hpo/src/convolutional.py in main(argv)
335 # Run all the initializers to prepare the trainable parameters.
336 init = tf.initialize_all_variables()
--> 337 sess.run(init)
338 print_out('Initialized!')
339
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
380 try:
381 result = self._run(None, fetches, feed_dict, options_ptr,
--> 382 run_metadata_ptr)
383 if run_metadata:
384 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
653 movers = self._update_with_movers(feed_dict_string, feed_map)
654 results = self._do_run(handle, target_list, unique_fetches,
--> 655 feed_dict_string, options, run_metadata)
656
657 # User may have fetched the same tensor multiple times, but we
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
721 if handle is None:
722 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 723 target_list, options, run_metadata)
724 else:
725 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
741 except KeyError:
742 pass
--> 743 raise type(e)(node_def, op, message)
744
745 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor with shape[50176,512]
[[Node: truncated_normal_394/TruncatedNormal = TruncatedNormal[T=DT_INT32, dtype=DT_FLOAT, seed=87654321, seed2=66478, _device="/job:localhost/replica:0/task:0/gpu:0"](truncated_normal_394/shape)]]
Caused by op u'truncated_normal_394/TruncatedNormal', defined at:
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/adsl/anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-ef0b5a7a6ac0>", line 8, in <module>
get_ipython().magic(u'run ./convolutional.py --config="./conv4.cfg" --filter_size=4 --conv1_depth=$i --conv2_depth=$j --fc_depth=$k')
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2163, in magic
return self.run_line_magic(magic_name, magic_arg_s)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2084, in run_line_magic
result = fn(*args,**kwargs)
File "<decorator-gen-58>", line 2, in run
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magic.py", line 193, in <lambda>
call = lambda f, *a, **k: f(*a, **k)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 743, in run
run()
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/magics/execution.py", line 729, in run
exit_ignore=exit_ignore)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2493, in safe_execfile
self.compile if kw['shell_futures'] else None)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/IPython/utils/py3compat.py", line 288, in execfile
builtin_mod.execfile(filename, *where)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 407, in <module>
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/webofthink/tf-hpo/src/convolutional.py", line 296, in main
vars = init_vars(FLAGS.filter_size, FLAGS.conv1_depth, FLAGS.conv2_depth, FLAGS.fc_depth)
File "/home/webofthink/tf-hpo/src/convolutional.py", line 133, in init_vars
seed=SEED))
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/random_ops.py", line 178, in truncated_normal
seed2=seed2)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_random_ops.py", line 291, in _truncated_normal
seed=seed, seed2=seed2, name=name)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2298, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/adsl/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
In [ ]:
# run all trains twice
for i in range(2):
%run ./cnn_layer2_multi.py --all
In [ ]:
# TODO: run 256, 1024, {128, 256, 512, 1024} later
In [ ]:
neurons = [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
for i in [256, 512] :
%run ./cnn_layer2_multi.py $i
In [ ]:
neurons = [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
for i in [128, 256] :
for j in [4, 8, 16, 32, 64, 128, 256, 512, 1024] :
%run ./cnn_layer2.py $i $j
In [ ]:
for i in [8] :
for j in [128] :
for k in [128] :
%run ./cnn_layer2_multi.py $i $j $k
In [ ]:
#missing training conditions
neurons = [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
for i in [1024] :
for j in [1024] :
%run ./cnn_layer2_multi.py $i $j
In [ ]:
!export LD_LIBRARY_PATH=/usr/local/cuda/lib64
In [ ]:
!env
In [ ]:
#remained trainings
import time
print time.time()
neurons = [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
for i in [1024] :
for j in [1024] :
for k in [1024] :
%run ./cnn_layer2.py $i $j $k
Content source: hyunghunny/tf-hpo
Similar notebooks: