Removing previous artefacts...
INFO:tensorflow:Using config: {'_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true
graph_options {
rewrite_options {
meta_optimizer_iterations: ONE
}
}
, '_keep_checkpoint_max': 3, '_task_type': 'worker', '_train_distribute': None, '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x11d9116d0>, '_model_dir': 'models/census/dnn_classifier-01', '_protocol': None, '_save_checkpoints_steps': 508, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': 19831060, '_save_summary_steps': 1, '_device_fn': None, '_experimental_distribute': None, '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_evaluation_master': '', '_eval_distribute': None, '_global_id_in_cluster': 0, '_master': ''}
Experiment started at 22:03:53
.......................................
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 0 into models/census/dnn_classifier-01/model.ckpt.
---------------------------------------------------------------------------
InvalidArgumentError Traceback (most recent call last)
<ipython-input-12-dec9477e6e51> in <module>()
----> 1 train_and_evaluate_experiment(params, run_config)
<ipython-input-10-1753eb81587c> in train_and_evaluate_experiment(params, run_config)
51 # )
52
---> 53 estimator.train(train_input_fn, steps=params.traning_steps)
54
55 time_end = datetime.utcnow()
/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.pyc in train(self, input_fn, hooks, steps, max_steps, saving_listeners)
352
353 saving_listeners = _check_listeners_type(saving_listeners)
--> 354 loss = self._train_model(input_fn, hooks, saving_listeners)
355 logging.info('Loss for final step: %s.', loss)
356 return self
/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.pyc in _train_model(self, input_fn, hooks, saving_listeners)
1205 return self._train_model_distributed(input_fn, hooks, saving_listeners)
1206 else:
-> 1207 return self._train_model_default(input_fn, hooks, saving_listeners)
1208
1209 def _train_model_default(self, input_fn, hooks, saving_listeners):
/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.pyc in _train_model_default(self, input_fn, hooks, saving_listeners)
1239 return self._train_with_estimator_spec(estimator_spec, worker_hooks,
1240 hooks, global_step_tensor,
-> 1241 saving_listeners)
1242
1243 def _train_model_distributed(self, input_fn, hooks, saving_listeners):
/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.pyc in _train_with_estimator_spec(self, estimator_spec, worker_hooks, hooks, global_step_tensor, saving_listeners)
1469 loss = None
1470 while not mon_sess.should_stop():
-> 1471 _, loss = mon_sess.run([estimator_spec.train_op, estimator_spec.loss])
1472 return loss
1473
/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/training/monitored_session.pyc in run(self, fetches, feed_dict, options, run_metadata)
669 feed_dict=feed_dict,
670 options=options,
--> 671 run_metadata=run_metadata)
672
673 def run_step_fn(self, step_fn):
/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/training/monitored_session.pyc in run(self, fetches, feed_dict, options, run_metadata)
1154 feed_dict=feed_dict,
1155 options=options,
-> 1156 run_metadata=run_metadata)
1157 except _PREEMPTION_ERRORS as e:
1158 logging.info('An error was raised. This may be due to a preemption in '
/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/training/monitored_session.pyc in run(self, *args, **kwargs)
1253 raise six.reraise(*original_exc_info)
1254 else:
-> 1255 raise six.reraise(*original_exc_info)
1256
1257
/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/training/monitored_session.pyc in run(self, *args, **kwargs)
1238 def run(self, *args, **kwargs):
1239 try:
-> 1240 return self._sess.run(*args, **kwargs)
1241 except _PREEMPTION_ERRORS:
1242 raise
/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/training/monitored_session.pyc in run(self, fetches, feed_dict, options, run_metadata)
1310 feed_dict=feed_dict,
1311 options=options,
-> 1312 run_metadata=run_metadata)
1313
1314 for hook in self._hooks:
/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/training/monitored_session.pyc in run(self, *args, **kwargs)
1074
1075 def run(self, *args, **kwargs):
-> 1076 return self._sess.run(*args, **kwargs)
1077
1078 def run_step_fn(self, step_fn, raw_session, run_with_hooks):
/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
927 try:
928 result = self._run(None, fetches, feed_dict, options_ptr,
--> 929 run_metadata_ptr)
930 if run_metadata:
931 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
1150 if final_fetches or final_targets or (handle and feed_dict_tensor):
1151 results = self._do_run(handle, final_targets, final_fetches,
-> 1152 feed_dict_tensor, options, run_metadata)
1153 else:
1154 results = []
/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
1326 if handle is None:
1327 return self._do_call(_run_fn, feeds, fetches, targets, options,
-> 1328 run_metadata)
1329 else:
1330 return self._do_call(_prun_fn, handle, feeds, fetches)
/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
1346 pass
1347 message = error_interpolation.interpolate(message, self._graph)
-> 1348 raise type(e)(node_def, op, message)
1349
1350 def _extend_graph(self):
InvalidArgumentError: Integer division by zero
[[node FloorMod (defined at <ipython-input-7-fc13f81c40ce>:17) = FloorMod[T=DT_INT32, _class=["loc:@cond/CosineDecay/Cast_1/Switch"], _device="/job:localhost/replica:0/task:0/device:CPU:0"](Cast, Mul)]]
Caused by op u'FloorMod', defined at:
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/runpy.py", line 162, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/ipykernel_launcher.py", line 16, in <module>
app.launch_new_instance()
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 486, in start
self.io_loop.start()
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tornado/ioloop.py", line 1008, in start
self._run_callback(self._callbacks.popleft())
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tornado/ioloop.py", line 759, in _run_callback
ret = callback()
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tornado/stack_context.py", line 276, in null_wrapper
return fn(*args, **kwargs)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 536, in <lambda>
self.io_loop.add_callback(lambda : self._handle_events(self.socket, 0))
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 450, in _handle_events
self._handle_recv()
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 480, in _handle_recv
self._run_callback(callback, msg)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 432, in _run_callback
callback(*args, **kwargs)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tornado/stack_context.py", line 276, in null_wrapper
return fn(*args, **kwargs)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 233, in dispatch_shell
handler(stream, idents, msg)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 208, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/ipykernel/zmqshell.py", line 537, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2714, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2824, in run_ast_nodes
if self.run_code(code, result):
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2878, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-12-dec9477e6e51>", line 1, in <module>
train_and_evaluate_experiment(params, run_config)
File "<ipython-input-10-1753eb81587c>", line 53, in train_and_evaluate_experiment
estimator.train(train_input_fn, steps=params.traning_steps)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 354, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 1207, in _train_model
return self._train_model_default(input_fn, hooks, saving_listeners)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 1237, in _train_model_default
features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 1195, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "<ipython-input-7-fc13f81c40ce>", line 62, in model_fn
learning_rate = find_learning_rate(params) if params.lr_search else update_learning_rate(params)
File "<ipython-input-7-fc13f81c40ce>", line 17, in update_learning_rate
cycle_step = tf.mod(training_step, current_cycle_length)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/ops/gen_math_ops.py", line 3142, in floor_mod
"FloorMod", x=x, y=y, name=name)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/util/deprecation.py", line 488, in new_func
return func(*args, **kwargs)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 3274, in create_op
op_def=op_def)
File "/Users/khalidsalama/Technology/python-venvs/py27-venv/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1770, in __init__
self._traceback = tf_stack.extract_stack()
InvalidArgumentError (see above for traceback): Integer division by zero
[[node FloorMod (defined at <ipython-input-7-fc13f81c40ce>:17) = FloorMod[T=DT_INT32, _class=["loc:@cond/CosineDecay/Cast_1/Switch"], _device="/job:localhost/replica:0/task:0/device:CPU:0"](Cast, Mul)]]