notebook.community

Edit and run

01 SEP 2017





In [1]:



    


%matplotlib inline
import importlib

import os, sys; sys.path.insert(1, os.path.join('../utils'))

import utils2; importlib.reload(utils2)
from utils2 import *

from scipy.optimize import fmin_l_bfgs_b
from scipy.misc import imsave
from keras import metrics

from vgg16_avg import VGG16_Avg



    


















    






Using TensorFlow backend.
/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)

















In [2]:



    


from bcolz_array_iterator import BcolzArrayIterator



    











In [3]:



    


limit_mem()



    











In [4]:



    


path = '../data/'
dpath = path



    











In [5]:



    


rn_mean = np.array([123.68, 116.779, 103.939], dtype=np.float32)
preproc = lambda x: (x - rn_mean)[:, :, :, ::-1]
deproc = lambda x,s: np.clip(x.reshape(s)[:, :, :, ::-1] + rn_mean, 0, 255)



    











In [6]:



    


arr_lr = bcolz.open(dpath+'trn_resized_72.bc')
arr_hr = bcolz.open(path+'trn_resized_288.bc')
parms = {'verbose': 0, 'callbacks': [TQDMNotebookCallback(leave_inner=True)]}



    











In [6]:



    


parms = {'verbose': 0, 'callbacks': [TQDMNotebookCallback(leave_inner=True)]}



    











In [7]:



    


def conv_block(x, filters, size, stride=(2,2), mode='same', act=True):
    x = Convolution2D(filters, size, size, subsample=stride, border_mode=mode)(x)
    x = BatchNormalization(mode=2)(x)
    return Activation('relu')(x) if act else x
def res_block(ip, nf=64):
    x = conv_block(ip, nf, 3, (1,1))
    x = conv_block(x, nf, 3, (1,1), act=False)
    return merge([x, ip], mode='sum')
def up_block(x, filters, size):
    x = keras.layers.UpSampling2D()(x)
    x = Convolution2D(filters, size, size, border_mode='same')(x)
    x = BatchNormalization(mode=2)(x)
    return Activation('relu')(x)
def get_model(arr):
    inp=Input(arr.shape[1:])
    x=conv_block(inp, 64, 9, (1,1))
    for i in range(4): x=res_block(x)
    x=up_block(x, 64, 3)
    x=up_block(x, 64, 3)
    x=Convolution2D(3, 9, 9, activation='tanh', border_mode='same')(x)
    outp=Lambda(lambda x: (x+1)*127.5)(x)
    return inp,outp



    











In [8]:



    


inp,outp=get_model(arr_lr)



    











In [9]:



    


shp = arr_hr.shape[1:]

vgg_inp=Input(shp)
vgg= VGG16(include_top=False, input_tensor=Lambda(preproc)(vgg_inp))
for l in vgg.layers: l.trainable=False



    











In [10]:



    


def get_outp(m, ln): return m.get_layer(f'block{ln}_conv2').output
vgg_content = Model(vgg_inp, [get_outp(vgg, o) for o in [1,2,3]])
vgg1 = vgg_content(vgg_inp)
vgg2 = vgg_content(outp)



    











In [11]:



    


def mean_sqr_b(diff): 
    dims = list(range(1,K.ndim(diff)))
    return K.expand_dims(K.sqrt(K.mean(diff**2, dims)), 0)



    











In [12]:



    


w=[0.1, 0.8, 0.1]
def content_fn(x): 
    res = 0; n=len(w)
    for i in range(n): res += mean_sqr_b(x[i]-x[i+n]) * w[i]
    return res



    











In [13]:



    


m_sr = Model([inp, vgg_inp], Lambda(content_fn)(vgg1+vgg2))
m_sr.compile('adam', 'mae')



    











In [14]:



    


def train(bs, niter=10):
    targ = np.zeros((bs, 1))
    bc = BcolzArrayIterator(arr_hr, arr_lr, batch_size=bs)
    for i in range(niter):
        hr,lr = next(bc)
        m_sr.train_on_batch([lr[:bs], hr[:bs]], targ)



    











In [15]:



    


its = len(arr_hr)//16; its



    


















    
Out[15]:








1214

















In [16]:



    


arr_lr.chunklen, arr_hr.chunklen



    


















    
Out[16]:








(64, 64)

















In [20]:



    


%time train(64, 18000)



    


















    






---------------------------------------------------------------------------
ResourceExhaustedError                    Traceback (most recent call last)
/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
   1138     try:
-> 1139       return fn(*args)
   1140     except errors.OpError as e:

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata)
   1120                                  feed_dict, fetch_list, target_list,
-> 1121                                  status, run_metadata)
   1122 

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/contextlib.py in __exit__(self, type, value, traceback)
     88             try:
---> 89                 next(self.gen)
     90             except StopIteration:

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py in raise_exception_on_not_ok_status()
    465           compat.as_text(pywrap_tensorflow.TF_Message(status)),
--> 466           pywrap_tensorflow.TF_GetCode(status))
    467   finally:

ResourceExhaustedError: OOM when allocating tensor with shape[64,64,288,288]
	 [[Node: Conv2D_25 = Conv2D[T=DT_FLOAT, data_format="NHWC", padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/gpu:0"](strided_slice_15, block1_conv1_W/read)]]
	 [[Node: mul_11/_393 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/gpu:0", send_device_incarnation=1, tensor_name="edge_7081_mul_11", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]()]]

During handling of the above exception, another exception occurred:

ResourceExhaustedError                    Traceback (most recent call last)
<ipython-input-20-89fc063cb3c0> in <module>()
----> 1 get_ipython().magic('time train(64, 18000)')

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/IPython/core/interactiveshell.py in magic(self, arg_s)
   2158         magic_name, _, magic_arg_s = arg_s.partition(' ')
   2159         magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
-> 2160         return self.run_line_magic(magic_name, magic_arg_s)
   2161 
   2162     #-------------------------------------------------------------------------

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/IPython/core/interactiveshell.py in run_line_magic(self, magic_name, line)
   2079                 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
   2080             with self.builtin_trap:
-> 2081                 result = fn(*args,**kwargs)
   2082             return result
   2083 

<decorator-gen-60> in time(self, line, cell, local_ns)

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/IPython/core/magic.py in <lambda>(f, *a, **k)
    186     # but it's overkill for just that one bit of state.
    187     def magic_deco(arg):
--> 188         call = lambda f, *a, **k: f(*a, **k)
    189 
    190         if callable(arg):

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/IPython/core/magics/execution.py in time(self, line, cell, local_ns)
   1179         if mode=='eval':
   1180             st = clock2()
-> 1181             out = eval(code, glob, local_ns)
   1182             end = clock2()
   1183         else:

<timed eval> in <module>()

<ipython-input-14-a86779a0a73f> in train(bs, niter)
      4     for i in range(niter):
      5         hr,lr = next(bc)
----> 6         m_sr.train_on_batch([lr[:bs], hr[:bs]], targ)

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/keras/engine/training.py in train_on_batch(self, x, y, sample_weight, class_weight)
   1318             ins = x + y + sample_weights
   1319         self._make_train_function()
-> 1320         outputs = self.train_function(ins)
   1321         if len(outputs) == 1:
   1322             return outputs[0]

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py in __call__(self, inputs)
   1941         session = get_session()
   1942         updated = session.run(self.outputs + [self.updates_op],
-> 1943                               feed_dict=feed_dict)
   1944         return updated[:len(self.outputs)]
   1945 

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata)
    787     try:
    788       result = self._run(None, fetches, feed_dict, options_ptr,
--> 789                          run_metadata_ptr)
    790       if run_metadata:
    791         proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
    995     if final_fetches or final_targets:
    996       results = self._do_run(handle, final_targets, final_fetches,
--> 997                              feed_dict_string, options, run_metadata)
    998     else:
    999       results = []

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
   1130     if handle is None:
   1131       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
-> 1132                            target_list, options, run_metadata)
   1133     else:
   1134       return self._do_call(_prun_fn, self._session, handle, feed_dict,

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
   1150         except KeyError:
   1151           pass
-> 1152       raise type(e)(node_def, op, message)
   1153 
   1154   def _extend_graph(self):

ResourceExhaustedError: OOM when allocating tensor with shape[64,64,288,288]
	 [[Node: Conv2D_25 = Conv2D[T=DT_FLOAT, data_format="NHWC", padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/gpu:0"](strided_slice_15, block1_conv1_W/read)]]
	 [[Node: mul_11/_393 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/gpu:0", send_device_incarnation=1, tensor_name="edge_7081_mul_11", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]()]]

Caused by op 'Conv2D_25', defined at:
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/ipykernel_launcher.py", line 16, in <module>
    app.launch_new_instance()
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance
    app.start()
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/ipykernel/kernelapp.py", line 477, in start
    ioloop.IOLoop.instance().start()
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/zmq/eventloop/ioloop.py", line 177, in start
    super(ZMQIOLoop, self).start()
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/tornado/ioloop.py", line 888, in start
    handler_func(fd_obj, events)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
    self._handle_recv()
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
    self._run_callback(callback, msg)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
    callback(*args, **kwargs)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 283, in dispatcher
    return self.dispatch_shell(stream, msg)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 235, in dispatch_shell
    handler(stream, idents, msg)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 399, in execute_request
    user_expressions, allow_stdin)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/ipykernel/ipkernel.py", line 196, in do_execute
    res = shell.run_cell(code, store_history=store_history, silent=silent)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/ipykernel/zmqshell.py", line 533, in run_cell
    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2718, in run_cell
    interactivity=interactivity, compiler=compiler, result=result)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2822, in run_ast_nodes
    if self.run_code(code, result):
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2882, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-10-037a1c058ae3>", line 3, in <module>
    vgg1 = vgg_content(vgg_inp)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/keras/engine/topology.py", line 572, in __call__
    self.add_inbound_node(inbound_layers, node_indices, tensor_indices)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/keras/engine/topology.py", line 635, in add_inbound_node
    Node.create_node(self, inbound_layers, node_indices, tensor_indices)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/keras/engine/topology.py", line 166, in create_node
    output_tensors = to_list(outbound_layer.call(input_tensors[0], mask=input_masks[0]))
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/keras/engine/topology.py", line 2247, in call
    output_tensors, output_masks, output_shapes = self.run_internal_graph(inputs, masks)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/keras/engine/topology.py", line 2390, in run_internal_graph
    computed_mask))
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/keras/layers/convolutional.py", line 475, in call
    filter_shape=self.W_shape)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py", line 2691, in conv2d
    x = tf.nn.conv2d(x, kernel, strides, padding=padding)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py", line 399, in conv2d
    data_format=data_format, name=name)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 767, in apply_op
    op_def=op_def)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 2506, in create_op
    original_op=self._default_original_op, op_def=op_def)
  File "/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1269, in __init__
    self._traceback = _extract_stack()

ResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[64,64,288,288]
	 [[Node: Conv2D_25 = Conv2D[T=DT_FLOAT, data_format="NHWC", padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/gpu:0"](strided_slice_15, block1_conv1_W/read)]]
	 [[Node: mul_11/_393 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/gpu:0", send_device_incarnation=1, tensor_name="edge_7081_mul_11", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]()]]

Finally starting to understand this problem. So ResourceExhaustedError isn't system memory (or at least not only) but graphics memory. The card (obviously) cannot handle a batch size of 64. But batch size must be a multiple of chunk length, which here is 64.. so I have to find a way to reduce the chunk length down to something my system can handle: no more than 8.





In [23]:



    


arr_lr_c8 = bcolz.carray(arr_lr, chunklen=8, rootdir=path+'trn_resized_72_c8.bc')
arr_lr_c8.flush()



    











In [25]:



    


arr_hr_c8 = bcolz.carray(arr_hr, chunklen=8, rootdir=path+'trn_resized_288_c8.bc')
arr_hr_c8.flush()



    











In [26]:



    


arr_lr_c8.chunklen, arr_hr_c8.chunklen



    


















    
Out[26]:








(8, 8)

That looks successful, now to redo the whole thing with the _c8 versions:





In [8]:



    


arr_lr_c8 = bcolz.open(path+'trn_resized_72_c8.bc')
arr_hr_c8 = bcolz.open(path+'trn_resized_288_c8.bc')



    











In [32]:



    


inp,outp=get_model(arr_lr_c8)

shp = arr_hr_c8.shape[1:]

vgg_inp=Input(shp)
vgg= VGG16(include_top=False, input_tensor=Lambda(preproc)(vgg_inp))
for l in vgg.layers: l.trainable=False
    
vgg_content = Model(vgg_inp, [get_outp(vgg, o) for o in [1,2,3]])
vgg1 = vgg_content(vgg_inp)
vgg2 = vgg_content(outp)

m_sr = Model([inp, vgg_inp], Lambda(content_fn)(vgg1+vgg2))
m_sr.compile('adam', 'mae')

def train(bs, niter=10):
    targ = np.zeros((bs, 1))
    bc = BcolzArrayIterator(arr_hr_c8, arr_lr_c8, batch_size=bs)
    for i in range(niter):
        hr,lr = next(bc)
        m_sr.train_on_batch([lr[:bs], hr[:bs]], targ)



    











In [33]:



    


%time train(8, 18000) # not sure what exactly the '18000' is for



    


















    






---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-33-f8facddb1c66> in <module>()
----> 1 get_ipython().magic("time train(8, 18000) # not sure what exactly the '18000' is for")

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/IPython/core/interactiveshell.py in magic(self, arg_s)
   2158         magic_name, _, magic_arg_s = arg_s.partition(' ')
   2159         magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
-> 2160         return self.run_line_magic(magic_name, magic_arg_s)
   2161 
   2162     #-------------------------------------------------------------------------

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/IPython/core/interactiveshell.py in run_line_magic(self, magic_name, line)
   2079                 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
   2080             with self.builtin_trap:
-> 2081                 result = fn(*args,**kwargs)
   2082             return result
   2083 

<decorator-gen-60> in time(self, line, cell, local_ns)

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/IPython/core/magic.py in <lambda>(f, *a, **k)
    186     # but it's overkill for just that one bit of state.
    187     def magic_deco(arg):
--> 188         call = lambda f, *a, **k: f(*a, **k)
    189 
    190         if callable(arg):

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/IPython/core/magics/execution.py in time(self, line, cell, local_ns)
   1179         if mode=='eval':
   1180             st = clock2()
-> 1181             out = eval(code, glob, local_ns)
   1182             end = clock2()
   1183         else:

<timed eval> in <module>()

<ipython-input-32-19c1cf317c27> in train(bs, niter)
     19     for i in range(niter):
     20         hr,lr = next(bc)
---> 21         m_sr.train_on_batch([lr[:bs], hr[:bs]], targ)

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/keras/engine/training.py in train_on_batch(self, x, y, sample_weight, class_weight)
   1312             sample_weight=sample_weight,
   1313             class_weight=class_weight,
-> 1314             check_batch_axis=True)
   1315         if self.uses_learning_phase and not isinstance(K.learning_phase, int):
   1316             ins = x + y + sample_weights + [1.]

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/keras/engine/training.py in _standardize_user_data(self, x, y, sample_weight, class_weight, check_batch_axis, batch_size)
   1039                           for (ref, sw, cw, mode)
   1040                           in zip(y, sample_weights, class_weights, self.sample_weight_modes)]
-> 1041         check_array_lengths(x, y, sample_weights)
   1042         check_loss_and_target_compatibility(y, self.loss_functions, self.internal_output_shapes)
   1043         if self.stateful and batch_size:

/home/wnixalo/miniconda3/envs/FAI3/lib/python3.6/site-packages/keras/engine/training.py in check_array_lengths(inputs, targets, weights)
    191                          'the same number of samples as target arrays. '
    192                          'Found ' + str(list(set_x)[0]) + ' input samples '
--> 193                          'and ' + str(list(set_y)[0]) + ' target samples.')
    194     if list(set_x)[0] != list(set_w)[0]:
    195         raise ValueError('Sample_weight arrays should have '

ValueError: Input arrays should have the same number of samples as target arrays. Found 7 input samples and 8 target samples.
















In [34]:



    


arr_lr.shape, arr_hr.shape, arr_lr_c8.shape, arr_hr_c8.shape



    


















    
Out[34]:








((19439, 72, 72, 3),
 (19439, 288, 288, 3),
 (19439, 72, 72, 3),
 (19439, 288, 288, 3))

















In [ ]:



    


# 19439//8 = 2429
%time train(8, 2430)

Content source: WNoxchi/Kaukasos

Similar notebooks:

notebook.community | gallery | about