In [4]:

    

import sys
sys.path.append('..')

from deepgraph.utils.logging import log
from deepgraph.utils.common import batch_parallel, ConfigMixin, shuffle_in_unison_inplace, pickle_dump
from deepgraph.utils.image import batch_pad_mirror
from deepgraph.constants import *
from deepgraph.conf import rng

from deepgraph.pipeline import Processor, Packet


    

In [5]:

    

from deepgraph.nn.init import *
class Transformer(Processor):
    """
    Apply online random augmentation.
    """
    def __init__(self, name, shapes, config, buffer_size=10):
        super(Transformer, self).__init__(name, shapes, config, buffer_size)
        self.mean = None

    def init(self):
        if self.conf("mean_file") is not None:
            self.mean = np.load(self.conf("mean_file"))
        else:
            log("Transformer - No mean file specified.", LOG_LEVEL_WARNING)

    def process(self):
        packet = self.pull()
        # Return if no data is there
        if not packet:
            return False
        # Unpack
        data, label = packet.data
        # Do processing
        log("Transformer - Processing data", LOG_LEVEL_VERBOSE)
        
        h = 240
        w = 320
        
        start = time.time()
        # Mean
        if packet.phase == PHASE_TRAIN or packet.phase == PHASE_VAL:
            data = data.astype(np.float32)
            if self.mean is not None:
                std = self.conf("std_norm")
                for idx in range(data.shape[0]):
                    # Subtract mean
                    data[idx] = data[idx] - self.mean.astype(np.float32)
                    if std is not None:
                        data[idx] =  data[idx] * std
            if self.conf("offset") is not None:
                label -= self.conf("offset")

        if packet.phase == PHASE_TRAIN:
             # Do elementwise operations
            data_old = data
            label_old = label
            data = np.zeros((data_old.shape[0], data_old.shape[1], h, w), dtype=np.float32)
            label = np.zeros((label_old.shape[0], h, w), dtype=np.float32)
            for idx in range(data.shape[0]):
                # Rotate
                # We rotate before cropping to be able to get filled corners
                # Maybe even adjust the border after rotating
                deg = np.random.randint(-5,6)
                # Operate on old data. Careful - data is already in float so we need to normalize and rescale afterwards
                # data_old[idx] = 255. * rotate_transformer_rgb_uint8(data_old[idx] * 0.003921568627, deg).astype(np.float32)
                # label_old[idx] = rotate_transformer_scalar_float32(label_old[idx], deg)
                
                # Take care of any empty areas, we crop on a smaller surface depending on the angle
                # TODO Remove this once loss supports masking

                shift = 0 #np.tan((deg/180.) * math.pi)
                # Random crops
                cy = rng.randint(data_old.shape[2] - h - shift, size=1)
                cx = rng.randint(data_old.shape[3] - w - shift, size=1)

                data[idx] = data_old[idx, :, cy:cy+h, cx:cx+w]
                label[idx] = label_old[idx, cy:cy+h, cx:cx+w]

                # Flip horizontally with probability 0.5
                p = rng.randint(2)
                if p > 0:
                    data[idx] = data[idx, :, :, ::-1]
                    label[idx] = label[idx, :, ::-1]

                # RGB we mult with a random value between 0.8 and 1.2
                r = rng.randint(80,121) / 100.
                g = rng.randint(80,121) / 100.
                b = rng.randint(80,121) / 100.
                data[idx, 0] = data[idx, 0] * r
                data[idx, 1] = data[idx, 1] * g
                data[idx, 2] = data[idx, 2] * b
                
            # Shuffle
            data, label = shuffle_in_unison_inplace(data, label)
            
        elif packet.phase == PHASE_VAL:
            # Center crop
            cy = (data.shape[2] - h) // 2
            cx = (data.shape[3] - w) // 2
            data = data[:, :, cy:cy+h, cx:cx+w]
            label = label[:, cy:cy+h, cx:cx+w]
            
        end = time.time()
        log("Transformer - Processing took " + str(end - start) + " seconds.", LOG_LEVEL_VERBOSE)
        # Try to push into queue as long as thread should not terminate
        self.push(Packet(identifier=packet.id, phase=packet.phase, num=2, data=(data, label)))
        return True

    def setup_defaults(self):
        super(Transformer, self).setup_defaults()
        self.conf_default("mean_file", None)
        self.conf_default("offset", None)
        self.conf_default("std_norm", 1.0)


    

In [6]:

    

from theano.tensor.nnet import relu

from deepgraph.graph import *
from deepgraph.nn.core import *
from deepgraph.nn.conv import *
from deepgraph.nn.loss import *
from deepgraph.solver import *
from deepgraph.nn.init import *

from deepgraph.pipeline import Optimizer, H5DBLoader, Pipeline


def build_u_graph():
    graph = Graph("u_depth")

    """
    Inputs
    """
    data = Data(graph, "data", T.ftensor4, shape=(-1, 3, 240, 320))
    label = Data(graph, "label", T.ftensor3, shape=(-1, 1, 240, 320), config={
        "phase": PHASE_TRAIN
    })
    """
    Contractive part
    """
    conv_1 = Conv2D(
        graph,
        "conv_1",
        config={
            "channels": 64,
            "kernel": (3, 3),
            "border_mode": (1, 1),
            "activation": relu,
            "weight_filler": xavier(gain="relu"),
            "bias_filler": constant(0)
        }
    )
    conv_2 = Conv2D(
        graph,
        "conv_2",
        config={
            "channels": 64,
            "kernel": (3, 3),
            "border_mode": (1, 1),
            "activation": relu,
            "weight_filler": xavier(gain="relu"),
            "bias_filler": constant(0)
        }
    )
    pool_2 = Pool(graph, "pool_2", config={
        "kernel": (2, 2)
    })
    conv_3 = Conv2D(
        graph,
        "conv_3",
        config={
            "channels": 128,
            "kernel": (3, 3),
            "border_mode": (1, 1),
            "activation": relu,
            "weight_filler": xavier(gain="relu"),
            "bias_filler": constant(0)
        }
    )
    conv_4 = Conv2D(
        graph,
        "conv_4",
        config={
            "channels": 128,
            "kernel": (3, 3),
            "border_mode": (1, 1),
            "activation": relu,
            "weight_filler": xavier(gain="relu"),
            "bias_filler": constant(0)
        }
    )
    pool_4 = Pool(graph, "pool_4", config={
        "kernel": (2, 2)
    })

    conv_5 = Conv2D(
        graph,
        "conv_5",
        config={
            "channels": 256,
            "kernel": (3, 3),
            "border_mode": (1, 1),
            "activation": relu,
            "weight_filler": xavier(gain="relu"),
            "bias_filler": constant(0)
        }
    )
    conv_6 = Conv2D(
        graph,
        "conv_6",
        config={
            "channels": 256,
            "kernel": (3, 3),
            "border_mode": (1, 1),
            "activation": relu,
            "weight_filler": xavier(gain="relu"),
            "bias_filler": constant(0)
        }
    )

    pool_6 = Pool(graph, "pool_6", config={
        "kernel": (2, 2)
    })

    conv_7 = Conv2D(
        graph,
        "conv_7",
        config={
            "channels": 512,
            "kernel": (3, 3),
            "border_mode": (1, 1),
            "activation": relu,
            "weight_filler": xavier(gain="relu"),
            "bias_filler": constant(0)
        }
    )
    conv_8 = Conv2D(
        graph,
        "conv_8",
        config={
            "channels": 512,
            "kernel": (3, 3),
            "border_mode": (1, 1),
            "activation": relu,
            "weight_filler": xavier(gain="relu"),
            "bias_filler": constant(0)
        }
    )
    pool_8 = Pool(graph, "pool_8", config={
        "kernel": (2, 2)
    })
    """
    conv_9 = Conv2D(
        graph,
        "conv_9",
        config={
            "channels": 1024,
            "kernel": (3, 3),
            "border_mode": (1, 1),
            "activation": relu,
            "weight_filler": xavier(gain="relu"),
            "bias_filler": constant(0)
        }
    )
    conv_10 = Conv2D(
            graph,
            "conv_10",
            config={
                "channels": 1024,
                "kernel": (3, 3),
                "border_mode": (1, 1),
                "activation": relu,
                "weight_filler": xavier(gain="relu"),
                "bias_filler": constant(0)
            }
    )
    """
    fl = Flatten(graph, "fl",config={
            "dims": 2
    })
    fc_8 = Dense(graph, "fc_8", config={
        "out": 4096,
        "activation": relu,
        "weight_filler": xavier(),
        "bias_filler": constant(0.1)
    })
    dp_8 = Dropout(graph, "dp_8")
    fc_9 = Dense(graph, "fc_9", config={
        "out": 19200,
        "activation": relu,
        "weight_filler": xavier(),
        "bias_filler": constant(0.1)
    })
    dp_9 = Dropout(graph, "dp_9")
    rs_10 = Reshape(graph, "rs_10", config={
        "shape": (-1, 64, 15, 20)
    })
    """
    Expansive path
    """
    up_11 = Upsample(graph, "up_11", config={
        "kernel": (2, 2)
    })
    conv_11 = Conv2D(
        graph,
        "conv_11",
        config={
            "channels": 512,
            "kernel": (3, 3),
            "border_mode": 1,
            "weight_filler": xavier(),
            "bias_filler": constant(0)
        }
    )
    conv_12 = Conv2D(
        graph,
        "conv_12",
        config={
            "channels": 512,
            "kernel": (3, 3),
            "border_mode": (1, 1),
            "activation": relu,
            "weight_filler": xavier(gain="relu"),
            "bias_filler": constant(0)
        }
    )
    conv_13 = Conv2D(
        graph,
        "conv_13",
        config={
            "channels": 512,
            "kernel": (3, 3),
            "border_mode": (1, 1),
            "activation": relu,
            "weight_filler": xavier(gain="relu"),
            "bias_filler": constant(0)
        }
    )
    up_14 = Upsample(graph, "up_14", config={
        "kernel": (2, 2)
    })
    conv_14 = Conv2D(
        graph,
        "conv_14",
        config={
            "channels": 256,
            "kernel": (3, 3),
            "border_mode": 1,
            "weight_filler": xavier(),
            "bias_filler": constant(0)
        }
    )
    conv_15 = Conv2D(
        graph,
        "conv_15",
        config={
            "channels": 256,
            "kernel": (3, 3),
            "border_mode": (1, 1),
            "activation": relu,
            "weight_filler": xavier(gain="relu"),
            "bias_filler": constant(0)
        }
    )
    conv_16 = Conv2D(
        graph,
        "conv_16",
        config={
            "channels": 256,
            "kernel": (3, 3),
            "border_mode": (1, 1),
            "activation": relu,
            "weight_filler": xavier(gain="relu"),
            "bias_filler": constant(0)
        }
    )

    up_17 = Upsample(graph, "up_17", config={
        "kernel": (2, 2)
    })
    conv_17 = Conv2D(graph, "conv_17", config={
            "channels": 128,
            "kernel": (3, 3),
            "border_mode": 1,
            "weight_filler": xavier(),
            "bias_filler": constant(0)
    })
    conv_18 = Conv2D(
        graph,
        "conv_18",
        config={
            "channels": 128,
            "kernel": (3, 3),
            "border_mode": (1, 1),
            "activation": relu,
            "weight_filler": xavier(gain="relu"),
            "bias_filler": constant(0)
        }
    )
    conv_19 = Conv2D(
        graph,
        "conv_19",
        config={
            "channels": 128,
            "kernel": (3, 3),
            "border_mode": (1, 1),
            "activation": relu,
            "weight_filler": xavier(gain="relu"),
            "bias_filler": constant(0)
        }
    )
    up_20 = Upsample(graph, "up_20", config={
        "mode": "constant",
        "kernel": (2, 2)
    })
    conv_20 = Conv2D(graph, "conv_20", config={
            "channels": 64,
            "kernel": (3, 3),
            "border_mode": 1,
            "weight_filler": xavier(),
            "bias_filler": constant(0)
    })
    conv_21 = Conv2D(
        graph,
        "conv_21",
        config={
            "channels": 64,
            "kernel": (3, 3),
            "border_mode": (1, 1),
            "activation": relu,
            "weight_filler": xavier(gain="relu"),
            "bias_filler": constant(0)
        }
    )
    conv_22 = Conv2D(
        graph,
        "conv_22",
        config={
            "channels": 64,
            "kernel": (3, 3),
            "border_mode": (1, 1),
            "activation": relu,
            "weight_filler": xavier(gain="relu"),
            "bias_filler": constant(0)
        }
    )
    conv_23 = Conv2D(
        graph,
        "conv_23",
        config={
            "channels": 1,
            "kernel": (1, 1),
            "activation": None,
            "weight_filler": xavier(),
            "bias_filler": constant(0),
            "is_output": True
        }
    )

    """
    Feed forward nodes
    """
    

    concat_20 = Concatenate(graph, "concat_20", config={
        "axis": 1
    })

    

    concat_17 = Concatenate(graph, "concat_17", config={
        "axis": 1
    })

    
    concat_14 = Concatenate(graph, "concat_14", config={
        "axis": 1
    })

    

    concat_11 = Concatenate(graph, "concat_11", config={
        "axis": 1
    })


    """
    Losses / Error
    """
    loss = EuclideanLoss(graph, "loss")

    error = MSE(graph, "mse", config={
        "root": True,
        "is_output": True,
        "phase": PHASE_TRAIN
    })


    """
    Make connections
    """
    data.connect(conv_1)
    conv_1.connect(conv_2)
    conv_2.connect(concat_20)
    conv_2.connect(pool_2)
    pool_2.connect(conv_3)
    conv_3.connect(conv_4)
    conv_4.connect(concat_17)
    conv_4.connect(pool_4)
    pool_4.connect(conv_5)
    conv_5.connect(conv_6)
    conv_6.connect(concat_14)
    conv_6.connect(pool_6)
    pool_6.connect(conv_7)
    conv_7.connect(conv_8)
    conv_8.connect(concat_11)
    conv_8.connect(pool_8)
    pool_8.connect(fl)
    fl.connect(fc_8)
    fc_8.connect(dp_8)
    dp_8.connect(fc_9)
    fc_9.connect(dp_9)
    dp_9.connect(rs_10)
    rs_10.connect(up_11)
    up_11.connect(conv_11)
    conv_11.connect(concat_11)
    concat_11.connect(conv_12)
    conv_12.connect(conv_13)
    conv_13.connect(up_14)
    up_14.connect(conv_14)
    conv_14.connect(concat_14)
    concat_14.connect(conv_15)
    conv_15.connect(conv_16)
    conv_16.connect(up_17)
    up_17.connect(conv_17)
    conv_17.connect(concat_17)
    concat_17.connect(conv_18)
    conv_18.connect(conv_19)
    conv_19.connect(up_20)
    up_20.connect(conv_20)
    conv_20.connect(concat_20)
    concat_20.connect(conv_21)
    conv_21.connect(conv_22)
    conv_22.connect(conv_23)

    conv_23.connect(loss)
    label.connect(loss)

    conv_23.connect(error)
    label.connect(error)

    return graph


    

In [ ]:

    

if __name__ == "__main__":

    batch_size = 4
    chunk_size = 10*batch_size
    transfer_shape = ((chunk_size, 3, 240, 320), (chunk_size, 240, 320))

    g = build_u_graph()

    # Build the training pipeline
    db_loader = H5DBLoader("db", ((chunk_size, 3, 480, 640), (chunk_size, 1, 480, 640)), config={
        "db": "/home/ga29mix/nashome/data/nyu_depth_combined_vnet2.hdf5",
        # "db": '../data/nyu_depth_unet_large.hdf5',
        "key_data": "images",
        "key_label": "depths",
        "chunk_size": chunk_size
    })
    transformer = Transformer("tr", transfer_shape, config={
        # Pixel & channelwise mean 
        "mean_file": "/home/ga29mix/nashome/data/nyu_depth_combined_vnet2.npy",
        # 1 / sqrt(var(data))
        "std_norm": 1.0 / 72.240064849107824
    })
    optimizer = Optimizer("opt", g, transfer_shape, config={
        "batch_size":  batch_size,
        "chunk_size": chunk_size,
        # "learning_rate": 0.0001, # for step 1
        "learning_rate": 0.00001, # for step 2
        "momentum": 0.9,
        "weight_decay": 0.0005,
        "print_freq": 20,
        "save_freq": 30000,
        "weights": "/data/vnet2_crop_start_with_120k_iter_90000.zip",
        "save_prefix": "/data/vnet2_crop_start_with_210k_lower_lr"
    })

    p = Pipeline(config={
        "validation_frequency": 20,
        "cycles": 9200
    })
    p.add(db_loader)
    p.add(transformer)
    p.add(optimizer)
    p.run()


    

    




[2016-04-19 23:04:05] INFO: H5DBLoader - Caching DB in memory
[2016-04-19 23:04:35] INFO: Pipeline - Starting computation
[2016-04-19 23:05:36] INFO: Graph - Loading parameters from file '/data/vnet2_crop_start_with_120k_iter_90000.zip'
[2016-04-19 23:05:36] INFO: Graph - Setting up graph
[2016-04-19 23:05:36] INFO: Node - data has shape (-1, 3, 240, 320)
[2016-04-19 23:05:36] INFO: Node - label has shape (-1, 1, 240, 320)
[2016-04-19 23:05:36] INFO: Node - conv_1 has shape (-1, 64, 240, 320)
[2016-04-19 23:05:36] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - conv_2 has shape (-1, 64, 240, 320)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - pool_2 has shape (-1, 64, 120, 160)
[2016-04-19 23:05:37] INFO: Pool - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - conv_3 has shape (-1, 128, 120, 160)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - conv_4 has shape (-1, 128, 120, 160)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - pool_4 has shape (-1, 128, 60, 80)
[2016-04-19 23:05:37] INFO: Pool - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - conv_5 has shape (-1, 256, 60, 80)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - conv_6 has shape (-1, 256, 60, 80)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - pool_6 has shape (-1, 256, 30, 40)
[2016-04-19 23:05:37] INFO: Pool - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - conv_7 has shape (-1, 512, 30, 40)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - conv_8 has shape (-1, 512, 30, 40)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - pool_8 has shape (-1, 512, 15, 20)
[2016-04-19 23:05:37] INFO: Pool - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - fl has shape (-1, 153600)
[2016-04-19 23:05:37] INFO: Node - fc_8 has shape (-1, 4096)
[2016-04-19 23:05:37] INFO: Node - dp_8 has shape (-1, 4096)
[2016-04-19 23:05:37] INFO: Node - fc_9 has shape (-1, 19200)
[2016-04-19 23:05:37] INFO: Node - dp_9 has shape (-1, 19200)
[2016-04-19 23:05:37] INFO: Node - rs_10 has shape (-1, 64, 15, 20)
[2016-04-19 23:05:37] INFO: Node - up_11 has shape (-1, 64, 30, 40)
[2016-04-19 23:05:37] INFO: Node - conv_11 has shape (-1, 512, 30, 40)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - concat_11 has shape (-1, 1024, 30, 40)
[2016-04-19 23:05:37] INFO: Node - conv_12 has shape (-1, 512, 30, 40)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - conv_13 has shape (-1, 512, 30, 40)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - up_14 has shape (-1, 512, 60, 80)
[2016-04-19 23:05:37] INFO: Node - conv_14 has shape (-1, 256, 60, 80)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - concat_14 has shape (-1, 512, 60, 80)
[2016-04-19 23:05:37] INFO: Node - conv_15 has shape (-1, 256, 60, 80)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - conv_16 has shape (-1, 256, 60, 80)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - up_17 has shape (-1, 256, 120, 160)
[2016-04-19 23:05:37] INFO: Node - conv_17 has shape (-1, 128, 120, 160)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - concat_17 has shape (-1, 256, 120, 160)
[2016-04-19 23:05:37] INFO: Node - conv_18 has shape (-1, 128, 120, 160)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - conv_19 has shape (-1, 128, 120, 160)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - up_20 has shape (-1, 128, 240, 320)
[2016-04-19 23:05:37] INFO: Node - conv_20 has shape (-1, 64, 240, 320)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - concat_20 has shape (-1, 128, 240, 320)
[2016-04-19 23:05:37] INFO: Node - conv_21 has shape (-1, 64, 240, 320)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - conv_22 has shape (-1, 64, 240, 320)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - conv_23 has shape (-1, 1, 240, 320)
[2016-04-19 23:05:37] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 23:05:37] INFO: Node - loss has shape (1,)
[2016-04-19 23:05:37] INFO: Node - mse has shape (1,)
[2016-04-19 23:05:57] INFO: Graph - Invoking Theano compiler
[2016-04-19 23:06:33] INFO: Optimizer - Compilation finished






    




/home/ga29mix/anaconda/envs/deep/lib/python2.7/site-packages/ipykernel/__main__.py:66: DeprecationWarning: converting an array with ndim > 0 to an index will result in an error in the future
/home/ga29mix/anaconda/envs/deep/lib/python2.7/site-packages/ipykernel/__main__.py:67: DeprecationWarning: converting an array with ndim > 0 to an index will result in an error in the future






    




[2016-04-19 23:06:53] INFO: Optimizer - Training score at iteration 20: {'loss': array(0.2706755995750427, dtype=float32), 'mse': array(0.5202649235725403, dtype=float32)}
[2016-04-19 23:07:13] INFO: Optimizer - Training score at iteration 40: {'loss': array(0.1604684442281723, dtype=float32), 'mse': array(0.4005851149559021, dtype=float32)}
[2016-04-19 23:07:33] INFO: Optimizer - Training score at iteration 60: {'loss': array(0.2311982363462448, dtype=float32), 'mse': array(0.48083075881004333, dtype=float32)}
[2016-04-19 23:07:54] INFO: Optimizer - Training score at iteration 80: {'loss': array(0.3183595836162567, dtype=float32), 'mse': array(0.5642336010932922, dtype=float32)}
[2016-04-19 23:08:15] INFO: Optimizer - Training score at iteration 100: {'loss': array(0.09625892341136932, dtype=float32), 'mse': array(0.3102562129497528, dtype=float32)}
[2016-04-19 23:08:36] INFO: Optimizer - Training score at iteration 120: {'loss': array(0.1351284235715866, dtype=float32), 'mse': array(0.36759817600250244, dtype=float32)}
[2016-04-19 23:08:57] INFO: Optimizer - Training score at iteration 140: {'loss': array(0.12460304796695709, dtype=float32), 'mse': array(0.35299158096313477, dtype=float32)}
[2016-04-19 23:09:18] INFO: Optimizer - Training score at iteration 160: {'loss': array(0.1859845072031021, dtype=float32), 'mse': array(0.4312592148780823, dtype=float32)}
[2016-04-19 23:09:38] INFO: Optimizer - Training score at iteration 180: {'loss': array(0.35202792286872864, dtype=float32), 'mse': array(0.5933194160461426, dtype=float32)}
[2016-04-19 23:09:59] INFO: Optimizer - Training score at iteration 200: {'loss': array(0.08082520961761475, dtype=float32), 'mse': array(0.28429773449897766, dtype=float32)}
[2016-04-19 23:10:20] INFO: Optimizer - Mean loss values for validation at iteration 200 is: {'loss': 0.55629492, 'mse': 0.71902633}
[2016-04-19 23:10:41] INFO: Optimizer - Training score at iteration 220: {'loss': array(0.12888501584529877, dtype=float32), 'mse': array(0.3590056002140045, dtype=float32)}
[2016-04-19 23:11:01] INFO: Optimizer - Training score at iteration 240: {'loss': array(0.16697393357753754, dtype=float32), 'mse': array(0.40862444043159485, dtype=float32)}
[2016-04-19 23:11:22] INFO: Optimizer - Training score at iteration 260: {'loss': array(0.2873557507991791, dtype=float32), 'mse': array(0.5360557436943054, dtype=float32)}
[2016-04-19 23:11:42] INFO: Optimizer - Training score at iteration 280: {'loss': array(0.20422685146331787, dtype=float32), 'mse': array(0.45191463828086853, dtype=float32)}
[2016-04-19 23:12:03] INFO: Optimizer - Training score at iteration 300: {'loss': array(0.2767564058303833, dtype=float32), 'mse': array(0.5260764360427856, dtype=float32)}
[2016-04-19 23:12:23] INFO: Optimizer - Training score at iteration 320: {'loss': array(0.14937004446983337, dtype=float32), 'mse': array(0.38648420572280884, dtype=float32)}
[2016-04-19 23:12:44] INFO: Optimizer - Training score at iteration 340: {'loss': array(0.20588171482086182, dtype=float32), 'mse': array(0.4537419080734253, dtype=float32)}
[2016-04-19 23:13:04] INFO: Optimizer - Training score at iteration 360: {'loss': array(0.28501424193382263, dtype=float32), 'mse': array(0.5338672399520874, dtype=float32)}
[2016-04-19 23:13:25] INFO: Optimizer - Training score at iteration 380: {'loss': array(0.21311452984809875, dtype=float32), 'mse': array(0.4616432785987854, dtype=float32)}
[2016-04-19 23:13:45] INFO: Optimizer - Training score at iteration 400: {'loss': array(0.3024302124977112, dtype=float32), 'mse': array(0.5499365329742432, dtype=float32)}
[2016-04-19 23:14:06] INFO: Optimizer - Mean loss values for validation at iteration 400 is: {'loss': 0.54210061, 'mse': 0.70854431}
[2016-04-19 23:14:26] INFO: Optimizer - Training score at iteration 420: {'loss': array(0.06619371473789215, dtype=float32), 'mse': array(0.2572813928127289, dtype=float32)}
[2016-04-19 23:14:47] INFO: Optimizer - Training score at iteration 440: {'loss': array(0.14689227938652039, dtype=float32), 'mse': array(0.38326528668403625, dtype=float32)}
[2016-04-19 23:15:07] INFO: Optimizer - Training score at iteration 460: {'loss': array(0.16712990403175354, dtype=float32), 'mse': array(0.4088152348995209, dtype=float32)}
[2016-04-19 23:15:28] INFO: Optimizer - Training score at iteration 480: {'loss': array(0.08202134072780609, dtype=float32), 'mse': array(0.2863936722278595, dtype=float32)}
[2016-04-19 23:15:48] INFO: Optimizer - Training score at iteration 500: {'loss': array(0.2063531130552292, dtype=float32), 'mse': array(0.45426106452941895, dtype=float32)}
[2016-04-19 23:16:09] INFO: Optimizer - Training score at iteration 520: {'loss': array(0.14647097885608673, dtype=float32), 'mse': array(0.38271528482437134, dtype=float32)}
[2016-04-19 23:16:29] INFO: Optimizer - Training score at iteration 540: {'loss': array(0.1656842976808548, dtype=float32), 'mse': array(0.40704336762428284, dtype=float32)}
[2016-04-19 23:16:50] INFO: Optimizer - Training score at iteration 560: {'loss': array(0.13212935626506805, dtype=float32), 'mse': array(0.36349600553512573, dtype=float32)}
[2016-04-19 23:17:10] INFO: Optimizer - Training score at iteration 580: {'loss': array(0.16968214511871338, dtype=float32), 'mse': array(0.41192492842674255, dtype=float32)}
[2016-04-19 23:17:31] INFO: Optimizer - Training score at iteration 600: {'loss': array(0.27915874123573303, dtype=float32), 'mse': array(0.5283547639846802, dtype=float32)}
[2016-04-19 23:17:52] INFO: Optimizer - Mean loss values for validation at iteration 600 is: {'loss': 0.52467078, 'mse': 0.69461447}
[2016-04-19 23:18:12] INFO: Optimizer - Training score at iteration 620: {'loss': array(0.10145361721515656, dtype=float32), 'mse': array(0.31851786375045776, dtype=float32)}
[2016-04-19 23:18:33] INFO: Optimizer - Training score at iteration 640: {'loss': array(0.26167032122612, dtype=float32), 'mse': array(0.5115371942520142, dtype=float32)}
[2016-04-19 23:18:53] INFO: Optimizer - Training score at iteration 660: {'loss': array(0.1670764535665512, dtype=float32), 'mse': array(0.40874984860420227, dtype=float32)}
[2016-04-19 23:19:14] INFO: Optimizer - Training score at iteration 680: {'loss': array(0.273492693901062, dtype=float32), 'mse': array(0.5229652523994446, dtype=float32)}
[2016-04-19 23:19:34] INFO: Optimizer - Training score at iteration 700: {'loss': array(0.18108676373958588, dtype=float32), 'mse': array(0.4255428910255432, dtype=float32)}
[2016-04-19 23:19:55] INFO: Optimizer - Training score at iteration 720: {'loss': array(0.14676377177238464, dtype=float32), 'mse': array(0.3830975890159607, dtype=float32)}
[2016-04-19 23:20:15] INFO: Optimizer - Training score at iteration 740: {'loss': array(0.11260595172643661, dtype=float32), 'mse': array(0.3355681002140045, dtype=float32)}
[2016-04-19 23:20:36] INFO: Optimizer - Training score at iteration 760: {'loss': array(0.1304275542497635, dtype=float32), 'mse': array(0.36114755272865295, dtype=float32)}
[2016-04-19 23:20:56] INFO: Optimizer - Training score at iteration 780: {'loss': array(0.16152827441692352, dtype=float32), 'mse': array(0.4019058048725128, dtype=float32)}
[2016-04-19 23:21:17] INFO: Optimizer - Training score at iteration 800: {'loss': array(0.11276382207870483, dtype=float32), 'mse': array(0.33580324053764343, dtype=float32)}
[2016-04-19 23:21:38] INFO: Optimizer - Mean loss values for validation at iteration 800 is: {'loss': 0.53273505, 'mse': 0.70117086}
[2016-04-19 23:21:58] INFO: Optimizer - Training score at iteration 820: {'loss': array(0.18046249449253082, dtype=float32), 'mse': array(0.4248087704181671, dtype=float32)}
[2016-04-19 23:22:19] INFO: Optimizer - Training score at iteration 840: {'loss': array(0.1223592534661293, dtype=float32), 'mse': array(0.34979885816574097, dtype=float32)}
[2016-04-19 23:22:39] INFO: Optimizer - Training score at iteration 860: {'loss': array(0.36683133244514465, dtype=float32), 'mse': array(0.6056660413742065, dtype=float32)}
[2016-04-19 23:23:00] INFO: Optimizer - Training score at iteration 880: {'loss': array(0.20772793889045715, dtype=float32), 'mse': array(0.455771803855896, dtype=float32)}
[2016-04-19 23:23:20] INFO: Optimizer - Training score at iteration 900: {'loss': array(0.1462080031633377, dtype=float32), 'mse': array(0.38237154483795166, dtype=float32)}
[2016-04-19 23:23:41] INFO: Optimizer - Training score at iteration 920: {'loss': array(0.1395174115896225, dtype=float32), 'mse': array(0.37352028489112854, dtype=float32)}
[2016-04-19 23:24:01] INFO: Optimizer - Training score at iteration 940: {'loss': array(0.10741475969552994, dtype=float32), 'mse': array(0.32774192094802856, dtype=float32)}
[2016-04-19 23:24:22] INFO: Optimizer - Training score at iteration 960: {'loss': array(0.13881298899650574, dtype=float32), 'mse': array(0.37257614731788635, dtype=float32)}
[2016-04-19 23:24:42] INFO: Optimizer - Training score at iteration 980: {'loss': array(0.11089766025543213, dtype=float32), 'mse': array(0.33301299810409546, dtype=float32)}
[2016-04-19 23:25:23] INFO: Optimizer - Mean loss values for validation at iteration 999 is: {'loss': 0.56835139, 'mse': 0.7280758}
[2016-04-19 23:25:24] INFO: Optimizer - Training score at iteration 1000: {'loss': array(0.15788273513317108, dtype=float32), 'mse': array(0.3973446190357208, dtype=float32)}
[2016-04-19 23:25:44] INFO: Optimizer - Training score at iteration 1020: {'loss': array(0.1861446499824524, dtype=float32), 'mse': array(0.43144482374191284, dtype=float32)}
[2016-04-19 23:26:05] INFO: Optimizer - Training score at iteration 1040: {'loss': array(0.3079083561897278, dtype=float32), 'mse': array(0.5548949241638184, dtype=float32)}
[2016-04-19 23:26:25] INFO: Optimizer - Training score at iteration 1060: {'loss': array(0.20949506759643555, dtype=float32), 'mse': array(0.4577063024044037, dtype=float32)}
[2016-04-19 23:26:46] INFO: Optimizer - Training score at iteration 1080: {'loss': array(0.22451138496398926, dtype=float32), 'mse': array(0.4738263189792633, dtype=float32)}
[2016-04-19 23:27:06] INFO: Optimizer - Training score at iteration 1100: {'loss': array(0.1344611495733261, dtype=float32), 'mse': array(0.36668944358825684, dtype=float32)}
[2016-04-19 23:27:27] INFO: Optimizer - Training score at iteration 1120: {'loss': array(0.19023267924785614, dtype=float32), 'mse': array(0.436156690120697, dtype=float32)}
[2016-04-19 23:27:47] INFO: Optimizer - Training score at iteration 1140: {'loss': array(0.16293178498744965, dtype=float32), 'mse': array(0.4036480784416199, dtype=float32)}
[2016-04-19 23:28:08] INFO: Optimizer - Training score at iteration 1160: {'loss': array(0.21666382253170013, dtype=float32), 'mse': array(0.46547162532806396, dtype=float32)}
[2016-04-19 23:28:28] INFO: Optimizer - Training score at iteration 1180: {'loss': array(0.2724468410015106, dtype=float32), 'mse': array(0.521964430809021, dtype=float32)}
[2016-04-19 23:29:09] INFO: Optimizer - Mean loss values for validation at iteration 1199 is: {'loss': 0.58398372, 'mse': 0.73615569}
[2016-04-19 23:29:10] INFO: Optimizer - Training score at iteration 1200: {'loss': array(0.1152733713388443, dtype=float32), 'mse': array(0.33951932191848755, dtype=float32)}
[2016-04-19 23:29:31] INFO: Optimizer - Training score at iteration 1220: {'loss': array(0.13533376157283783, dtype=float32), 'mse': array(0.36787736415863037, dtype=float32)}
[2016-04-19 23:29:51] INFO: Optimizer - Training score at iteration 1240: {'loss': array(0.17831341922283173, dtype=float32), 'mse': array(0.422271728515625, dtype=float32)}
[2016-04-19 23:30:12] INFO: Optimizer - Training score at iteration 1260: {'loss': array(0.20371632277965546, dtype=float32), 'mse': array(0.4513494372367859, dtype=float32)}
[2016-04-19 23:30:32] INFO: Optimizer - Training score at iteration 1280: {'loss': array(0.18728314340114594, dtype=float32), 'mse': array(0.4327622354030609, dtype=float32)}
[2016-04-19 23:30:53] INFO: Optimizer - Training score at iteration 1300: {'loss': array(0.05838797613978386, dtype=float32), 'mse': array(0.24163603782653809, dtype=float32)}
[2016-04-19 23:31:13] INFO: Optimizer - Training score at iteration 1320: {'loss': array(0.057426244020462036, dtype=float32), 'mse': array(0.23963773250579834, dtype=float32)}
[2016-04-19 23:31:33] INFO: Optimizer - Training score at iteration 1340: {'loss': array(0.2677927315235138, dtype=float32), 'mse': array(0.5174869298934937, dtype=float32)}
[2016-04-19 23:31:54] INFO: Optimizer - Training score at iteration 1360: {'loss': array(0.2641156315803528, dtype=float32), 'mse': array(0.5139217972755432, dtype=float32)}
[2016-04-19 23:32:14] INFO: Optimizer - Training score at iteration 1380: {'loss': array(0.14143700897693634, dtype=float32), 'mse': array(0.37608110904693604, dtype=float32)}
[2016-04-19 23:32:56] INFO: Optimizer - Mean loss values for validation at iteration 1399 is: {'loss': 0.53921866, 'mse': 0.70599699}
[2016-04-19 23:32:56] INFO: Optimizer - Training score at iteration 1400: {'loss': array(0.1892423927783966, dtype=float32), 'mse': array(0.43501999974250793, dtype=float32)}
[2016-04-19 23:33:17] INFO: Optimizer - Training score at iteration 1420: {'loss': array(0.12430830299854279, dtype=float32), 'mse': array(0.35257381200790405, dtype=float32)}
[2016-04-19 23:33:37] INFO: Optimizer - Training score at iteration 1440: {'loss': array(0.32978907227516174, dtype=float32), 'mse': array(0.574272632598877, dtype=float32)}
[2016-04-19 23:33:58] INFO: Optimizer - Training score at iteration 1460: {'loss': array(0.16060352325439453, dtype=float32), 'mse': array(0.4007536768913269, dtype=float32)}
[2016-04-19 23:34:18] INFO: Optimizer - Training score at iteration 1480: {'loss': array(0.0817769393324852, dtype=float32), 'mse': array(0.2859666645526886, dtype=float32)}
[2016-04-19 23:34:39] INFO: Optimizer - Training score at iteration 1500: {'loss': array(0.12596385180950165, dtype=float32), 'mse': array(0.3549138605594635, dtype=float32)}
[2016-04-19 23:34:59] INFO: Optimizer - Training score at iteration 1520: {'loss': array(0.18278393149375916, dtype=float32), 'mse': array(0.4275323748588562, dtype=float32)}
[2016-04-19 23:35:20] INFO: Optimizer - Training score at iteration 1540: {'loss': array(0.1756187379360199, dtype=float32), 'mse': array(0.4190689027309418, dtype=float32)}
[2016-04-19 23:35:40] INFO: Optimizer - Training score at iteration 1560: {'loss': array(0.16239266097545624, dtype=float32), 'mse': array(0.4029797315597534, dtype=float32)}
[2016-04-19 23:36:01] INFO: Optimizer - Training score at iteration 1580: {'loss': array(0.15420207381248474, dtype=float32), 'mse': array(0.39268574118614197, dtype=float32)}
[2016-04-19 23:36:42] INFO: Optimizer - Mean loss values for validation at iteration 1599 is: {'loss': 0.56767988, 'mse': 0.72626495}
[2016-04-19 23:36:42] INFO: Optimizer - Training score at iteration 1600: {'loss': array(0.12768816947937012, dtype=float32), 'mse': array(0.35733479261398315, dtype=float32)}
[2016-04-19 23:37:03] INFO: Optimizer - Training score at iteration 1620: {'loss': array(0.2239157259464264, dtype=float32), 'mse': array(0.473197340965271, dtype=float32)}
[2016-04-19 23:37:23] INFO: Optimizer - Training score at iteration 1640: {'loss': array(0.24788884818553925, dtype=float32), 'mse': array(0.4978843629360199, dtype=float32)}
[2016-04-19 23:37:44] INFO: Optimizer - Training score at iteration 1660: {'loss': array(0.21592210233211517, dtype=float32), 'mse': array(0.464674174785614, dtype=float32)}
[2016-04-19 23:38:04] INFO: Optimizer - Training score at iteration 1680: {'loss': array(0.11516111344099045, dtype=float32), 'mse': array(0.3393539488315582, dtype=float32)}
[2016-04-19 23:38:25] INFO: Optimizer - Training score at iteration 1700: {'loss': array(0.3209328055381775, dtype=float32), 'mse': array(0.5665093064308167, dtype=float32)}
[2016-04-19 23:38:46] INFO: Optimizer - Training score at iteration 1720: {'loss': array(0.21056251227855682, dtype=float32), 'mse': array(0.45887091755867004, dtype=float32)}
[2016-04-19 23:39:06] INFO: Optimizer - Training score at iteration 1740: {'loss': array(0.23280183970928192, dtype=float32), 'mse': array(0.48249542713165283, dtype=float32)}
[2016-04-19 23:39:27] INFO: Optimizer - Training score at iteration 1760: {'loss': array(0.2996459901332855, dtype=float32), 'mse': array(0.5473992824554443, dtype=float32)}
[2016-04-19 23:39:47] INFO: Optimizer - Training score at iteration 1780: {'loss': array(0.17970788478851318, dtype=float32), 'mse': array(0.4239196479320526, dtype=float32)}
[2016-04-19 23:40:27] INFO: Optimizer - Mean loss values for validation at iteration 1798 is: {'loss': 0.54666984, 'mse': 0.71055835}
[2016-04-19 23:40:29] INFO: Optimizer - Training score at iteration 1800: {'loss': array(0.22094836831092834, dtype=float32), 'mse': array(0.47005143761634827, dtype=float32)}
[2016-04-19 23:40:49] INFO: Optimizer - Training score at iteration 1820: {'loss': array(0.17760193347930908, dtype=float32), 'mse': array(0.4214284420013428, dtype=float32)}
[2016-04-19 23:41:10] INFO: Optimizer - Training score at iteration 1840: {'loss': array(0.16668756306171417, dtype=float32), 'mse': array(0.4082738757133484, dtype=float32)}
[2016-04-19 23:41:30] INFO: Optimizer - Training score at iteration 1860: {'loss': array(0.1131724938750267, dtype=float32), 'mse': array(0.3364112079143524, dtype=float32)}
[2016-04-19 23:41:51] INFO: Optimizer - Training score at iteration 1880: {'loss': array(0.19981403648853302, dtype=float32), 'mse': array(0.44700562953948975, dtype=float32)}
[2016-04-19 23:42:12] INFO: Optimizer - Training score at iteration 1900: {'loss': array(0.2262762039899826, dtype=float32), 'mse': array(0.4756849706172943, dtype=float32)}
[2016-04-19 23:42:32] INFO: Optimizer - Training score at iteration 1920: {'loss': array(0.08033069968223572, dtype=float32), 'mse': array(0.2834267020225525, dtype=float32)}
[2016-04-19 23:42:53] INFO: Optimizer - Training score at iteration 1940: {'loss': array(0.06723694503307343, dtype=float32), 'mse': array(0.2593008875846863, dtype=float32)}
[2016-04-19 23:43:13] INFO: Optimizer - Training score at iteration 1960: {'loss': array(0.16486535966396332, dtype=float32), 'mse': array(0.4060361385345459, dtype=float32)}
[2016-04-19 23:43:34] INFO: Optimizer - Training score at iteration 1980: {'loss': array(0.11989404261112213, dtype=float32), 'mse': array(0.34625717997550964, dtype=float32)}
[2016-04-19 23:44:14] INFO: Optimizer - Mean loss values for validation at iteration 1998 is: {'loss': 0.54681122, 'mse': 0.71226001}
[2016-04-19 23:44:15] INFO: Optimizer - Training score at iteration 2000: {'loss': array(0.34942442178726196, dtype=float32), 'mse': array(0.5911213159561157, dtype=float32)}
[2016-04-19 23:44:36] INFO: Optimizer - Training score at iteration 2020: {'loss': array(0.275638610124588, dtype=float32), 'mse': array(0.5250129699707031, dtype=float32)}
[2016-04-19 23:44:56] INFO: Optimizer - Training score at iteration 2040: {'loss': array(0.14671765267848969, dtype=float32), 'mse': array(0.38303738832473755, dtype=float32)}
[2016-04-19 23:45:17] INFO: Optimizer - Training score at iteration 2060: {'loss': array(0.2110370397567749, dtype=float32), 'mse': array(0.4593876600265503, dtype=float32)}
[2016-04-19 23:45:37] INFO: Optimizer - Training score at iteration 2080: {'loss': array(0.19194065034389496, dtype=float32), 'mse': array(0.4381103217601776, dtype=float32)}
[2016-04-19 23:45:58] INFO: Optimizer - Training score at iteration 2100: {'loss': array(0.1975812464952469, dtype=float32), 'mse': array(0.4445011019706726, dtype=float32)}
[2016-04-19 23:46:18] INFO: Optimizer - Training score at iteration 2120: {'loss': array(0.32983022928237915, dtype=float32), 'mse': array(0.574308454990387, dtype=float32)}
[2016-04-19 23:46:39] INFO: Optimizer - Training score at iteration 2140: {'loss': array(0.1057792529463768, dtype=float32), 'mse': array(0.3252372145652771, dtype=float32)}
[2016-04-19 23:46:59] INFO: Optimizer - Training score at iteration 2160: {'loss': array(0.14448410272598267, dtype=float32), 'mse': array(0.38011065125465393, dtype=float32)}
[2016-04-19 23:47:20] INFO: Optimizer - Training score at iteration 2180: {'loss': array(0.15157397091388702, dtype=float32), 'mse': array(0.38932502269744873, dtype=float32)}
[2016-04-19 23:48:00] INFO: Optimizer - Mean loss values for validation at iteration 2198 is: {'loss': 0.54285282, 'mse': 0.70951623}
[2016-04-19 23:48:01] INFO: Optimizer - Training score at iteration 2200: {'loss': array(0.21067523956298828, dtype=float32), 'mse': array(0.45899373292922974, dtype=float32)}
[2016-04-19 23:48:22] INFO: Optimizer - Training score at iteration 2220: {'loss': array(0.14619052410125732, dtype=float32), 'mse': array(0.3823486864566803, dtype=float32)}
[2016-04-19 23:48:42] INFO: Optimizer - Training score at iteration 2240: {'loss': array(0.24176926910877228, dtype=float32), 'mse': array(0.4917003810405731, dtype=float32)}
[2016-04-19 23:49:03] INFO: Optimizer - Training score at iteration 2260: {'loss': array(0.15263037383556366, dtype=float32), 'mse': array(0.39067935943603516, dtype=float32)}
[2016-04-19 23:49:24] INFO: Optimizer - Training score at iteration 2280: {'loss': array(0.2687004506587982, dtype=float32), 'mse': array(0.5183632373809814, dtype=float32)}
[2016-04-19 23:49:44] INFO: Optimizer - Training score at iteration 2300: {'loss': array(0.05553796514868736, dtype=float32), 'mse': array(0.2356649488210678, dtype=float32)}
[2016-04-19 23:50:05] INFO: Optimizer - Training score at iteration 2320: {'loss': array(0.2853468358516693, dtype=float32), 'mse': array(0.534178614616394, dtype=float32)}
[2016-04-19 23:50:26] INFO: Optimizer - Training score at iteration 2340: {'loss': array(0.16006968915462494, dtype=float32), 'mse': array(0.4000870883464813, dtype=float32)}
[2016-04-19 23:50:46] INFO: Optimizer - Training score at iteration 2360: {'loss': array(0.12721064686775208, dtype=float32), 'mse': array(0.3566659986972809, dtype=float32)}
[2016-04-19 23:51:07] INFO: Optimizer - Training score at iteration 2380: {'loss': array(0.20934520661830902, dtype=float32), 'mse': array(0.4575425684452057, dtype=float32)}
[2016-04-19 23:51:47] INFO: Optimizer - Mean loss values for validation at iteration 2398 is: {'loss': 0.56484902, 'mse': 0.72284573}
[2016-04-19 23:51:48] INFO: Optimizer - Training score at iteration 2400: {'loss': array(0.17778311669826508, dtype=float32), 'mse': array(0.42164334654808044, dtype=float32)}
[2016-04-19 23:52:09] INFO: Optimizer - Training score at iteration 2420: {'loss': array(0.3486189544200897, dtype=float32), 'mse': array(0.5904396176338196, dtype=float32)}
[2016-04-19 23:52:29] INFO: Optimizer - Training score at iteration 2440: {'loss': array(0.27928268909454346, dtype=float32), 'mse': array(0.5284720063209534, dtype=float32)}
[2016-04-19 23:52:50] INFO: Optimizer - Training score at iteration 2460: {'loss': array(0.21324078738689423, dtype=float32), 'mse': array(0.46178001165390015, dtype=float32)}
[2016-04-19 23:53:10] INFO: Optimizer - Training score at iteration 2480: {'loss': array(0.1171937808394432, dtype=float32), 'mse': array(0.3423357605934143, dtype=float32)}
[2016-04-19 23:53:31] INFO: Optimizer - Training score at iteration 2500: {'loss': array(0.23245947062969208, dtype=float32), 'mse': array(0.48214051127433777, dtype=float32)}
[2016-04-19 23:53:51] INFO: Optimizer - Training score at iteration 2520: {'loss': array(0.14554481208324432, dtype=float32), 'mse': array(0.3815033435821533, dtype=float32)}
[2016-04-19 23:54:12] INFO: Optimizer - Training score at iteration 2540: {'loss': array(0.2858799993991852, dtype=float32), 'mse': array(0.5346774458885193, dtype=float32)}
[2016-04-19 23:54:32] INFO: Optimizer - Training score at iteration 2560: {'loss': array(0.2444666624069214, dtype=float32), 'mse': array(0.49443569779396057, dtype=float32)}
[2016-04-19 23:54:53] INFO: Optimizer - Training score at iteration 2580: {'loss': array(0.0677102729678154, dtype=float32), 'mse': array(0.2602119743824005, dtype=float32)}
[2016-04-19 23:55:32] INFO: Optimizer - Mean loss values for validation at iteration 2597 is: {'loss': 0.53611761, 'mse': 0.70398152}
[2016-04-19 23:55:34] INFO: Optimizer - Training score at iteration 2600: {'loss': array(0.19834691286087036, dtype=float32), 'mse': array(0.44536155462265015, dtype=float32)}
[2016-04-19 23:55:55] INFO: Optimizer - Training score at iteration 2620: {'loss': array(0.30919942259788513, dtype=float32), 'mse': array(0.5560570359230042, dtype=float32)}
[2016-04-19 23:56:15] INFO: Optimizer - Training score at iteration 2640: {'loss': array(0.12074494361877441, dtype=float32), 'mse': array(0.34748372435569763, dtype=float32)}
[2016-04-19 23:56:36] INFO: Optimizer - Training score at iteration 2660: {'loss': array(0.30715471506118774, dtype=float32), 'mse': array(0.5542153716087341, dtype=float32)}
[2016-04-19 23:56:56] INFO: Optimizer - Training score at iteration 2680: {'loss': array(0.26217448711395264, dtype=float32), 'mse': array(0.512029767036438, dtype=float32)}
[2016-04-19 23:57:17] INFO: Optimizer - Training score at iteration 2700: {'loss': array(0.15634217858314514, dtype=float32), 'mse': array(0.3954012989997864, dtype=float32)}
[2016-04-19 23:57:37] INFO: Optimizer - Training score at iteration 2720: {'loss': array(0.11893228441476822, dtype=float32), 'mse': array(0.3448655903339386, dtype=float32)}
[2016-04-19 23:57:58] INFO: Optimizer - Training score at iteration 2740: {'loss': array(0.22176508605480194, dtype=float32), 'mse': array(0.4709194004535675, dtype=float32)}
[2016-04-19 23:58:18] INFO: Optimizer - Training score at iteration 2760: {'loss': array(0.16015128791332245, dtype=float32), 'mse': array(0.400189071893692, dtype=float32)}
[2016-04-19 23:58:39] INFO: Optimizer - Training score at iteration 2780: {'loss': array(0.06503349542617798, dtype=float32), 'mse': array(0.25501665472984314, dtype=float32)}
[2016-04-19 23:59:18] INFO: Optimizer - Mean loss values for validation at iteration 2797 is: {'loss': 0.61699343, 'mse': 0.75826281}
[2016-04-19 23:59:20] INFO: Optimizer - Training score at iteration 2800: {'loss': array(0.14425744116306305, dtype=float32), 'mse': array(0.3798123896121979, dtype=float32)}
[2016-04-19 23:59:41] INFO: Optimizer - Training score at iteration 2820: {'loss': array(0.1351417750120163, dtype=float32), 'mse': array(0.36761632561683655, dtype=float32)}
[2016-04-20 00:00:01] INFO: Optimizer - Training score at iteration 2840: {'loss': array(0.3026054799556732, dtype=float32), 'mse': array(0.5500958561897278, dtype=float32)}
[2016-04-20 00:00:22] INFO: Optimizer - Training score at iteration 2860: {'loss': array(0.2627353072166443, dtype=float32), 'mse': array(0.5125771164894104, dtype=float32)}
[2016-04-20 00:00:29] INFO: Pipeline - Stopping.

In [ ]:

    

%matplotlib inline
import matplotlib.pyplot as plt
l = np.array([s["loss"] for s in optimizer.losses])
e = np.array([s["mse"] for s in optimizer.losses])
print l.mean()
plt.plot(l)
plt.show()
plt.plot(e)
plt.show()


    

In [ ]:

    

import h5py, numpy as np
f = h5py.File("/home/ga29mix/nashome/data/nyu_depth_v2_combined_50.hdf5")
b = int(f["images"].shape[0] * 0.9)
images = np.array(f["images"][b:])
depths = np.array(f["depths"][b:])
print images.shape
mean = np.load("/home/ga29mix/nashome/data/nyu_depth_v2_combined_50.npy")


    

In [4]:

    

g = build_u_graph()
g.load_weights("/data/vnet2_crop_start_with_120k_iter_90000.zip")
g.compile()


    

    




[2016-04-19 22:24:34] INFO: Graph - Loading parameters from file '/data/vnet2_crop_start_with_120k_iter_90000.zip'
[2016-04-19 22:24:34] INFO: Graph - Setting up graph
[2016-04-19 22:24:34] INFO: Node - data has shape (-1, 3, 240, 320)
[2016-04-19 22:24:34] INFO: Node - label has shape (-1, 1, 240, 320)
[2016-04-19 22:24:34] INFO: Node - conv_1 has shape (-1, 64, 240, 320)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - conv_2 has shape (-1, 64, 240, 320)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - pool_2 has shape (-1, 64, 120, 160)
[2016-04-19 22:24:34] INFO: Pool - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - conv_3 has shape (-1, 128, 120, 160)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - conv_4 has shape (-1, 128, 120, 160)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - pool_4 has shape (-1, 128, 60, 80)
[2016-04-19 22:24:34] INFO: Pool - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - conv_5 has shape (-1, 256, 60, 80)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - conv_6 has shape (-1, 256, 60, 80)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - pool_6 has shape (-1, 256, 30, 40)
[2016-04-19 22:24:34] INFO: Pool - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - conv_7 has shape (-1, 512, 30, 40)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - conv_8 has shape (-1, 512, 30, 40)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - pool_8 has shape (-1, 512, 15, 20)
[2016-04-19 22:24:34] INFO: Pool - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - fl has shape (-1, 153600)
[2016-04-19 22:24:34] INFO: Node - fc_8 has shape (-1, 4096)
[2016-04-19 22:24:34] INFO: Node - dp_8 has shape (-1, 4096)
[2016-04-19 22:24:34] INFO: Node - fc_9 has shape (-1, 19200)
[2016-04-19 22:24:34] INFO: Node - dp_9 has shape (-1, 19200)
[2016-04-19 22:24:34] INFO: Node - rs_10 has shape (-1, 64, 15, 20)
[2016-04-19 22:24:34] INFO: Node - up_11 has shape (-1, 64, 30, 40)
[2016-04-19 22:24:34] INFO: Node - conv_11 has shape (-1, 512, 30, 40)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - concat_11 has shape (-1, 1024, 30, 40)
[2016-04-19 22:24:34] INFO: Node - conv_12 has shape (-1, 512, 30, 40)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - conv_13 has shape (-1, 512, 30, 40)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - up_14 has shape (-1, 512, 60, 80)
[2016-04-19 22:24:34] INFO: Node - conv_14 has shape (-1, 256, 60, 80)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - concat_14 has shape (-1, 512, 60, 80)
[2016-04-19 22:24:34] INFO: Node - conv_15 has shape (-1, 256, 60, 80)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - conv_16 has shape (-1, 256, 60, 80)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - up_17 has shape (-1, 256, 120, 160)
[2016-04-19 22:24:34] INFO: Node - conv_17 has shape (-1, 128, 120, 160)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - concat_17 has shape (-1, 256, 120, 160)
[2016-04-19 22:24:34] INFO: Node - conv_18 has shape (-1, 128, 120, 160)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - conv_19 has shape (-1, 128, 120, 160)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - up_20 has shape (-1, 128, 240, 320)
[2016-04-19 22:24:34] INFO: Node - conv_20 has shape (-1, 64, 240, 320)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - concat_20 has shape (-1, 128, 240, 320)
[2016-04-19 22:24:34] INFO: Node - conv_21 has shape (-1, 64, 240, 320)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - conv_22 has shape (-1, 64, 240, 320)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - conv_23 has shape (-1, 1, 240, 320)
[2016-04-19 22:24:34] INFO: Conv2D - Using DNN CUDA Module
[2016-04-19 22:24:34] INFO: Node - loss has shape (1,)
[2016-04-19 22:24:34] INFO: Node - mse has shape (1,)
[2016-04-19 22:24:54] INFO: Graph - Invoking Theano compiler

In [ ]:

    

%matplotlib inline
import matplotlib.pyplot as plt
from deepgraph.nn.core import Dropout

plot = True
idx = 0
diffs = []
Dropout.set_dp_off()
for image in images:
    tmp = image.astype(np.float32)
    tmp -= mean
    tmp *= 1.0 / 72.240064849107824
    
    res = g.infer([tmp.reshape((1,3,240,320))])["conv_23"]
    res = res.squeeze()
    depth = depths[idx]
    if plot and idx % 9 == 0:
        
        plt.imshow(image.transpose((1,2,0)).astype(np.uint8))
        plt.show()
        plt.imshow(depth)
        plt.show()
        plt.imshow(res)
        plt.show()
        print "RMSE: " + str(np.sqrt(np.mean((res-depth)**2)))
    diffs.append(res - depth)
    
    idx += 1
    
diffs = np.array(diffs)
rmse = np.sqrt(np.mean(diffs ** 2))
print "Accumulated RMSE: " + str(rmse)


    

In [53]:

    

import urllib, cStringIO
from PIL import Image
%matplotlib inline
import matplotlib.pyplot as plt
from deepgraph.nn.core import Dropout


Dropout.set_dp_off()


urls = [
    'http://www.hotel-im-wasserturm.de/fileadmin/_processed_/csm_Deluxe_Doppelzimmer_2_6455fa52be.jpg',
    'http://hotel-airport-zuerich.dorint.com/fileadmin/_processed_/csm_ZRH_Zimmer_DZ_04_bc3a12f87e.jpg',
    'http://www.erlebnis-erlensee.de/images/bilder/partyservice/schnitzel.jpg',
    'http://www.hankewitz.com/wp-content/uploads/2012/12/bolognese21.jpg'
       ]
for URL in urls:
    file = cStringIO.StringIO(urllib.urlopen(URL).read())
    img = Image.open(file)

    rs = img.resize((320,240))
    mean = np.load("/home/ga29mix/nashome/data/nyu_depth_v2_combined_50.npy")
    arr = np.array(rs).transpose((2,0,1)).astype(np.float32)
    arr -= mean
    arr *= 1.0 / 72.240064849107824
    res = g.infer([arr.reshape((1,3,240,320))])["conv_23"]
    # Montage to manipulate color scale
    mmax = 8.0
    # res[0,0,0,0] = 0.0
    # res[0,0,0,1] = mmax
    plt.imshow(rs)
    plt.show()
    plt.imshow(res.squeeze().clip(0.0,mmax))
    plt.show()

    print res.max()


    

    













    













    




3.8033






    













    













    




8.92005






    













    













    




4.18137






    













    













    




3.20027

In [28]:

    

import urllib, cStringIO
from PIL import Image
%matplotlib inline
import matplotlib.pyplot as plt
from deepgraph.nn.core import Dropout
import numpy as np

Dropout.set_dp_off()

data = np.load('/home/ga29mix/nashome/data/kinectv2_2016_03_30_13_33_25_bgr.npy')[:,:,0:3]
dd = np.load('/home/ga29mix/nashome/data/kinectv2_2016_03_30_13_33_25_depth.npy')

img = Image.fromarray(np.array(data[:,:,::-1]))
rs = img.resize((320,240))
mean = np.load("/home/ga29mix/nashome/data/nyu_depth_v2_combined_50.npy")
arr = np.array(rs).transpose((2,0,1)).astype(np.float32)
arr -= mean
arr *= 1.0 / 72.240064849107824
res = g.infer([arr.reshape((1,3,240,320))])["conv_23"]

plt.imshow(img)
plt.show()
plt.imshow(res.squeeze())
plt.show()
plt.imshow(dd)
plt.show()


    

In [ ]: