In [1]:

    

import sys
sys.path.append('../src')
import settings
sys.path.append(settings.CAFFE_PYTHON_PATH)
import caffe
import dataset
import cub_utils
import utils
import rects
import parts
import storage
import numpy as np
import matplotlib.pylab as plt
import skimage
import cv2
import scipy.stats
%matplotlib inline


    

In [2]:

    

cub = dataset.CUB_200_2011(settings.CUB_ROOT)
all_img_infos = cub.get_all_image_infos()
all_seg_infos = cub.get_all_segmentation_infos()
IDtrain, IDtest = cub.get_train_test_id()
cub_parts = cub.get_parts()
cub_bbox = cub.get_bbox()


    

In [3]:

    

def vis_anno(img_id):
    fig = plt.figure(figsize=(18, 6))
    ax1 = fig.add_subplot(131)
    ax2 = fig.add_subplot(132)
    ax3 = fig.add_subplot(133)


    img = caffe.io.load_image(all_img_infos[img_id])
    seg = caffe.io.load_image(all_seg_infos[img_id])
    img_b = utils.draw_bbox(img, cub_bbox[img_id - 1], color=(1, 0, 0), width=4)

    img_p = cub_parts.for_image(img_id)
    img_p.draw_part(ax2)

    ax1.imshow(img_b)
    ax1.set_title('Bounding box')
    ax2.imshow(img)
    ax2.set_title('Parts')
    ax3.imshow(seg)
    ax3.set_title('Segmentation')


    

In [4]:

    

vis_anno(3761)
vis_anno(1020)
vis_anno(9845)


    

In [35]:

    

knnresults = [34.63, 32.86, 35.98, 38.05, 38.98, 40.33, 40.21]
ks = [1, 3, 5, 7, 10, 15, 20]
with plt.style.context('bmh'):
    fig = plt.figure(figsize=(10, 4))
    ax = fig.add_subplot(111)
    ax.plot(range(len(ks)), knnresults, linewidth=3.0)
    ax.set_xticklabels(ks)
    ax.set_ylabel('mean accuracy (%)')
    ax.set_xlabel('K')


    

In [37]:

    

mAs = np.array([0.653106, 0.653106, 0.653106, 0.653183, 0.657028, 0.663286, 0.648536, 0.593018]) * 100
Cs = ['$10^{1}$', '$10^{0}$', '$10^{-1}$', '$10^{-2}$', '$10^{-3}$', '$10^{-4}$', '$10^{-5}$', '$10^{-6}$']
with plt.style.context('bmh'):
    fig = plt.figure(figsize=(10, 4))
    ax = fig.add_subplot(111)
    ax.plot(range(len(mAs)), mAs, linewidth=3.0)
    ax.set_xticklabels(Cs)
    ax.set_ylabel('mean accuracy (%)')
    ax.set_ylim(bottom=55)
    ax.set_xlabel('C')
    for tick in ax.xaxis.get_major_ticks():
        tick.label.set_fontsize(12)


    

In [5]:

    

ss_storage = storage.datastore(settings.storage('ss-cache'))
final_storage = storage.datastore(settings.storage('nn-cache'))
ss_net_name = 'f_cccftt_1-60000'
ssfeature_loader_deep = cub_utils.DeepSSFeatureLoader(cub, ss_storage,
                                                 net=cub_utils.DeepHelper.get_custom_net(settings.model(ss_net_name), settings.pretrained(ss_net_name)),
                                                 net_name=ss_net_name, layer_name='pool5')
ssfeature_loader_hog = cub_utils.HOGSSFeatureLoader(cub, ss_storage)

nn_finder_deep = cub_utils.NNFinder(final_storage, ssfeature_loader_deep, cub, normalize=True)
nn_finder_hog = cub_utils.NNFinder(final_storage, ssfeature_loader_hog, cub, normalize=True)


    

In [9]:

    

nn_finder_deep.setup()
nn_finder_hog.setup()


    

In [10]:

    

nn_finder_deep._pre_calculate(force=True)


    

In [11]:

    

nn_finder_hog._pre_calculate(force=True)


    

In [13]:

    

def vis_neighbors(nn_finder, tid, n_neighbors=3):
    o_img = caffe.io.load_image(all_img_infos[IDtest[tid]])
    neighbors = nn_finder.nn_model.kneighbors(nn_finder.Xtest[tid], n_neighbors, return_distance=False)[0]
    fig, axes = plt.subplots(nrows=1, ncols=n_neighbors+1, figsize=(5*(n_neighbors+1), 5))
    axes[0].imshow(o_img)
    axes[0].axis('off')
    for i in range(n_neighbors):
        axes[i+1].imshow(caffe.io.load_image(all_img_infos[IDtrain[neighbors[i]]]))
        axes[i+1].axis('off')


    

In [28]:

    

I = 145
vis_neighbors(nn_finder_hog, I)
vis_neighbors(nn_finder_deep, I)


    

In [93]:

    

vis_neighbors(nn_finder, 845)
vis_neighbors(nn_finder, 2652+26)


    

In [96]:

    

nn_finder = cub_utils.NNFinder(final_storage, ssfeature_loader_hog, cub, normalize=True, n_neighbors=4)


    

In [98]:

    

nn_finder.setup()
nn_finder._pre_calculate(force=True)


    

In [104]:

    

vis_neighbors(nn_finder, 10)
vis_neighbors(nn_finder, 4)
vis_neighbors(nn_finder, 298)


    

In [106]:

    

vis_neighbors(nn_finder, 407)


    

In [4]:

    

BRGh = rects.BerkeleyRG(settings.BERKELEY_ANNOTATION_BASE_PATH, cub, 'head')
BRGh.setup()

BRGb = rects.BerkeleyRG(settings.BERKELEY_ANNOTATION_BASE_PATH, cub, 'body')
BRGb.setup()

BRGw = rects.BerkeleyRG(settings.BERKELEY_ANNOTATION_BASE_PATH, cub, 'bbox')


    

In [5]:

    

ss_storage = storage.datastore(settings.storage('ss-cache'))
final_storage = storage.datastore(settings.storage('nn-cache'))
ss_net_name = 'f_cccftt_1-60000'
ssfeature_loader_deep = cub_utils.DeepSSFeatureLoader(cub, ss_storage,
                                                 net=cub_utils.DeepHelper.get_custom_net(settings.model(ss_net_name), settings.pretrained(ss_net_name)),
                                                 net_name=ss_net_name, layer_name='pool5')
ssfeature_loader_hog = cub_utils.HOGSSFeatureLoader(cub, ss_storage)

nn_finder = cub_utils.NNFinder(final_storage, ssfeature_loader_deep, cub, normalize=True)
NRGh = rects.NonparametricRG(nn_finder, BRGh, cub)
NRGb = rects.NonparametricRG(nn_finder, BRGb, cub)

RFRGb = rects.RandomForestRG(storage.datastore(settings.storage('rf')), BRGb,
                             cub_utils.DeepHelper.get_bvlc_net(), 'caffenet', cub, random_state=313, point_gen_strategy='unif',
                             use_seg=True, pt_n_part=20, pt_n_bg=100)

RFRGh = rects.RandomForestRG(storage.datastore(settings.storage('rf')), BRGh,
                             cub_utils.DeepHelper.get_bvlc_net(), 'caffenet', cub, random_state=313, point_gen_strategy='unif',
                             use_seg=True, pt_n_part=20, pt_n_bg=100)

RFRGw = rects.RandomForestRG(storage.datastore(settings.storage('rf')), BRGw,
                             cub_utils.DeepHelper.get_bvlc_net(), 'def', cub, random_state=313, point_gen_strategy='unif',
                             use_seg=True, pt_n_part=20, pt_n_bg=100)


    

In [6]:

    

RFRGh.setup()


    

In [7]:

    

RFRGb.setup()


    

In [9]:

    

RFRGw.setup()


    

In [8]:

    

NRGh.setup()


    

In [16]:

    

def draw_box_from_points(i):
    parts = cub_parts.for_image(i)
    img = caffe.io.load_image(all_img_infos[i])

    fig = plt.figure(figsize=(10, 10))
    ax = fig.add_subplot(111)

    body = BRGb.generate(i)
    head = BRGh.generate(i)

    img = body.draw_rect(img, color=(1, 0, 0), width=2)
    img = head.draw_rect(img, color=(0, 1, 0), width=2)


    parts.draw_part(ax)
    ax.imshow(img)
    ax.axis('off')


    

In [17]:

    

draw_box_from_points(842)


    

In [28]:

    

draw_box_from_points(1)


    

In [31]:

    

draw_box_from_points(52)


    

In [40]:

    

draw_box_from_points(8823)


    

In [9]:

    

RFRGh.generate(1, max_prob_lower=0.5)


    

    
Out[9]:





Rect: 	 xmin:83 	 xmax:124 	 ymin:139 	 ymax:205 		 info:Found using RandomForestRG(lf:BerkeleyRG(head)(Oracle), net:caffenet, ntree:10, maxd:10, rands:313, ptgenst:unif, useg:True, ptnprt:20, ptnbg:100)

In [69]:

    

for v in RFRGh.net.blobs.keys():
    print v, RFRGh.net.blobs[v].data.shape


    

    




data (10, 3, 227, 227)
conv1 (10, 96, 55, 55)
pool1 (10, 96, 27, 27)
norm1 (10, 96, 27, 27)
conv2 (10, 256, 27, 27)
pool2 (10, 256, 13, 13)
norm2 (10, 256, 13, 13)
conv3 (10, 384, 13, 13)
conv4 (10, 384, 13, 13)
conv5 (10, 256, 13, 13)
pool5 (10, 256, 6, 6)
fc6 (10, 4096, 1, 1)
fc7 (10, 4096, 1, 1)
fc8 (10, 1000, 1, 1)
prob (10, 1000, 1, 1)

In [75]:

    

plt.imshow(RFRGh.net.deprocess('data', RFRGh.net.blobs['data'].data[0]))
plt.axis('off')


    

    
Out[75]:





(-0.5, 226.5, 226.5, -0.5)






    

In [123]:

    

l = 'conv5'
ind = 0
fig = plt.figure()
ax = fig.add_subplot(111)
ax.matshow(RFRGh.net.blobs[l].data[0][ind, :, :])
ax.axis('off')


    

    
Out[123]:





(-0.5, 12.5, 12.5, -0.5)






    

In [120]:

    

def vis(self, img_info, is_path=False, max_prob_lower=None):
    if is_path:
        img = caffe.io.load_image(img_info)
    else:
        img = caffe.io.load_image(self.all_image_infos[img_info])

    img_r = skimage.transform.resize(img, self.resize_dim)
    X = self._features_for_image(img)
    preds, preds_prob = self._predict_feature_of_image(X, max_prob_lower, return_prob=True)

    fig = plt.figure(figsize=(15, 5))

    preds = self._post_process_preds(preds)
    rect = self._find_rect_from_preds(preds)
    rect.denorm_for_size(img.shape, self.resize_dim[0])
    
    preds_prob_resize = cv2.resize(preds_prob, (img.shape[1], img.shape[0]))

    if not is_path:
        oracle_rect = self.learn_from.generate(img_info)
#         img = oracle_rect.draw_rect(img, color=(0, 1, 0))
#     img = rect.draw_rect(img, color=(1, 0, 0), width=2)

    img_gray = skimage.color.rgb2gray(img)

    ax = fig.add_subplot(121)
    ax.imshow(img)
    ax.axis('off')

    ax = fig.add_subplot(122)
    cax = ax.matshow(preds_prob_resize, cmap=plt.cm.Reds, alpha=1)
    ax.imshow(img_gray, alpha=0.3, cmap=plt.cm.gray)
    
    fig.colorbar(cax)
    ax.axis('off')


    

In [123]:

    

vis(RFRGh, '/home/ipl/Desktop/four.png', is_path=True)


    

In [95]:

    

def vis(self, img_info, is_path=False, max_prob_lower=None):
    
    def _find_rect_from_preds(preds):
        """
        returns the -1 rect if there is no rect found
        """
        L, N = skimage.measure.label(preds, return_num=True, background=0)
        if N > 0:
            L_no_bg = L[L != -1].flatten()
            vals, counts = scipy.stats.mode(L_no_bg)
            part_label = int(vals[0])

            indices = np.where(L == part_label)
            xmin = indices[0].min()
            xmax = indices[0].max()
            ymin = indices[1].min()
            ymax = indices[1].max()

            return rects.Rect(xmin, xmax, ymin, ymax, 'Found using %s' % self.get_name()), L
        else:
            return rects.Rect(-1, -1, -1, -1, '%s - Not found any rect.' % self.get_name()), L
        
    if is_path:
        img = caffe.io.load_image(img_info)
    else:
        img = caffe.io.load_image(self.all_image_infos[img_info])

    img_r = skimage.transform.resize(img, self.resize_dim)
    X = self._features_for_image(img)
    preds, preds_prob = self._predict_feature_of_image(X, max_prob_lower, return_prob=True)

    fig = plt.figure(figsize=(24, 5))

    ax = fig.add_subplot(143)
    ax.matshow(preds)
    ax.axis('off')

    preds = self._post_process_preds(preds)
    rect, L = _find_rect_from_preds(preds)
    rect.denorm_for_size(img.shape, self.resize_dim[0])

    if not is_path:
        oracle_rect = self.learn_from.generate(img_info)
    img = rect.draw_rect(img, color=(1, 0, 0), width=2)

    ax = fig.add_subplot(141)
    ax.imshow(img)
    ax.axis('off')

    ax = fig.add_subplot(142)
    cax = ax.matshow(preds_prob)
    fig.colorbar(cax)
    ax.axis('off')

    ax = fig.add_subplot(144)
    ax.matshow(L)
    ax.axis('off')


    

In [119]:

    

vis(RFRGh, 1204)


    

In [5]:

    

noise_vals = np.array([0, 1, 5, 10, 15, 20])
means = np.array([78.08, 77.74, 74.51, 68.39, 62.70, 58.68])
stds = np.array([0, 0.11, 0.20, 0.30, 0.38, 0.14])

npt_res = np.array([68.56] * noise_vals.shape[0])


    

In [9]:

    

with plt.style.context('bmh'):
    fig = plt.figure(figsize=(10, 4))
    ax = fig.add_subplot(111)
    ax.plot(range(1, noise_vals.shape[0]+1), npt_res, linewidth=3.0, label='DNPT(pool5)')
    ax.errorbar(range(1, noise_vals.shape[0]+1), means, yerr=stds, label='DNPT(oracle) + noise', ls='--')
    ax.set_xticklabels(noise_vals)
    ax.set_ylabel('mean accuracy (%)')
    ax.set_ylim(bottom=55)
    ax.set_xlabel('x: noise variance')
    ax.legend(loc=1)
    for tick in ax.xaxis.get_major_ticks():
        tick.label.set_fontsize(12)


    

In [64]:

    

for i in range(750, 800):
    I = IDtest[i]
    img = caffe.io.load_image(all_img_infos[I])

    head = RFRGh.generate(I)
    body = RFRGb.generate(I)
    bbox = RFRGw.generate(I)

    img = bbox.draw_rect(img, (0,1,1))
    img = body.draw_rect(img, (0,1,0))
    img = head.draw_rect(img, (1,0,0))


    fig = plt.figure(figsize=(10, 7))
    ax = fig.add_subplot(111)
    ax.imshow(img)
    ax.axis('off')


    

In [ ]: