In [1]:
import sys
sys.path.append('../src/')
sys.path.append('/home/ipl/installs/caffe-rc/python/')
from matplotlib import pylab as plt
from datetime import datetime as dt

import numpy as np
import caffe
import cv2

from deep_extractor import CNN_Features_CAFFE_REFERENCE
from dataset import CUB_200_2011
from storage import datastore
import utils
import settings
import parts
import cub_utils
import rects
import skimage.transform

%matplotlib inline

In [28]:
cub = CUB_200_2011(settings.CUB_ROOT)
all_image_infos = cub.get_all_image_infos()
all_seg_infos = cub.get_all_segmentation_infos()
IDtrain, IDtest = cub.get_train_test_id()
cub_parts = cub.get_parts()
dh = cub_utils.DeepHelper()
bah = cub_utils.BerkeleyAnnotationsHelper(settings.BERKELEY_ANNOTATION_BASE_PATH, IDtrain, IDtest)
bboxes = cub.get_bbox()

BRGh = rects.BerkeleyRG(settings.BERKELEY_ANNOTATION_BASE_PATH, cub, 'head')
BRGh.setup()

BRGb = rects.BerkeleyRG(settings.BERKELEY_ANNOTATION_BASE_PATH, cub, 'body')
BRGb.setup()

BRGw = rects.BerkeleyRG(settings.BERKELEY_ANNOTATION_BASE_PATH, cub, 'bbox')
BRGw.setup()

SRGh = rects.SharifRG(cub, 'head', alpha=0.666)
SRGh.setup()

In [3]:
ss_storage = datastore(settings.storage('ss-cache'))
final_storage = datastore(settings.storage('nn-cache'))
ss_net_name = 'cccft2st-50000'
ssfeature_loader_deep = cub_utils.DeepSSFeatureLoader(cub, ss_storage,
                                                 net=cub_utils.DeepHelper.get_custom_net(settings.model(ss_net_name), settings.pretrained(ss_net_name)),
                                                 net_name=ss_net_name, layer_name='pool5')
ssfeature_loader_hog = cub_utils.HOGSSFeatureLoader(cub, ss_storage)

nn_finder = cub_utils.NNFinder(final_storage, ssfeature_loader_deep, cub, normalize=True)
NRGh = rects.NonparametricRG(nn_finder, BRGh, cub)
NRGb = rects.NonparametricRG(nn_finder, BRGb, cub)

RFRGb = rects.RandomForestRG(datastore(settings.storage('rf')), BRGb,
                             cub_utils.DeepHelper.get_bvlc_net(), 'caffenet', cub, random_state=313, point_gen_strategy='unif',
                             use_seg=True, pt_n_part=20, pt_n_bg=100)

RFRGh = rects.RandomForestRG(datastore(settings.storage('rf')), BRGh,
                             cub_utils.DeepHelper.get_bvlc_net(), 'caffenet', cub, random_state=313, point_gen_strategy='unif',
                             use_seg=True, pt_n_part=20, pt_n_bg=100)


---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-3-35696c93ca3b> in <module>()
      3 ss_net_name = 'cccft2st-50000'
      4 ssfeature_loader_deep = cub_utils.DeepSSFeatureLoader(cub, ss_storage,
----> 5                                                  net=cub_utils.DeepHelper.get_custom_net(settings.model(ss_net_name), settings.pretrained(ss_net_name)),
      6                                                  net_name=ss_net_name, layer_name='pool5')
      7 ssfeature_loader_hog = cub_utils.HOGSSFeatureLoader(cub, ss_storage)

/home/ipl/repo/omgh/src/cub_utils.pyc in get_custom_net(model_def, pretrained_file, test_phase, gpu_mode)
     73     @staticmethod
     74     def get_custom_net(model_def, pretrained_file, test_phase=True, gpu_mode=True):
---> 75         net = caffe.Classifier(model_def, pretrained_file, mean=np.load(settings.ILSVRC_MEAN),  channel_swap=(2, 1, 0), raw_scale=255)
     76         if test_phase:
     77             net.set_phase_test()

/home/ipl/installs/caffe-rc/python/caffe/classifier.pyc in __init__(self, model_file, pretrained_file, image_dims, gpu, mean, input_scale, raw_scale, channel_swap)
     24             preprocessing options.
     25         """
---> 26         caffe.Net.__init__(self, model_file, pretrained_file)
     27         self.set_phase_test()
     28 

RuntimeError: Could not open file /home/ipl/installs/caffe-rc/models/finetune_2step_cub_cropped/deploy.prototxt

In [4]:
NRGh.setup()
NRGb.setup()

In [5]:
# RFRGb.setup()
RFRGh.setup()

In [7]:
I = 1765
RFRGb.vis(I)
RFRGh.vis(I)
estimate_rect = RFRGb.generate(I)
oracle_rect = BRGb.generate(I)
img = caffe.io.load_image(all_image_infos[I])
print estimate_rect.evalIOU(oracle_rect, img.shape)


0.212264150943

In [8]:
plt.plot(RFRGb.model_rf.feature_importances_)


Out[8]:
[<matplotlib.lines.Line2D at 0x32c0db50>]

In [12]:
S = 0
for tid in IDtest[:100]:
    img = cv2.imread(cub.get_image_info(tid))
    oracle = BRGb.generate(tid)
#     oracle.expand()
    est = RFRGb.generate(tid)
    
    pcp_score = est.evalPCP(oracle, img.shape)
    S += pcp_score

print S
print 'Head PCP score is: ', float(S) / IDtest[:100].shape[0]


34
Head PCP score is:  0.34

In [7]:
S = 0
for tid in IDtest:
    img = cv2.imread(cub.get_image_info(tid))
    oracle = BRGb.generate(tid)
    est = NRGb.generate(tid)
    
    pcp_score = est.evalPCP(oracle, img.shape)
    S += pcp_score

print S
print 'Body PCP score is: ', float(S) / IDtest.shape[0]


5002
Body PCP score is:  0.863306869175

In [50]:
I = IDtest[1822]
N = 36
img = caffe.io.load_image(all_image_infos[I])
seg = cub_utils.thresh_segment_mean(caffe.io.load_image(all_seg_infos[I]))
head = BRGw.generate(I)

img_r = skimage.transform.resize(img, (227, 227))

fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(111)

# rects.RandomForestRG._get_unif_points(head, 36, no_side=False).draw_part(ax, 'red')
full_image_rect = rects.Rect(0, img.shape[0], 0, img.shape[1])
# rects.RandomForestRG._get_unif_points(full_image_rect, 100, no_side=True).draw_part(ax, 'blue')

rfrg = rects.RandomForestRG(datastore(''), BRGh, cub_utils.DeepHelper.get_bvlc_net(), 'caffenet', cub, point_gen_strategy='unif')

part_points = rfrg._get_part_points(head, seg, 500)
part_points.draw_part(ax, (1,1,0))

bg_points = rfrg._get_bg_points(head, seg, 100)
bg_points.draw_part(ax, (0, 0, 1))

img = head.draw_rect(img, color=(1, 1, 0))
ax.imshow(img)
ax.axis('off')


Out[50]:
(-0.5, 499.5, 366.5, -0.5)

Bounding box regression (HEAD)


In [5]:
orc_bbs_train = []
est_bbs_train = []
orc_bbs_test = []
est_bbs_test = []

In [13]:
for i in IDtrain:
    orc_bbs_train.append(BRGh.generate(i))
    est_bbs_train.append(RFRGh.generate(i))

In [14]:
for i in IDtest:
    orc_bbs_test.append(BRGh.generate(i))
    est_bbs_test.append(RFRGh.generate(i))

In [6]:
import cPickle as pickle

In [7]:
faddr = '/home/ipl/Desktop/bbs.p'

save bbs


In [18]:
with open(faddr, 'wb') as f:
    pickle.dump(orc_bbs_train, f)
    pickle.dump(est_bbs_train, f)
    pickle.dump(orc_bbs_test, f)
    pickle.dump(est_bbs_test, f)

load bbs


In [8]:
with open(faddr, 'rb') as f:
    orc_bbs_train = pickle.load(f)
    est_bbs_train = pickle.load(f)
    orc_bbs_test = pickle.load(f)
    est_bbs_test = pickle.load(f)

In [90]:
# convert the orc_bbs_train and est_bbs_train to Xtrain and ytrain
    # how to produce Xtrain: Xtrain are the pool5 features. Here I'm going to use the precomputed one from storage: cache-cccftt
    # we should definitely try and compare with different ones like caffenet, or 2st or ...
# convert the orc_bbs_test and est_bbs_test to Xtest and ytest
# train sklearn.linear_model.Ridge with them (only go for one and find out the best alpha)
# then go for all of them using the alpha provided.

computing ys


In [102]:
def convert_to_y(orc_bbs, est_bbs):
    y = []
    IDS = []
    for i in range(len(orc_bbs)):
        if not orc_bbs[i].is_invalid() and not est_bbs[i].is_invalid():
            Gx, Gy, Gw, Gh = orc_bbs[i]._get_cendim()
            Px, Py, Pw, Ph = est_bbs[i]._get_cendim()
            tx = (Gx - Px)
            ty = (Gy - Py)
            tw = (Gw - Pw)
            th = (Gh - Ph)
            y.append([tx, ty, tw, th])
            IDS.append(i)
#         else:
#             print i
    return np.array(y), IDS

def score(orc_bbs, est_bbs, IDS):
    S = 0
    for tid in range(IDS.shape[0]):
        img = cv2.imread(all_image_infos[IDS[tid]])
        oracle = orc_bbs[tid]
        est = est_bbs[tid]
        pcp_score = est.evalPCP(oracle, img.shape)
        S += pcp_score

    print 'PCP score is: ', float(S) / IDS.shape[0]
    return S

Score before regression


In [103]:
ytrain, yids_train = convert_to_y(orc_bbs_train, est_bbs_train)
tic = dt.now()
print score(orc_bbs_train, est_bbs_train, IDtrain)
print dt.now() - tic


PCP score is:  0.125291958625
751
0:00:11.912144

In [104]:
ytest, yids_test = convert_to_y(orc_bbs_test, est_bbs_test)
tic = dt.now()
print score(orc_bbs_test, est_bbs_test, IDtest)
print dt.now() - tic


PCP score is:  0.110631687953
641
0:00:11.658548

load pool features


In [94]:
st = datastore(settings.storage('cache-cccftt'))
st.super_name = 'features'
st.sub_name = 'cccftt-100000'
st.ip = st.get_instance_path(st.super_name, st.sub_name, 'feat_cache_pool5')

feat_pool5 = st.load_large_instance(st.ip, 4)

make Xtrain and Xtest


In [95]:
Xtrain = feat_pool5[IDtrain[yids_train] - 1]
Xtest = feat_pool5[IDtest[yids_test] - 1]

Learning Rigde model


In [96]:
import sklearn.linear_model
import sklearn.metrics
import sklearn.grid_search

finding the right alpha


In [105]:
model_base = sklearn.linear_model.Ridge()

In [112]:
alphas = {'alpha': [1e4, 1e5, 1e6, 1e7, 1e8, 1e9]}
which_one = 2
scorer = sklearn.metrics.make_scorer(sklearn.metrics.mean_squared_error)

In [113]:
grid_searcher = sklearn.grid_search.GridSearchCV(model_base, alphas, n_jobs=3, cv=5, scoring=scorer)
grid_searcher.fit(Xtrain, ytrain[:, which_one])
print grid_searcher.best_params_
grid_searcher.grid_scores_


{'alpha': 10000.0}
Out[113]:
[mean: 1372.59259, std: 89.36077, params: {'alpha': 10000.0},
 mean: 934.35009, std: 106.19967, params: {'alpha': 100000.0},
 mean: 646.88183, std: 128.35604, params: {'alpha': 1000000.0},
 mean: 581.73445, std: 141.03780, params: {'alpha': 10000000.0},
 mean: 621.60985, std: 153.87368, params: {'alpha': 100000000.0},
 mean: 677.20526, std: 160.26349, params: {'alpha': 1000000000.0}]

Actually learning the regression model


In [135]:
alphas = [1e7, 1e7, 1e7, 1e7]

In [136]:
model_reg = sklearn.linear_model.Ridge(alpha=alphas)

In [137]:
model_reg.fit(Xtrain, ytrain)


Out[137]:
Ridge(alpha=[10000000.0, 10000000.0, 10000000.0, 10000000.0], copy_X=True,
   fit_intercept=True, max_iter=None, normalize=False, solver='auto',
   tol=0.001)

In [138]:
preds = model_reg.predict(Xtest)

In [143]:
sklearn.metrics.mean_squared_error(ytest, preds)


Out[143]:
908.15015922691475

Make a new est from regression model


In [148]:
est_reg_bbs_test = []
for i in range(ytest.shape[0]):
    dx = preds[i, 0]
    dy = preds[i, 1]
    dw = preds[i, 2]
    dh = preds[i, 3]
    
    Px, Py, Pw, Ph = est_bbs_test[yids_test[i]]._get_cendim()
    
    centerx = dx + Px
    centery = dy + Py
    dimw = dw + Pw
    dimh = dh + Ph
    
    cendim = (centerx, centery, dimw, dimh)
    r = rects.Rect(0, 0, 0, 0, 'rfrg + regression, ftid:%d' % i)
    r._set_rect_from_cendim(cendim)
    est_reg_bbs_test.append(r)

In [149]:
len(est_reg_bbs_test), ytest.shape


Out[149]:
(5785, (5785, 4))

In [150]:
S = score([orc_bbs_test[i] for i in yids_test], est_reg_bbs_test, IDtest[yids_test])


PCP score is:  0.33794295592

vis what the regressor does


In [165]:
ftid = 4
def vis_reg_head(ftid):
    tid = yids_test[ftid]
    img_id = IDtest[tid]

    img = caffe.io.load_image(all_image_infos[img_id])
    oracle_head = orc_bbs_test[tid]
    est_head = est_bbs_test[tid]
    est_reg_head = est_reg_bbs_test[ftid]

    img = oracle_head.draw_rect(img, color=(0, 1, 0), width=3)
    img = est_head.draw_rect(img, color=(1, 0, 0))
    img = est_reg_head.draw_rect(img, color=(0, 0, 1))

    fig = plt.figure(figsize=(10, 10))
    ax = fig.add_subplot(111)

    ax.imshow(img)

In [169]:
for i in range(150,151):
    vis_reg_head(i)


Experiment with segmentation


In [3]:
dh = cub_utils.DeepHelper()

In [4]:
dense_points = cub_utils.gen_dense_points()

In [14]:
I = 48
img = caffe.io.load_image(all_image_infos[I])
img_r = skimage.transform.resize(img, (227, 227))
plt.imshow(img_r)
plt.title(I)


Out[14]:
<matplotlib.text.Text at 0x256e2890>

In [15]:
dh.init_with_image(img)

In [16]:
feats_copy = dh.ffeats.copy()
feats_copy = feats_copy.reshape((227 * 227, 1376))

In [17]:
import sklearn.mixture
import skimage.color

In [29]:
gmm = sklearn.mixture.GMM(n_components=20, covariance_type='diag', random_state=92204744)

In [30]:
gmm.fit(feats_copy)
preds = gmm.predict(feats_copy)
preds_square = preds.reshape((227, 227))

In [31]:
plt.imshow(skimage.color.label2rgb(preds_square, image=skimage.transform.resize(img, (227, 227))))


Out[31]:
<matplotlib.image.AxesImage at 0x2894aa90>

In [32]:
plt.matshow(preds_square)


Out[32]:
<matplotlib.image.AxesImage at 0x2897c210>

In [28]:
gmm.converged_


Out[28]:
True

Other experiments


In [32]:
img_addr = '/home/ipl/Desktop/q.jpg'
rect = RFRGh.generate_addr(img_addr)
head_img = rect.get_rect(cv2.imread(img_addr))
plt.imshow(head_img)
cv2.imwrite('/home/ipl/Desktop/q_h.jpg', head_img)


Out[32]:
True

In [ ]: