In [1]:
import sys
sys.path.append('../src/')
sys.path.append('/home/ipl/installs/caffe-rc/python/')
from matplotlib import pylab as plt
from datetime import datetime as dt
import numpy as np
import caffe
import cv2
from deep_extractor import CNN_Features_CAFFE_REFERENCE
from dataset import CUB_200_2011
from storage import datastore
import utils
import settings
import parts
import cub_utils
import rects
import skimage.transform
%matplotlib inline
In [28]:
cub = CUB_200_2011(settings.CUB_ROOT)
all_image_infos = cub.get_all_image_infos()
all_seg_infos = cub.get_all_segmentation_infos()
IDtrain, IDtest = cub.get_train_test_id()
cub_parts = cub.get_parts()
dh = cub_utils.DeepHelper()
bah = cub_utils.BerkeleyAnnotationsHelper(settings.BERKELEY_ANNOTATION_BASE_PATH, IDtrain, IDtest)
bboxes = cub.get_bbox()
BRGh = rects.BerkeleyRG(settings.BERKELEY_ANNOTATION_BASE_PATH, cub, 'head')
BRGh.setup()
BRGb = rects.BerkeleyRG(settings.BERKELEY_ANNOTATION_BASE_PATH, cub, 'body')
BRGb.setup()
BRGw = rects.BerkeleyRG(settings.BERKELEY_ANNOTATION_BASE_PATH, cub, 'bbox')
BRGw.setup()
SRGh = rects.SharifRG(cub, 'head', alpha=0.666)
SRGh.setup()
In [3]:
ss_storage = datastore(settings.storage('ss-cache'))
final_storage = datastore(settings.storage('nn-cache'))
ss_net_name = 'cccft2st-50000'
ssfeature_loader_deep = cub_utils.DeepSSFeatureLoader(cub, ss_storage,
net=cub_utils.DeepHelper.get_custom_net(settings.model(ss_net_name), settings.pretrained(ss_net_name)),
net_name=ss_net_name, layer_name='pool5')
ssfeature_loader_hog = cub_utils.HOGSSFeatureLoader(cub, ss_storage)
nn_finder = cub_utils.NNFinder(final_storage, ssfeature_loader_deep, cub, normalize=True)
NRGh = rects.NonparametricRG(nn_finder, BRGh, cub)
NRGb = rects.NonparametricRG(nn_finder, BRGb, cub)
RFRGb = rects.RandomForestRG(datastore(settings.storage('rf')), BRGb,
cub_utils.DeepHelper.get_bvlc_net(), 'caffenet', cub, random_state=313, point_gen_strategy='unif',
use_seg=True, pt_n_part=20, pt_n_bg=100)
RFRGh = rects.RandomForestRG(datastore(settings.storage('rf')), BRGh,
cub_utils.DeepHelper.get_bvlc_net(), 'caffenet', cub, random_state=313, point_gen_strategy='unif',
use_seg=True, pt_n_part=20, pt_n_bg=100)
In [4]:
NRGh.setup()
NRGb.setup()
In [5]:
# RFRGb.setup()
RFRGh.setup()
In [7]:
I = 1765
RFRGb.vis(I)
RFRGh.vis(I)
estimate_rect = RFRGb.generate(I)
oracle_rect = BRGb.generate(I)
img = caffe.io.load_image(all_image_infos[I])
print estimate_rect.evalIOU(oracle_rect, img.shape)
In [8]:
plt.plot(RFRGb.model_rf.feature_importances_)
Out[8]:
In [12]:
S = 0
for tid in IDtest[:100]:
img = cv2.imread(cub.get_image_info(tid))
oracle = BRGb.generate(tid)
# oracle.expand()
est = RFRGb.generate(tid)
pcp_score = est.evalPCP(oracle, img.shape)
S += pcp_score
print S
print 'Head PCP score is: ', float(S) / IDtest[:100].shape[0]
In [7]:
S = 0
for tid in IDtest:
img = cv2.imread(cub.get_image_info(tid))
oracle = BRGb.generate(tid)
est = NRGb.generate(tid)
pcp_score = est.evalPCP(oracle, img.shape)
S += pcp_score
print S
print 'Body PCP score is: ', float(S) / IDtest.shape[0]
In [50]:
I = IDtest[1822]
N = 36
img = caffe.io.load_image(all_image_infos[I])
seg = cub_utils.thresh_segment_mean(caffe.io.load_image(all_seg_infos[I]))
head = BRGw.generate(I)
img_r = skimage.transform.resize(img, (227, 227))
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(111)
# rects.RandomForestRG._get_unif_points(head, 36, no_side=False).draw_part(ax, 'red')
full_image_rect = rects.Rect(0, img.shape[0], 0, img.shape[1])
# rects.RandomForestRG._get_unif_points(full_image_rect, 100, no_side=True).draw_part(ax, 'blue')
rfrg = rects.RandomForestRG(datastore(''), BRGh, cub_utils.DeepHelper.get_bvlc_net(), 'caffenet', cub, point_gen_strategy='unif')
part_points = rfrg._get_part_points(head, seg, 500)
part_points.draw_part(ax, (1,1,0))
bg_points = rfrg._get_bg_points(head, seg, 100)
bg_points.draw_part(ax, (0, 0, 1))
img = head.draw_rect(img, color=(1, 1, 0))
ax.imshow(img)
ax.axis('off')
Out[50]:
In [5]:
orc_bbs_train = []
est_bbs_train = []
orc_bbs_test = []
est_bbs_test = []
In [13]:
for i in IDtrain:
orc_bbs_train.append(BRGh.generate(i))
est_bbs_train.append(RFRGh.generate(i))
In [14]:
for i in IDtest:
orc_bbs_test.append(BRGh.generate(i))
est_bbs_test.append(RFRGh.generate(i))
In [6]:
import cPickle as pickle
In [7]:
faddr = '/home/ipl/Desktop/bbs.p'
In [18]:
with open(faddr, 'wb') as f:
pickle.dump(orc_bbs_train, f)
pickle.dump(est_bbs_train, f)
pickle.dump(orc_bbs_test, f)
pickle.dump(est_bbs_test, f)
In [8]:
with open(faddr, 'rb') as f:
orc_bbs_train = pickle.load(f)
est_bbs_train = pickle.load(f)
orc_bbs_test = pickle.load(f)
est_bbs_test = pickle.load(f)
In [90]:
# convert the orc_bbs_train and est_bbs_train to Xtrain and ytrain
# how to produce Xtrain: Xtrain are the pool5 features. Here I'm going to use the precomputed one from storage: cache-cccftt
# we should definitely try and compare with different ones like caffenet, or 2st or ...
# convert the orc_bbs_test and est_bbs_test to Xtest and ytest
# train sklearn.linear_model.Ridge with them (only go for one and find out the best alpha)
# then go for all of them using the alpha provided.
In [102]:
def convert_to_y(orc_bbs, est_bbs):
y = []
IDS = []
for i in range(len(orc_bbs)):
if not orc_bbs[i].is_invalid() and not est_bbs[i].is_invalid():
Gx, Gy, Gw, Gh = orc_bbs[i]._get_cendim()
Px, Py, Pw, Ph = est_bbs[i]._get_cendim()
tx = (Gx - Px)
ty = (Gy - Py)
tw = (Gw - Pw)
th = (Gh - Ph)
y.append([tx, ty, tw, th])
IDS.append(i)
# else:
# print i
return np.array(y), IDS
def score(orc_bbs, est_bbs, IDS):
S = 0
for tid in range(IDS.shape[0]):
img = cv2.imread(all_image_infos[IDS[tid]])
oracle = orc_bbs[tid]
est = est_bbs[tid]
pcp_score = est.evalPCP(oracle, img.shape)
S += pcp_score
print 'PCP score is: ', float(S) / IDS.shape[0]
return S
In [103]:
ytrain, yids_train = convert_to_y(orc_bbs_train, est_bbs_train)
tic = dt.now()
print score(orc_bbs_train, est_bbs_train, IDtrain)
print dt.now() - tic
In [104]:
ytest, yids_test = convert_to_y(orc_bbs_test, est_bbs_test)
tic = dt.now()
print score(orc_bbs_test, est_bbs_test, IDtest)
print dt.now() - tic
In [94]:
st = datastore(settings.storage('cache-cccftt'))
st.super_name = 'features'
st.sub_name = 'cccftt-100000'
st.ip = st.get_instance_path(st.super_name, st.sub_name, 'feat_cache_pool5')
feat_pool5 = st.load_large_instance(st.ip, 4)
In [95]:
Xtrain = feat_pool5[IDtrain[yids_train] - 1]
Xtest = feat_pool5[IDtest[yids_test] - 1]
In [96]:
import sklearn.linear_model
import sklearn.metrics
import sklearn.grid_search
In [105]:
model_base = sklearn.linear_model.Ridge()
In [112]:
alphas = {'alpha': [1e4, 1e5, 1e6, 1e7, 1e8, 1e9]}
which_one = 2
scorer = sklearn.metrics.make_scorer(sklearn.metrics.mean_squared_error)
In [113]:
grid_searcher = sklearn.grid_search.GridSearchCV(model_base, alphas, n_jobs=3, cv=5, scoring=scorer)
grid_searcher.fit(Xtrain, ytrain[:, which_one])
print grid_searcher.best_params_
grid_searcher.grid_scores_
Out[113]:
In [135]:
alphas = [1e7, 1e7, 1e7, 1e7]
In [136]:
model_reg = sklearn.linear_model.Ridge(alpha=alphas)
In [137]:
model_reg.fit(Xtrain, ytrain)
Out[137]:
In [138]:
preds = model_reg.predict(Xtest)
In [143]:
sklearn.metrics.mean_squared_error(ytest, preds)
Out[143]:
In [148]:
est_reg_bbs_test = []
for i in range(ytest.shape[0]):
dx = preds[i, 0]
dy = preds[i, 1]
dw = preds[i, 2]
dh = preds[i, 3]
Px, Py, Pw, Ph = est_bbs_test[yids_test[i]]._get_cendim()
centerx = dx + Px
centery = dy + Py
dimw = dw + Pw
dimh = dh + Ph
cendim = (centerx, centery, dimw, dimh)
r = rects.Rect(0, 0, 0, 0, 'rfrg + regression, ftid:%d' % i)
r._set_rect_from_cendim(cendim)
est_reg_bbs_test.append(r)
In [149]:
len(est_reg_bbs_test), ytest.shape
Out[149]:
In [150]:
S = score([orc_bbs_test[i] for i in yids_test], est_reg_bbs_test, IDtest[yids_test])
In [165]:
ftid = 4
def vis_reg_head(ftid):
tid = yids_test[ftid]
img_id = IDtest[tid]
img = caffe.io.load_image(all_image_infos[img_id])
oracle_head = orc_bbs_test[tid]
est_head = est_bbs_test[tid]
est_reg_head = est_reg_bbs_test[ftid]
img = oracle_head.draw_rect(img, color=(0, 1, 0), width=3)
img = est_head.draw_rect(img, color=(1, 0, 0))
img = est_reg_head.draw_rect(img, color=(0, 0, 1))
fig = plt.figure(figsize=(10, 10))
ax = fig.add_subplot(111)
ax.imshow(img)
In [169]:
for i in range(150,151):
vis_reg_head(i)
In [3]:
dh = cub_utils.DeepHelper()
In [4]:
dense_points = cub_utils.gen_dense_points()
In [14]:
I = 48
img = caffe.io.load_image(all_image_infos[I])
img_r = skimage.transform.resize(img, (227, 227))
plt.imshow(img_r)
plt.title(I)
Out[14]:
In [15]:
dh.init_with_image(img)
In [16]:
feats_copy = dh.ffeats.copy()
feats_copy = feats_copy.reshape((227 * 227, 1376))
In [17]:
import sklearn.mixture
import skimage.color
In [29]:
gmm = sklearn.mixture.GMM(n_components=20, covariance_type='diag', random_state=92204744)
In [30]:
gmm.fit(feats_copy)
preds = gmm.predict(feats_copy)
preds_square = preds.reshape((227, 227))
In [31]:
plt.imshow(skimage.color.label2rgb(preds_square, image=skimage.transform.resize(img, (227, 227))))
Out[31]:
In [32]:
plt.matshow(preds_square)
Out[32]:
In [28]:
gmm.converged_
Out[28]:
In [32]:
img_addr = '/home/ipl/Desktop/q.jpg'
rect = RFRGh.generate_addr(img_addr)
head_img = rect.get_rect(cv2.imread(img_addr))
plt.imshow(head_img)
cv2.imwrite('/home/ipl/Desktop/q_h.jpg', head_img)
Out[32]:
In [ ]: