In [1]:
import sys
import os
sys.path.append('../src/')
sys.path.append('/home/ipl/installs/caffe-rc/python/')
import matplotlib
%matplotlib inline
from matplotlib import pylab as plt
import numpy as np
import scipy.misc
import scipy.stats
import caffe
import cv2
import re
from sklearn import svm
from sklearn import metrics
import utils
from datetime import datetime as dt
from dataset import CUB_200_2011
from storage import datastore
from deep_extractor import CNN_Features_CAFFE_REFERENCE
from datetime import datetime as dt
import settings
from parts import *
from cub_utils import *
import skimage
In [2]:
cub = CUB_200_2011(settings.CUB_ROOT)
cub_parts = cub.get_parts()
IDtrain, IDtest = cub.get_train_test_id()
all_image_infos = cub.get_all_image_infos()
all_segmentaion_infos = cub.get_all_segmentation_infos()
cub_parts = cub.get_parts()
In [3]:
fig = plt.figure(figsize=(15, 10))
ax_i = fig.add_subplot(121)
ax_s = fig.add_subplot(122)
img_id = IDtrain[3263]
print all_image_infos[img_id]
seg = thresh_segment_mean(caffe.io.load_image(all_segmentaion_infos[img_id]))
img = caffe.io.load_image(all_image_infos[img_id])
parts = cub_parts.for_image(img_id)
parts_head = parts.filter_by_name(Parts.HEAD_PART_NAMES)
parts_head.draw_part(ax_i)
img_d = parts_head.draw_rect(img)
head_positive = gen_part_points(parts_head.get_rect_info(img), seg, parts_head)
head_positive.draw_part(ax_s, 'red')
head_negative = gen_bg_points(parts_head.get_rect_info(img), seg, parts_head)
head_negative.draw_part(ax_s, 'blue')
ax_i.imshow(img_d)
ax_s.imshow(seg)
In [4]:
deep_helper = DeepHelper()
In [11]:
deep_helper.init_with_image(img)
fig = plt.figure(figsize=(20, 5))
I = 1
layer = 'conv4'
feat = deep_helper.feats[layer][:, :, I]
ax = fig.add_subplot(132)
ax.matshow(deep_helper.net.blobs[layer].data[0, I, :, :])
ax = fig.add_subplot(133)
ax.matshow(feat)
ax = fig.add_subplot(131)
ax.imshow(deep_helper.net.deprocess('data', deep_helper.net.blobs['data'].data[0]))
Out[11]:
In [12]:
tic = dt.now()
head_positive.norm_for_size(img.shape[1], img.shape[0], 227)
head_negative.norm_for_size(img.shape[1], img.shape[0], 227)
feats_positive = deep_helper.features(head_positive)
feats_negative = deep_helper.features(head_negative)
print dt.now() - tic
positive_y = np.ones((len(head_positive))) * 1
In [3]:
dh = DeepHelper()
In [8]:
Xtrain, ytrain = dh.part_features_for_rf(all_image_infos, all_segmentaion_infos
, cub_parts, IDtrain, Parts.HEAD_PART_NAMES)
Out[8]:
In [8]:
Xtest, ytest = dh.part_features_for_rf(all_image_infos, all_segmentaion_infos
, cub_parts, IDtest, Parts.HEAD_PART_NAMES)
In [14]:
Xtrain_i_p = '/home/ipl/Desktop/test/Xtrain.mat'
ytrain_i_p = '/home/ipl/Desktop/test/ytrain.mat'
Xtest_i_p = '/home/ipl/Desktop/test/Xtest.mat'
ytest_i_p = '/home/ipl/Desktop/test/ytest.mat'
from storage import datastore
dt = datastore('')
In [ ]:
dt.save_large_instance(Xtrain_i_p, Xtrain, 113)
dt.save_instance(ytrain_i_p, ytrain)
In [12]:
dt.save_large_instance(Xtest_i_p, Xtest, 43)
dt.save_instance(ytest_i_p, ytest)
In [15]:
Xtrain = dt.load_large_instance(Xtrain_i_p, 113)
Xtest = dt.load_large_instance(Xtest_i_p, 43)
ytrain = dt.load_instance(ytrain_i_p)
ytest = dt.load_instance(ytest_i_p)
ytrain = ytrain[0, :]
ytest = ytest[0, :]
In [16]:
import sklearn.ensemble
import sklearn.svm
import sklearn.metrics
In [17]:
model_rf = sklearn.ensemble.RandomForestClassifier(n_estimators=10, bootstrap=False
, max_depth=10, n_jobs=3, random_state=None, verbose=0)
In [18]:
model_rf.fit(Xtrain, ytrain)
Out[18]:
In [19]:
preds_rf = model_rf.predict(Xtest)
In [20]:
sklearn.metrics.accuracy_score(ytest, preds_rf)
Out[20]:
In [21]:
print sklearn.metrics.classification_report(ytest, preds_rf)
In [16]:
model_svm = sklearn.svm.LinearSVC(C=0.001)
model_svm.fit(Xtrain, ytrain)
Out[16]:
In [18]:
preds_svm = model_svm.predict(Xtest)
print sklearn.metrics.classification_report(ytest, preds_svm)
In [19]:
sklearn.metrics.accuracy_score(ytest, preds_svm)
Out[19]:
In [22]:
dense_points = Parts()
for i in range(227):
for j in range(227):
dense_points.append(Part(-1, '?', -1, i, j, 1))
In [23]:
img_id = IDtest[6]
img_id = 3030
def vis_head(img_id):
img = caffe.io.load_image(all_image_infos[img_id])
seg = thresh_segment_mean(caffe.io.load_image(all_segmentaion_infos[img_id]))
parts = cub_parts.for_image(img_id)
parts_head = parts.filter_by_name(Parts.HEAD_PART_NAMES)
dh.init_with_image(img)
X = dh.features(dense_points)
preds_prob = model_rf.predict_proba(X)
max_prob = np.max(preds_prob[:, 1])
preds_prob = preds_prob[:, 1].reshape((227, 227))
img_r = dh.net.deprocess('data', dh.net.blobs['data'].data[0])
preds = model_rf.predict(X)
preds = preds.reshape((227, 227))
preds = preds_prob >= (max_prob/2)
fig = plt.figure(figsize=(10, 10))
ax = fig.add_subplot(221)
ax.imshow(img_r)
ax.set_title(img_id)
ax = fig.add_subplot(224)
cax = ax.matshow(preds_prob.T)
fig.colorbar(cax)
ax = fig.add_subplot(223)
ax.matshow(preds.T)
imga = np.zeros((img_r.shape[0], img_r.shape[1], 4))
imga[:, :, :3] = img_r
imga[:, :, 3] = preds.T
ax = fig.add_subplot(222)
ax.imshow(imga)
for i in [970, 1765, 2490, 2498, 2596, 3642, 3761, 4128, 4144, 4353, 6097, 6122, 6125, 7348, 7919, 8020, 8036, 9446, 9460, 10988]:
vis_head(i)
break
In [24]:
vis_head(2707)
In [25]:
def vis_head_process(img_id):
# img = caffe.io.load_image('/home/ipl/Desktop/pres/Chicken_-_melbourne_show_2005.jpg')
img = caffe.io.load_image(all_image_infos[img_id])
dh.init_with_image(img)
X = dh.features(dense_points)
preds_prob = model_rf.predict_proba(X)
max_prob = np.max(preds_prob[:, 1])
preds_prob = preds_prob[:, 1].reshape((227, 227)).T
img_r = dh.net.deprocess('data', dh.net.blobs['data'].data[0])
preds = preds_prob >= (max_prob/2)
fig = plt.figure(figsize=(10, 10))
ax = fig.add_subplot(223)
ax.set_title('RF output')
ax.matshow(preds)
preds = skimage.morphology.closing(preds, skimage.morphology.square(10))
preds = skimage.morphology.remove_small_objects(preds, min_size=10, connectivity=1)
L, N = skimage.measure.label(preds, return_num=True, background=0)
L_no_bg = L[L!=-1].flatten()
vals, counts = scipy.stats.mode(L_no_bg)
part_label = int(vals[0])
indices = np.where(L==part_label)
xmin = indices[0].min()
xmax = indices[0].max()
ymin = indices[1].min()
ymax = indices[1].max()
rect_info = xmin, xmax, ymin, ymax
pmin = Part(-1, '?', -1, xmin, ymin, 1)
pmax = Part(-1, '?', -1, xmax, ymax, 1)
rect_parts = Parts(parts=[pmin, pmax])
rect_parts.denorm_for_size(img.shape[0], img.shape[1], size=227)
rect_info = rect_parts[0].x, rect_parts[1].x, rect_parts[0].y, rect_parts[1].y
xmin, xmax, ymin, ymax = rect_info
# img_or = cv2.resize(img, (227, 227))
cv2.rectangle(img, (ymin, xmin), (ymax, xmax), 100, 5)
ax = fig.add_subplot(221)
ax.imshow(img)
ax = fig.add_subplot(224)
ax.set_title('Processed')
cax = ax.matshow(L)
imga = np.zeros((img_r.shape[0], img_r.shape[1], 4))
imga[:, :, :3] = img_r
imga[:, :, 3] = preds_prob
ax = fig.add_subplot(222)
ax.imshow(imga)
ax.set_title(img_id)
for i in range(1, 2):
vis_head_process(i)
In [26]:
vis_head_process(2706)
In [20]:
instance_split = 10
rf_safe = datastore(settings.storage('rf'))
rf_safe.super_name = 'features'
rf_safe.sub_name = 'head-points'
rf_safe.other_sub_name = 'head-final-features'
Xtrain_rf_ip = rf_safe.get_instance_path(rf_safe.super_name, rf_safe.sub_name, 'Xtrain_rf')
Xtest_rf_ip = rf_safe.get_instance_path(rf_safe.super_name, rf_safe.sub_name, 'Xtest_rf')
ytrain_rf_ip = rf_safe.get_instance_path(rf_safe.super_name, rf_safe.sub_name, 'ytrain_rf.mat')
ytest_rf_ip = rf_safe.get_instance_path(rf_safe.super_name, rf_safe.sub_name, 'ytest_rf.mat')
Xtrain_ip = rf_safe.get_instance_path(rf_safe.super_name, rf_safe.other_sub_name, 'Xtrain')
Xtest_ip = rf_safe.get_instance_path(rf_safe.super_name, rf_safe.other_sub_name, 'Xtest')
In [21]:
Xtrain_rf = rf_safe.load_large_instance(Xtrain_rf_ip, instance_split)
ytrain_rf = rf_safe.load_instance(ytrain_rf_ip)
ytrain_rf = ytrain_rf[0, :]
In [22]:
features_storage_r = datastore(settings.storage('ccrft'))
feature_extractor_r = CNN_Features_CAFFE_REFERENCE(features_storage_r, make_net=False)
features_storage_c = datastore(settings.storage('cccft'))
feature_extractor_c = CNN_Features_CAFFE_REFERENCE(features_storage_c, make_net=False)
features_storage_p_h = datastore(settings.storage('ccpheadft-100000'))
feature_extractor_p_h = CNN_Features_CAFFE_REFERENCE(features_storage_p_h, make_net=False)
Xtrain_r, ytrain_r, Xtest_r, ytest_r = cub.get_train_test(feature_extractor_r.extract_one)
Xtrain_c, ytrain_c, Xtest_c, ytest_c = cub.get_train_test(feature_extractor_c.extract_one)
Xtrain_p_h, ytrain_p_h, Xtest_p_h, ytest_p_h = cub.get_train_test(feature_extractor_p_h.extract_one)
In [23]:
new_Xtest_p_h = rf_safe.load_large_instance(Xtest_ip, instance_split)
new_Xtrain_p_h = rf_safe.load_large_instance(Xtrain_ip, instance_split)
In [24]:
C = 0.0001
Xtrain = new_Xtrain_p_h
Xtest = new_Xtest_p_h
ytrain = ytrain_r
ytest = ytest_r
model = svm.LinearSVC(C=C)
model.fit(Xtrain, ytrain)
predictions = model.predict(Xtest)
print 'accuracy', metrics.accuracy_score(ytest, predictions), 'mean accuracy', utils.mean_accuracy(ytest, predictions)
In [117]:
def vis_head_video(frame):
img = frame.copy()
img = img.astype(np.float32)
img = img/255.
img_or = img.copy()
img_or = img_or * 255
dh.init_with_image(img)
X = dh.features(dense_points)
preds_prob = model_rf.predict_proba(X)
max_prob = np.max(preds_prob[:, 1])
preds_prob = preds_prob[:, 1].reshape((227, 227)).T
img_r = dh.net.deprocess('data', dh.net.blobs['data'].data[0])
preds = preds_prob >= max(0.3, (max_prob/2))
preds = skimage.morphology.closing(preds, skimage.morphology.square(10))
preds = skimage.morphology.remove_small_objects(preds, min_size=10, connectivity=1)
L, N = skimage.measure.label(preds, return_num=True, background=0)
if N > 0:
L_no_bg = L[L!=-1].flatten()
vals, counts = scipy.stats.mode(L_no_bg)
part_label = int(vals[0])
indices = np.where(L==part_label)
xmin = indices[0].min()
xmax = indices[0].max()
ymin = indices[1].min()
ymax = indices[1].max()
rect_info = xmin, xmax, ymin, ymax
pmin = Part(-1, '?', -1, xmin, ymin, 1)
pmax = Part(-1, '?', -1, xmax, ymax, 1)
rect_parts = Parts(parts=[pmin, pmax])
rect_parts.denorm_for_size(img.shape[0], img.shape[1], size=227)
rect_info = rect_parts[0].x, rect_parts[1].x, rect_parts[0].y, rect_parts[1].y
xmin, xmax, ymin, ymax = rect_info
cv2.rectangle(img_or, (ymin, xmin), (ymax, xmax), 100, 5)
return img_or
In [140]:
vid_address = '/home/ipl/Desktop/bird_videos/crazy-bird.mp4'
cap = cv2.VideoCapture(vid_address)
In [141]:
_, frame = cap.read()
In [142]:
i = 1
while True:
_, frame = cap.read()
if frame is None:
break
else:
if i % 25 == 0:
print i
frame = frame[:, :, [2, 1, 0]]
res = vis_head_video(frame)
res = res[:, :, [2, 1, 0]]
cv2.imwrite('/home/ipl/Desktop/bird_head_tracking/temp/%d.jpg' % i, res)
i += 1
In [ ]: