In [1]:
%matplotlib inline
import json, os, sys, math
import numpy as np
import matplotlib.pyplot as plt
from scipy.ndimage import imread
from scipy.ndimage.filters import gaussian_filter
from scipy.misc import imsave
import cv2, os
stats = {}
video_files = ["videos/" + f for f in os.listdir("videos/") if ".mp4" in f]
for video in video_files:
vcap = cv2.VideoCapture(video) # 0=camera
if vcap.isOpened():
width = vcap.get(3) # float
height = vcap.get(4) # float
stats[video] = {"width": width, "height": height}
o_width = 1920.0
o_height = 960.0
s_width = 500.0
s_height = 282.0
fps = 29.97
def face_scale_x(x):
return x * (s_width / o_width)
def face_scale_y(y):
return y * (s_height / o_height)
def image_saliency_fn_to_seconds(fn):
v = float(fn.split(".png")[0])
return v * 0.5
def face_tracking_frame_to_seconds(frame, base):
v = float(frame)
if base != "nocuts":
return v * 0.5
else:
return v * 1.0/fps
def get_immediate_subdirectories(a_dir):
return [name for name in os.listdir(a_dir)
if os.path.isdir(os.path.join(a_dir, name))]
def image_from_face_tracking(frame_list, w, h):
im = np.zeros((h, w))
for item in frame_list:
for frame in item:
min_bb_x = int(math.floor(face_scale_x(float(frame["face_minBBox"]["x"]))))
min_bb_y = int(math.floor(face_scale_y(float(frame["face_minBBox"]["y"]))))
max_bb_x = int(math.floor(face_scale_x(float(frame["face_maxBBox"]["x"]))))
max_bb_y = int(math.floor(face_scale_y(float(frame["face_maxBBox"]["y"]))))
im[ min_bb_y:max_bb_y,min_bb_x:max_bb_x ] = 1.0
return im
def image_from_optical_flow(frame_list, w, h):
window = 5
im = np.zeros((h,w))
for frame in frame_list:
pxl_x = int(frame["b"][0])
pxl_y = int(frame["b"][1])
pxl_value = frame["movement"]
im[ (pxl_y-window):(pxl_y+window),(pxl_x-window):(pxl_x+window) ] = pxl_value
return im
def image_from_image_saliency(fn_list, root, w, h):
im_list = []
for fn in fn_list:
return imread(root + fn)
return np.zeros((h,w))
def isarrnan(fti):
return np.argwhere(np.isnan(fti)).shape[0] > 0
In [2]:
# for a given video
time_ref = {
"snowboarding": [151.868109 + 29, 241],
"hpo-preview": [0, 60.5],
"surfing": [45, 105],
"dining-at-the-met": [30, 96],
"knives": [0,35],
"trees": [82,150],
"invasion": [120, 180],
"ice-art": [4,70],
"volcano": [93,160]
}
bases = [s for s in get_immediate_subdirectories("analysis/face-tracking/") if s != "nocuts" and s != "invasion"]
bases = time_ref.keys()
for base in bases:
print base
image_saliency = "analysis/image-saliency/" + base + "-small/"
face_tracking = "analysis/face-tracking/" + base + "/out.json"
optical_flow = "analysis/optical-flow/" + base + "-small.json"
all_saliency = "analysis/all-saliency/" + base + "/"
optical_flow_images = "analysis/optical-flow-images/" + base + "/"
face_detection_images = "analysis/face-detection-images/" + base + "/"
v_file = "videos/" + base + ".mp4"
o_width = stats[v_file]["width"]
o_height = stats[v_file]["height"]
face_tracking_json = None
optical_flow_json = None
face_tracking_times = {}
image_saliency_times = {}
optical_flow_times = {}
if not os.path.exists(all_saliency):
os.mkdir(all_saliency)
if not os.path.exists(optical_flow_images):
os.mkdir(optical_flow_images)
if not os.path.exists(face_detection_images):
os.mkdir(face_detection_images)
if os.path.exists(face_tracking):
with open(face_tracking) as f:
face_tracking_json = json.load(f)
for key in face_tracking_json.keys():
face_tracking_times[key] = face_tracking_frame_to_seconds(key, base)
if os.path.exists(image_saliency):
fns = [k for k in os.listdir(image_saliency) if ".png" in k]
for fn in fns:
image_saliency_times[fn] = image_saliency_fn_to_seconds(fn)
if os.path.exists(optical_flow):
with open(optical_flow) as f:
optical_flow_json = json.load(f)
for i in range(len(optical_flow_json)):
item = optical_flow_json[i]
optical_flow_times[str(i)] = float(item["time"])
f_times = [face_tracking_times[k] for k in face_tracking_times.keys()]
i_times = [image_saliency_times[k] for k in image_saliency_times.keys()]
o_times = [optical_flow_times[k] for k in optical_flow_times.keys()]
all_time_jsons = []
if f_times or i_times or o_times:
times = f_times + i_times + o_times
max_time = int(math.floor(max(times)))
for t_low in range(max_time):
t_high = t_low + 1.0
rel_optical_flow_frames = [k for k in optical_flow_times.keys() if optical_flow_times[k] >= t_low and optical_flow_times[k] < t_high]
rel_image_saliency_frames = [k for k in image_saliency_times.keys() if image_saliency_times[k] >= t_low and image_saliency_times[k] < t_high]
rel_face_tracking_frames = [k for k in face_tracking_times.keys() if face_tracking_times[k] >= t_low and face_tracking_times[k] < t_high]
w = int(s_width)
h = int(s_height)
ofi = None
fti = None
isi = None
ofi = image_from_optical_flow([optical_flow_json[int(k)] for k in rel_optical_flow_frames], w, h)
ofi /= ofi.max()/1.0
fti = image_from_face_tracking([face_tracking_json[k] for k in rel_face_tracking_frames], w, h)
fti /= fti.max()/1.0
isi = image_from_image_saliency(rel_image_saliency_frames, image_saliency, w, h)
if float(isi.max()):
isi = isi/float(isi.max())
# print isi.shape, ofi.shape, fti.shape
meow = None
if ofi.any() and not isarrnan(ofi):
ofi = gaussian_filter(ofi, sigma=3)
plt.imsave(optical_flow_images + str(t_low) + ".png",ofi)
if meow != None and meow.any():
meow += ofi
else:
meow = ofi
if fti.any() and not isarrnan(fti):
fti = gaussian_filter(fti, sigma=3)
plt.imsave(face_detection_images + str(t_low) + ".png",fti)
if meow != None and meow.any():
meow += fti
else:
meow = fti
if isi.any() and not isarrnan(isi):
isi = gaussian_filter(isi, sigma=3)
if meow != None and meow.any():
meow += isi
else:
meow = isi
elif meow == None or not meow.any() or isarrnan(meow):
meow = isi
plt.imsave(all_saliency + str(t_low) + ".png", meow)
In [96]:
isi.max()
Out[96]:
In [129]:
a = np.array([1,1,1,1])
In [131]:
all(v == 0 for v in a)
print np.all(a, 1)
In [5]:
"analysis/face-tracking/"
Out[5]:
In [42]:
m = np.argwhere(np.isnan(fti)).shape[0] > 0
print m
print fti.shape
In [43]:
print np.argwhere(np.isnan(isi)).shape
print isi.shape
np.argwhere(np.isnan(isi)).shape[0] > 0
Out[43]:
In [52]:
import cv2, os
stats = {}
video_files = ["videos/" + f for f in os.listdir("videos/") if ".mp4" in f]
for video in video_files:
vcap = cv2.VideoCapture(video) # 0=camera
if vcap.isOpened():
width = vcap.get(3) # float
height = vcap.get(4) # float
stats[video] = {"width": width, "height": height}
In [53]:
stats
Out[53]:
In [ ]: