In [8]:
import Leap, sys, thread, time
from Leap import CircleGesture, KeyTapGesture, ScreenTapGesture, SwipeGesture
from IPython.html.widgets import TextWidget, ImageWidget, HTMLWidget
from IPython.display import display
import numpy as np
import cv2
from PIL import Image as PIL_Image
from io import BytesIO
cv2.startWindowThread()
def img_to_png(ima, cvt=None):
if cvt:
ima = cv2.cvtColor(ima, cvt)
im = PIL_Image.fromarray(ima)
bio = BytesIO()
im.save(bio, format='png')
return bio.getvalue()
In [9]:
# Initial Intrinsic Camaera Parameters of the iSight HD Camera of my MacBookAir 11" 2012
# the resolution is 1280x720, so if your camera is not, resize the image to be 1280x720
cm0 = np.float32([[ 1.25542912e+03, 0.00000000e+00, 6.39500000e+02],
[ 0.00000000e+00, 1.23283056e+03, 3.59500000e+02],
[ 0.00000000e+00, 0.00000000e+00, 1.00000000e+00]])
dist0 = np.float32([[ 0.54520465, -1.47166739, 0.02451086, -0.02110954, 1.96975927]])
In [10]:
controller = Leap.Controller()
In [11]:
cm=None
In [12]:
# Interactive calibration routine
# touch red point and press space key, when done, press 'q' key.
test_pos=[(0.1+(1.0*i/4)*0.8,0.1+(1.0*j/2)*0.8) for i in range(5) for j in range(3)]
result = []
screen_pos = []
cap = cv2.VideoCapture(0)
controller.set_policy_flags(Leap.Controller.POLICY_BACKGROUND_FRAMES)
txt = TextWidget()
txt.set_css({"width":"800px"})
display(txt)
hand1txt = TextWidget()
hand1txt.set_css({"width":"800px"})
display(hand1txt)
#img_widget = ImageWidget(width=600)
#display(img_widget)
idx = 0
while True:
ret, img = cap.read()
# img = cv2.resize(img, (960,540))
H, W = img.shape[:2]
#cv2.circle(img, (int(W*0.8), int(H*0.8)), 10, (0,0,255), -1)
#img_widget.value = img_to_png(img, cvt=cv2.COLOR_BGR2RGB)
frame = controller.frame()
keycode = cv2.waitKey(1) & 0xff
txt.value = "Frame id: %d, hands: %d, fingers: %d, tools: %d, gestures: %d" % (
i, len(frame.hands), len(frame.fingers), len(frame.tools), len(frame.gestures()))
tip = None
if len(frame.hands)>0:
hand1 = frame.hands[0]
fingers = hand1.fingers.finger_type(1)
if len(fingers):
tip = fingers[0].tip_position
tip_s="(%5.1f, %5.1f, %5.1f)"%(tip[0], tip[1], tip[2])
#cv2.circle(img, (int(400-tip[0]*2), int(H-tip[1]*2)), 10, (0,0,255), -1)
if cm is not None:
xy = cv2.projectPoints(np.float32([(tip[0], tip[1], tip[2])]), rvec[0], tvec[0], cm, dist)[0][0][0]
cv2.circle(img, (int(xy[0]), int(xy[1])), 10, (0,255,0), -1)
else:
tip_s=""
hand1txt.value = "pinch %5.2f, grab %5.2f, tip %s"%(hand1.pinch_strength, hand1.grab_strength, tip_s)
else:
hand1txt.value = ""
if keycode == ord('q'):
break
elif keycode == ord(' '):
if tip:
screen_pos.append((int(test_pos[idx][0]*W), int(test_pos[idx][1]*H)))
result.append((tip[0], tip[1], tip[2]))
idx += 1
if idx >= len(test_pos):
idx = 0
cv2.circle(img, (int(test_pos[idx][0]*W), int(test_pos[idx][1]*H)), 10, (0,0,255) if tip else (255,0,0), -1)
img = cv2.flip(img, 1)
cv2.imshow('frame', img)
#time.sleep(0.3)
cv2.destroyWindow('frame')
cap.release()
controller.clear_policy(Leap.Controller.POLICY_BACKGROUND_FRAMES)
In [14]:
#calibrate the camera
screen_pos2 = [(W-1-x, y) for (x,y) in screen_pos]
retval, cm, dist, rvec, tvec = cv2.calibrateCamera(np.float32([result]),
np.float32([screen_pos]), (W, H), cm0.copy(), dist0.copy(),
flags=cv2.CALIB_USE_INTRINSIC_GUESS)
print retval, rvec, tvec, cm, dist
In [15]:
# try and test your parameter
# if not good, input data and calibrate again
cap = cv2.VideoCapture(0)
controller.set_policy_flags(Leap.Controller.POLICY_BACKGROUND_FRAMES)
txt = TextWidget()
txt.set_css({"width":"800px"})
display(txt)
hand1txt = TextWidget()
hand1txt.set_css({"width":"800px"})
display(hand1txt)
def to_np(v):
return np.float32([v[0], v[1], v[2]])
def to_tuple(v):
return ([v[0], v[1], v[2]])
while True:
ret, img = cap.read()
#img = cv2.resize(img, (960,540))
H, W = img.shape[:2]
frame = controller.frame()
keycode = cv2.waitKey(1) & 0xff
txt.value = "Frame id: %d, hands: %d, fingers: %d, tools: %d, gestures: %d" % (
i, len(frame.hands), len(frame.fingers), len(frame.tools), len(frame.gestures()))
tip = None
for hand1 in frame.hands:
for f in hand1.fingers:
tp = f.tip_position
xy = cv2.projectPoints(np.float32([to_np(tp)]), rvec[0], tvec[0], cm, dist)[0][0][0]
cv2.circle(img, (int(xy[0]), int(xy[1])), 10, (0,255,0), -1)
for bn in range(3):
bone = f.bone(bn)
if bone.is_valid:
xy1 = cv2.projectPoints(np.float32([to_np(bone.prev_joint)]), rvec[0], tvec[0], cm, dist)[0][0][0]
xy2 = cv2.projectPoints(np.float32([to_np(bone.next_joint)]), rvec[0], tvec[0], cm, dist)[0][0][0]
try:
cv2.line(img, tuple(xy1) , tuple(xy2), (255,255,255) , 3)
cv2.circle(img, tuple(xy1), 5, (255,255,128), -1)
cv2.circle(img, tuple(xy2), 5, (255,255,128), -1)
except:
pass
fingers = hand1.fingers.finger_type(1)
for f in fingers:
tip = f.tip_position
tip_s="(%5.1f, %5.1f, %5.1f)"%(tip[0], tip[1], tip[2])
xy = cv2.projectPoints(np.float32([(tip[0], tip[1], tip[2])]), rvec[0], tvec[0], cm, dist)[0][0][0]
try:
cv2.circle(img, tuple(xy), 10, (0,0,255), -1)
except:
pass
else:
tip_s=""
hand1txt.value = "pinch %5.2f, grab %5.2f, tip %s"%(hand1.pinch_strength, hand1.grab_strength, tip_s)
else:
hand1txt.value = ""
if keycode == ord('q'):
break
img = cv2.flip(img, 1)
cv2.imshow('frame', img)
cv2.destroyWindow('frame')
cap.release()
controller.clear_policy(Leap.Controller.POLICY_BACKGROUND_FRAMES)
In [11]:
# print the camera parameter
print """
({r:%s,
t:%s,
k1:%s, k2:%s, k3:%s, p1:%s, p2:%s,
fx:%s, fy:%s, cx:%s, cy:%s})"""%(list(cv2.Rodrigues(rvec[0])[0].flatten()),
list(tvec[0].flatten()),
dist[0][0], dist[0][1], dist[0][4], dist[0][2], dist[0][3],
cm[0][0], cm[1][1], cm[0][2],cm[1][2]
)
#x,y,z = tuple(np.dot(cv2.Rodrigues(R)[0], np.float32(p))+T.flatten())
#map(float, cm.flatten())
In [ ]:
project