# Example One, getting frames off a webcam.
import numpy as np
import cv2

cap = cv2.VideoCapture(0)

    ret, frame =
    cv2.imshow('Basic Web Cam',frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):


# Example two, advance camera properties. 
import numpy as np
import cv2
import time

# create a map to keep track of all these names
prop_map = {
    "avi_ratio" ,
    "width" ,
    "height" ,
    "fps" ,
    "fourcc" ,
    "format" ,
    "mode" ,
    "brightness" ,
    "contrast" ,
    "hue" ,
    "gain" ,
    "exposure" ,
    "convert_rgb" ,
 #   "white_balance" ,

# get a camera property
def get_prop(cam,name,prop_map):
    return cam.get(prop_map[name])

# set a camera property
def set_prop(cam,name,prop_map,value):

# print out all of the properites
def poll_props(cam,prop_map):
    out_map = {}
    for k,v in prop_map.items():
        result = cam.get(v)
        if( result == -1.0 ):
            out_map[k] = None
            out_map[k] = result
    return out_map

# create a camera and get its property
cam = cv2.VideoCapture(0)
properties = poll_props(cam,prop_map)

# list our properties
for k,v in properties.items():
    print "{0:<12}\t:{1:<12}".format(k,v)

    # toggle properties and get results. 
    sat = get_prop(cam,"saturation",prop_map)
    if( sat > 0.5 ):
    ret, frame =
    cv2.imshow('Basic Web Cam',frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):

# put our toys back on the shelf 

Go Pro Cameras

  • All GoPro Cameras through 3+ use a similar interface.

  • Each camera acts as a wifi access point with a simple web server.

  • We can hack this interface for nefarious purposes.

  • Default path is

  • GoProHero is a nice library wrapping a lot of commands.

Some Specifics on GoPros

  • Interface is hitting specific routes with parameters.

  • Images and video are collected using urllib and beautiful soup.

  • Naming conventions are a little convoluted -- need some helper code.

  • Control over resolutions and frame rates. Gain and exposure not so much.

  • Live streaming is a bit wonky.

In [1]:
from goprohero import GoProHero # helper lib
import urllib # used to "wget" files
import bs4 # beautiful soup, to parse file
import time
from IPython.display import Image

def download_and_save(name,route=""):
    Download a specific file on a connected GoPro camera.
    grab = route+name
    result = urllib.urlopen(grab) # basically wget
    if( result.code == 200 ): # 200 means OK
        with open(name,'wb') as fp: 
            fp.write( # write to file

def get_new(last=[],url=""):
    Find files recently added to our go pro. Requires a list of files
    unique = None
    last = set(last)
    out = urllib.urlopen(url)
    if(out.code == 200):
        soup = bs4.BeautifulSoup( # read the file tree
        # make a set of all <a href> in the tree
        fresh = set([row.renderContents() for row in soup.findAll('a')]) 
        # do a set difference with our initial set
        unique = list(fresh.difference(last))
    return unique

In [5]:
# check the camera file system
out = get_new()
# setup our controller
cam = GoProHero(password="herpderp")
# put the camera in burst mode
# tell it to take a shoth
# wait to finish
# get the new files
out = get_new(last=out)
for o in out:
    print o

KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-5-816fa52483e5> in <module>()
      1 # check the camera file system
----> 2 out = get_new()
      3 # setup our controller
      4 cam = GoProHero(password="herpderp")
      5 # put the camera in burst mode

<ipython-input-1-a1f2cd395959> in get_new(last, url)
     22     unique = None
     23     last = set(last)
---> 24     out = urllib.urlopen(url)
     25     if(out.code == 200):
     26         soup = bs4.BeautifulSoup( # read the file tree

/usr/lib/python2.7/urllib.pyc in urlopen(url, data, proxies)
     85         opener = _urlopener
     86     if data is None:
---> 87         return
     88     else:
     89         return, data)

/usr/lib/python2.7/urllib.pyc in open(self, fullurl, data)
    206         try:
    207             if data is None:
--> 208                 return getattr(self, name)(url)
    209             else:
    210                 return getattr(self, name)(url, data)

/usr/lib/python2.7/urllib.pyc in open_http(self, url, data)
    343         if realhost: h.putheader('Host', realhost)
    344         for args in self.addheaders: h.putheader(*args)
--> 345         h.endheaders(data)
    346         errcode, errmsg, headers = h.getreply()
    347         fp = h.getfile()

/usr/lib/python2.7/httplib.pyc in endheaders(self, message_body)
    973         else:
    974             raise CannotSendHeader()
--> 975         self._send_output(message_body)
    977     def request(self, method, url, body=None, headers={}):

/usr/lib/python2.7/httplib.pyc in _send_output(self, message_body)
    833             msg += message_body
    834             message_body = None
--> 835         self.send(msg)
    836         if message_body is not None:
    837             #message_body was not a string (i.e. it is a file) and

/usr/lib/python2.7/httplib.pyc in send(self, data)
    795         if self.sock is None:
    796             if self.auto_open:
--> 797                 self.connect()
    798             else:
    799                 raise NotConnected()

/usr/lib/python2.7/httplib.pyc in connect(self)
    776         """Connect to the host and port specified in __init__."""
    777         self.sock = socket.create_connection((,self.port),
--> 778                                              self.timeout, self.source_address)
    780         if self._tunnel_host:

/usr/lib/python2.7/socket.pyc in create_connection(address, timeout, source_address)
    560             if source_address:
    561                 sock.bind(source_address)
--> 562             sock.connect(sa)
    563             return sock

/usr/lib/python2.7/socket.pyc in meth(name, self, *args)
    223 def meth(name,self,*args):
--> 224     return getattr(self._sock,name)(*args)
    226 for _m in _socketmethods:


In [ ]:

A lot of cameras work just like our GoPro

  • Most newer CCTV cameras, baby monitors, etc. are IP web cameras.

  • Each camera acts as a little web server and streams images.

  • Even better, every one of us has one in our pocket!

  • A large variety of Android/IOS apps as cameras.

  • Harder to write control interface, but generally easy to stream.

A lot of cameras work just like our GoPro

  • Most of these apps support motion jpeg.

  • MJPEG simply dumps a series of jpegs, i.e. only *intra* frame compression.

  • The trick is to find the begining and end of each file.

  • Ususally of very low quality to support streaming.

  • There are lots of them on the web.

Read and display a simple MJPEG stream given a URL
import cv2
import urllib 
import numpy as np
# open our url stream
# create a buffer of bytes
invert = False
while True:
    # read some bytes from the stream
    # how much to read is a matter of taste*8)
    # \xff\xd8 is the start of a jpg
    # \xff\xd9 is the end of a jpg
    a = bytes.find('\xff\xd8')
    b = bytes.find('\xff\xd9')
    # if we find them
    if a!=-1 and b!=-1:
        jpg = bytes[a:b+2] # create a buffer
        bytes= bytes[b+2:] # save the remaining bytes
        #numpy can handle strings as image data
        img = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8),cv2.CV_LOAD_IMAGE_COLOR)
        # Add some bling to show we can process
        if invert:
            img = 255-img
        # show our image
    # MVP keyboard input
    my_key = cv2.waitKey(1)
    if my_key & 0xFF == ord('q'):
    if my_key & 0xFF == ord('i'):
        invert = not invert 
# put our toys away when done. 

Cameras, Cameras, Everywhere

  • Lots of cool streaming sources, Youtube, Twtich, Ustream, etc.

  • Can we treat them like a camera or use our own?

  • Parsing the urls is kinda tough, python can help.

  • Livestreamer is a tool for parsing stream urls

  • Has some painful dependencies, but installed relatively easy.

Streaming vs Progressive Downloading

  • Live streamer can parse and point us to a file buffer.

  • What's in this buffer?

  • Turns out is mainly a series of small mpeg files. Streaming isn't streaming, it is progressive downloading.

  • You get the head of a file and append clips at the end.

  • It acutally plays well with OpenCV!

  • This example is simple, in real applications we should use a threading model.

import cv2
import numpy as np
import time
import livestreamer

# use live streamer to figure out the stream info
streams = livestreamer.streams("")
stream = streams['best']
# open our out file. 
fname = "test.mpg"
vid_file = open(fname,"wb")
# dump from the stream into an mpg file -- get a buffer going
fd =
for i in range(0,1048):
    if i%256==0:
        print "Buffering..."
    new_bytes =
# open the video file from the begining
print "Done buffering."
cam = cv2.VideoCapture(fname)
while True:
    ret, img =                      
        if ret:
            img = 255-img # invert the colors
        print "DERP"
    if (0xFF & cv2.waitKey(5) == 27) or img.size == 0:
    # dump some more data to the stream so we don't run out. 
    new_bytes =*8)
del fd
del vid_file
del new_bytes

Digital Single-Lens Reflex (DSLR) Cameras

  • Fairly common camera type with large sensors.

  • Lots and lots of different lenses and accessories.

  • Allow precise control of optics and camera parameters

  • Bad: images are large and interface is vendor specific.

  • Good: raw data and resolution give us lots of data to play with.

Digital Single-Lens Reflex (DSLR) Cameras

  • GPhoto2 library is good for Canon brand cameras.

  • Interface with swig, heavy handed but it works.

  • Setting must match camera modes.

  • Use cases: HDR, Computational Photography, Science!

import logging
import os
import subprocess
import sys
import gphoto2 as gp
import time

def setup():
    Attempt to attach to a gphoto device and grab the camera and context. Return the results.
        format='%(levelname)s: %(name)s: %(message)s', level=logging.WARNING)
    context = gp.gp_context_new()
    camera = gp.check_result(gp.gp_camera_new())
    gp.check_result(gp.gp_camera_init(camera, context))
    text = gp.check_result(gp.gp_camera_get_summary(camera, context))
    print text.text
    return camera,context

def recurse_config(child,params={}):
    The gphoto control structure is a byzantine swig structure.
    This function traverses it recursively and puts it in a
    nice python dictionary.
    if(child.count_children() <= 0):
        my_choices = []
            n = child.count_choices()
            if( n > 0 ):
                for k in range(0,n):
            return my_choices
        return my_choices
        for i in range(0, child.count_children()):
            chill = child.get_child(i)
            name = chill.get_name()
            params[name] = recurse_config(chill,{})
        return params

def print_config_dict(cdict,lvl=""):
    Print a the python config dictionary for a camera.
    if isinstance(cdict,dict):
        for k,v in cdict.items():
            print "{0}{1}".format(lvl,k)
    elif isinstance(cdict,list):
        for l in cdict:
            print "{0}{1}".format(lvl,l)

def set_config(camera,context,config,path,value):
    Given a gphoto camera, context, and config 
    traverse the path of the config tree and set
    a parameter value. The path is basically the nodes
    to address in the control structure. Once the config
    object has been modified we have to set it on the camera.
    current = config
    for p in path:
        current = current.get_child_by_name(p)
    gp.check_result(gp.gp_camera_set_config(camera,config, context))
    print "Set {0} to {1}".format(current.get_name(),current.get_value())

def capture_image(camera,context,name):
    Use gphoto to capture an image and retrieve it.
    Place the file in /tmp/name
    file_path = gp.check_result(gp.gp_camera_capture(
        camera, gp.GP_CAPTURE_IMAGE, context))
    target = os.path.join('/tmp', name)
    print 'Copying image to {0}'.format(target)
    camera_file = gp.check_result(gp.gp_camera_file_get(
            camera, file_path.folder,,
            gp.GP_FILE_TYPE_NORMAL, context))
    gp.check_result(gp.gp_file_save(camera_file, target))
    gp.check_result(gp.gp_camera_exit(camera, context))

def main():
    # set up our camera.
    camera,context = setup()
    # grab a single test image. 
    # Get the configuration of the camera
    config = gp.check_result(gp.gp_camera_get_config(camera, context))
    # Pythonify and print the configuration of the camera so
    # we can see what parameters we can play with. 
    pconfig = recurse_config(config)
    # Put the camera in AV mode, or aperture priority. 
    # Camera needs this to fiddle with aperture. 
    count = 0
    # for all of the available aperture settings...
    for param in pconfig["capturesettings"]["aperture"]:
        # get the camera configuration
        config = gp.check_result(gp.gp_camera_get_config(camera, context))
        # set the new configuration
        # and capture an image.
        fname = "Capture{:0>5}.jpg".format(count)
        count += 1

if __name__ == "__main__":

Machine Vision Cameras and Robot Operating System

  • There are lots of industrial machine vision cameras out there.

  • Vendors like Allied, Basler, IDS, Point Grey, Matrox, Teledyne Dalsa, Axis, FLIR.

  • Each camera can address specific application needs.

  • There cameras can have esoteric interfaces (GigE, Firewire, etc).

  • How to get the into Python?

Why ROS?

  • Roboticists use these cameras, so there are a lot of packages.

  • The ROS allows you to use C++ packages from python easily.

  • Lots of tools for viewing, logging, controlling, debugging, and calibrating cameras.

  • Package management for camera libraries and native OpenCV support.

  • Con: a lot of stuff to learn and install, but worth it (see previous talks).

  • ROS uses a message bus, python can poll and controll cameras from this bus.

What we are going to do.

  • Get depth images from a structure sensor (depth camera).

  • Use the OpenNI C++ package to interface to the structure sensor.

  • ROS publishes images to a ROS topic.

  • We are going to subscribe to this bus and process the images.

  • Take a look at some of the tools.

A ROS Node Example

#!/usr/bin/env python
import rospy
# This is the tool that marshals images into OpenCV
from cv_bridge import CvBridge, CvBridgeError 
# Import some stock ROS message types.
from sensor_msgs.msg import Image
# import some utils.
import numpy as np
import cv
import SimpleCV as scv
import copy as copy

class ProcessDepth:
    def __init__(self):
        # Allows conversion between numpy arrays and ROS sensor_msgs/Image
        self.bridge = CvBridge() 

        # Allow our topics to be dynamic.
        self.input_camera_topic = rospy.get_param('~input_camera_topic', '/camera/depth/image_rect')
        self.output_camera_topic = rospy.get_param('~output_camera_topic', '/processed')

        # WE are going to publish a debug image as it comes in. = rospy.Publisher(self.output_camera_topic, Image,queue_size=10)
        rospy.Subscriber(self.input_camera_topic, Image, self._process_depth_img)
        # run the node

    # Keep the node alive
    def _run(self):

    def _process_depth_img(self,input):
        # convert our image to CV2 numpy format from ROS format
        latest_image = self.bridge.imgmsg_to_cv2(input)

        if( latest_image is not None ):
                # convert the image to SimpleCV
                # The input image is single channel float and we want rgb uint8
                # it is also full of nasty nans. We get the min and max and scale
                # the image from [0,flt_max] to [0,255]
                dmin = np.nanmin(latest_image)
                dmax = np.nanmax(latest_image)
                latest_image = latest_image - dmin
                sf = 255.0/(dmax-dmin)
                latest_image = sf*latest_image
                # Convert to uint8
                temp = latest_image.astype(np.uint8)
                # move to SimpleCV RGB
                img = scv.Image(temp, cv2image=True, colorSpace=scv.ColorSpace.RGB)
                # get values less than 128
                lt = img.threshold(128).invert()
                # get values greater than 64
                gt = img.threshold(64) 
                # do the logical and of the two depths
                range = lt*gt
                # apply the mask to the input image
                blobs = img.findBlobsFromMask(mask=range)
                # draw the results. 
                if( blobs ):
                img = img.applyLayers()
                # convert SimpleCV to CV2 Numpy Format
                cv2img = img.getNumpyCv2()
                # Convert Cv2 numpy to ROS format
                img_msg = self.bridge.cv2_to_imgmsg(cv2img, "bgr8")
                # publish the topic.
      , "bgr8"))
            except CvBridgeError, e:
                rospy.logwarn("PROCESSING EXCEPTION {0}".format(e))

# Boilerplate node spin up. 
if __name__ == '__main__':
        p = ProcessDepth()
    except rospy.ROSInterruptException:

In [ ]: