They don't see like we do!

A set of samples showing how pre-trained networks can't detect set of images. In this example I am using DeepLab and lots of images from the interent. See References for link to sources.

References



In [1]:

    
#@title Imports

import os
from io import BytesIO
import tarfile 
import tempfile
from six.moves import urllib

from matplotlib import gridspec
from matplotlib import pyplot as plt
import numpy as np
from PIL import Image

import tensorflow as tf



In [2]:

    
#@title Helper methods


class DeepLabModel(object):
  """Class to load deeplab model and run inference."""

  INPUT_TENSOR_NAME = 'ImageTensor:0'
  OUTPUT_TENSOR_NAME = 'SemanticPredictions:0'
  INPUT_SIZE = 513
  FROZEN_GRAPH_NAME = 'frozen_inference_graph'

  def __init__(self, tarball_path):
    """Creates and loads pretrained deeplab model."""
    self.graph = tf.Graph()

    graph_def = None
    # Extract frozen graph from tar archive.
    tar_file = tarfile.open(tarball_path)
    for tar_info in tar_file.getmembers():
      if self.FROZEN_GRAPH_NAME in os.path.basename(tar_info.name):
        file_handle = tar_file.extractfile(tar_info)
        graph_def = tf.GraphDef.FromString(file_handle.read())
        break

    tar_file.close()

    if graph_def is None:
      raise RuntimeError('Cannot find inference graph in tar archive.')

    with self.graph.as_default():
      tf.import_graph_def(graph_def, name='')

    self.sess = tf.Session(graph=self.graph)

  def run(self, image):
    """Runs inference on a single image.

    Args:
      image: A PIL.Image object, raw input image.

    Returns:
      resized_image: RGB image resized from original input image.
      seg_map: Segmentation map of `resized_image`.
    """
    width, height = image.size
    resize_ratio = 1.0 * self.INPUT_SIZE / max(width, height)
    target_size = (int(resize_ratio * width), int(resize_ratio * height))
    resized_image = image.convert('RGB').resize(target_size, Image.ANTIALIAS)
    batch_seg_map = self.sess.run(
        self.OUTPUT_TENSOR_NAME,
        feed_dict={self.INPUT_TENSOR_NAME: [np.asarray(resized_image)]})
    seg_map = batch_seg_map[0]
    return resized_image, seg_map


def create_pascal_label_colormap():
  """Creates a label colormap used in PASCAL VOC segmentation benchmark.

  Returns:
    A Colormap for visualizing segmentation results.
  """
  colormap = np.zeros((256, 3), dtype=int)
  ind = np.arange(256, dtype=int)

  for shift in reversed(range(8)):
    for channel in range(3):
      colormap[:, channel] |= ((ind >> channel) & 1) << shift
    ind >>= 3

  return colormap


def label_to_color_image(label):
  """Adds color defined by the dataset colormap to the label.

  Args:
    label: A 2D array with integer type, storing the segmentation label.

  Returns:
    result: A 2D array with floating type. The element of the array
      is the color indexed by the corresponding element in the input label
      to the PASCAL color map.

  Raises:
    ValueError: If label is not of rank 2 or its value is larger than color
      map maximum entry.
  """
  if label.ndim != 2:
    raise ValueError('Expect 2-D input label')

  colormap = create_pascal_label_colormap()

  if np.max(label) >= len(colormap):
    raise ValueError('label value too large.')

  return colormap[label]


def vis_segmentation(image, seg_map):
  """Visualizes input image, segmentation map and overlay view."""
  plt.figure(figsize=(15, 5))
  grid_spec = gridspec.GridSpec(1, 4, width_ratios=[6, 6, 6, 1])

  plt.subplot(grid_spec[0])
  plt.imshow(image)
  plt.axis('off')
  plt.title('input image')

  plt.subplot(grid_spec[1])
  seg_image = label_to_color_image(seg_map).astype(np.uint8)
  plt.imshow(seg_image)
  plt.axis('off')
  plt.title('segmentation map')

  plt.subplot(grid_spec[2])
  plt.imshow(image)
  plt.imshow(seg_image, alpha=0.7)
  plt.axis('off')
  plt.title('segmentation overlay')

  unique_labels = np.unique(seg_map)
  ax = plt.subplot(grid_spec[3])
  plt.imshow(
      FULL_COLOR_MAP[unique_labels].astype(np.uint8), interpolation='nearest')
  ax.yaxis.tick_right()
  plt.yticks(range(len(unique_labels)), LABEL_NAMES[unique_labels])
  plt.xticks([], [])
  ax.tick_params(width=0.0)
  plt.grid('off')
  plt.show()  

LABEL_NAMES = np.asarray([
    'background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus',
    'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike',
    'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tv'
])

FULL_LABEL_MAP = np.arange(len(LABEL_NAMES)).reshape(len(LABEL_NAMES), 1)
FULL_COLOR_MAP = label_to_color_image(FULL_LABEL_MAP)



In [3]:

    
#@title Some more helper functions

def run_visualization(url):
  """Inferences DeepLab model and visualizes result."""
  try:
    f = urllib.request.urlopen(url)
    jpeg_str = f.read()
    original_im = Image.open(BytesIO(jpeg_str))
  except IOError:
    print('Cannot retrieve image. Please check url: ' + url)
    return

  print('running deeplab on image %s...' % url)
  resized_im, seg_map = MODEL.run(original_im)

  vis_segmentation(resized_im, seg_map)


def show_images(images, cols = 1, titles = None):
    """Display a list of images in a single figure with matplotlib.
    
    Parameters
    ---------
    images: List of np.arrays compatible with plt.imshow.
    
    cols (Default = 1): Number of columns in figure (number of rows is 
                        set to np.ceil(n_images/float(cols))).
    
    titles: List of titles corresponding to each image. Must have
            the same length as titles.
    """
    assert((titles is None) or (len(images) == len(titles)))
    n_images = len(images)
    if titles is None: titles = ['Image (%d)' % i for i in range(1,n_images + 1)]
    fig = plt.figure()
    for n, (image, title) in enumerate(zip(images, titles)):
        a = fig.add_subplot(cols, np.ceil(n_images/float(cols)), n + 1)
        plt.imshow(image)
        a.set_title(title)
    fig.set_size_inches(np.array(fig.get_size_inches()) * n_images)
    plt.show()
    
    
def image_grabber(url):
  f = urllib.request.urlopen(url)
  jpeg_str = f.read()
  original_im = Image.open(BytesIO(jpeg_str))
  return original_im



In [4]:

    
#@title Select and download models {display-mode: "form"}

MODEL_NAME = 'mobilenetv2_coco_voctrainaug'  # @param ['mobilenetv2_coco_voctrainaug', 'mobilenetv2_coco_voctrainval', 'xception_coco_voctrainaug', 'xception_coco_voctrainval']

_DOWNLOAD_URL_PREFIX = 'http://download.tensorflow.org/models/'
_MODEL_URLS = {
    'mobilenetv2_coco_voctrainaug':
        'deeplabv3_mnv2_pascal_train_aug_2018_01_29.tar.gz',
    'mobilenetv2_coco_voctrainval':
        'deeplabv3_mnv2_pascal_trainval_2018_01_29.tar.gz',
    'xception_coco_voctrainaug':
        'deeplabv3_pascal_train_aug_2018_01_04.tar.gz',
    'xception_coco_voctrainval':
        'deeplabv3_pascal_trainval_2018_01_04.tar.gz',
}
_TARBALL_NAME = 'deeplab_model.tar.gz'

model_dir = tempfile.mkdtemp()
tf.gfile.MakeDirs(model_dir)

download_path = os.path.join(model_dir, _TARBALL_NAME)
print('downloading model, this might take a while...')
urllib.request.urlretrieve(_DOWNLOAD_URL_PREFIX + _MODEL_URLS[MODEL_NAME],
                   download_path)
print('download completed! loading DeepLab model...')

MODEL = DeepLabModel(download_path)
print('model loaded successfully!')









    



downloading model, this might take a while...
download completed! loading DeepLab model...
model loaded successfully!

The Dark Chickens 🐔🐔

Goth Chicken or Dark Chicken is a cool breed of chickens. These type of chickens are entierly black! Let's see if pre-trained NNs can detect these types of chickens correctly. 🤔

Source



In [5]:

    
img1 = Image.open('chicken1.jpg')
img2 = Image.open('chicken2.jpg')



In [6]:

    
resized, seg_map = MODEL.run(img1)
vis_segmentation(resized, seg_map)









    



/home/taesiri/.conda/envs/deep36/lib/python3.6/site-packages/matplotlib/cbook/__init__.py:424: MatplotlibDeprecationWarning: 
Passing one of 'on', 'true', 'off', 'false' as a boolean is deprecated; use an actual boolean (True/False) instead.
  warn_deprecated("2.2", "Passing one of 'on', 'true', 'off', 'false' as a "



In [7]:

    
resized, seg_map = MODEL.run(img2)
vis_segmentation(resized, seg_map)

It's working!

It seems being different breed is not a problem for our pre-trained model! Lets check a very rare breed of vulture!

Source



In [8]:

    
IMAGE_URL = 'https://kids.nationalgeographic.com/content/dam/kids/photos/articles/Other%20Explore%20Photos/R-Z/Wacky%20Weekend/Strange%20Birds/ww-birds-king-vulture.adapt.945.1.jpg'

run_visualization(IMAGE_URL)









    



running deeplab on image https://kids.nationalgeographic.com/content/dam/kids/photos/articles/Other%20Explore%20Photos/R-Z/Wacky%20Weekend/Strange%20Birds/ww-birds-king-vulture.adapt.945.1.jpg...

What's Next?

Maybe there are lots of vulutre in training set, we don't know for sure. How about not trivial depiction of birds?

Source	Source	Source	Source
Source	Source	Source	Source
Source	Source	Source	Source
Source	Source	Source	Source
Source	Source	Source	Source
Source	Source	Source	Source



In [9]:

    
_URL_BANK = ['https://cdn.shopify.com/s/files/1/0709/4185/products/cockerelcard_400x.jpg',
             'https://i.pinimg.com/564x/c8/a7/13/c8a713883a068a1f8a62b25befaf7db6.jpg',
             'https://i.pinimg.com/564x/5a/a2/67/5aa267368b2b60f98a4a4dbecdc2e4a1.jpg',
             'https://i.pinimg.com/564x/b6/7a/be/b67abeaf7ddc73c5c74d21c6cf9cc45f.jpg',
             'https://i.pinimg.com/564x/38/6a/09/386a09a5cae1aedb1b38c05d7ef128b0.jpg',
             'https://i.pinimg.com/564x/81/f8/0f/81f80f2def146915af289ad8c4d5f672.jpg',
             'https://i.pinimg.com/564x/f3/32/81/f33281bc8114c2bb4e85b5daa87df9e6.jpg',
             'https://i.pinimg.com/564x/b1/c0/94/b1c094c4a6ee60c9c8b5fce4b6093f51.jpg',
             'https://i.pinimg.com/564x/b8/64/63/b86463279cf12318b346de040c953226.jpg',
             'https://i.pinimg.com/564x/a8/c9/b9/a8c9b98d20baeec4fe9b0f25c1ecf03a.jpg',
             'https://i.pinimg.com/564x/46/31/69/4631694e7a8640e948123bc72c5a0705.jpg',
             'https://i.pinimg.com/564x/c5/54/2b/c5542b07e8be59230b95743fd7269eaf.jpg',
             'https://i.pinimg.com/564x/81/1c/62/811c62215cd25bbcb0c7f2b74ff283b7.jpg',
             'https://i.pinimg.com/564x/68/14/5b/68145b897a7ec9bc83aa5a60236eda47.jpg',
             'https://i.pinimg.com/564x/08/8c/a7/088ca78bb8c6f5b9bd2b889ac9c5d3cf.jpg',
             'https://i.pinimg.com/564x/79/0f/1b/790f1b218f028dd9602314b283e34f56.jpg',
             'https://i.pinimg.com/564x/6d/61/ac/6d61ac676b571f9ac0c6e9bb88fa735f.jpg',
             'https://i.pinimg.com/564x/ef/88/12/ef88122fd6d817903603657526be5dc3.jpg',
             'https://i.pinimg.com/564x/7f/69/fa/7f69fa9c83c4514d9f9d3dd27ed63444.jpg',
             'https://i.pinimg.com/564x/ba/78/f6/ba78f69538e7aaf23ab678ce9289b830.jpg']



In [10]:

    
# download images 
image_list = [image_grabber(url) for url in _URL_BANK]
# get segmentation map for each image
packed = [MODEL.run(img) for img in image_list]
resized_images = [item[0] for item in packed]
seg_maps =  [item[1] for item in packed]

# visualize segment maps
for i in range(len(packed)):
  vis_segmentation(resized_images[i], seg_maps[i])

Conclusion!

We can't conclude anything yet!