Licensed under the Apache License, Version 2.0 (the "License");


In [0]:
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

Cosine Similarity Overlays

Cosine similarity overlays are a simple interface for understanding what nerual networks consider similar and different. They were originally introduced by Alex Mordvintsev.


In [2]:
!pip install --quiet lucid==0.0.5
!npm install -g svelte-cli@2.2.0
#tensorflow_version only works on colab
%tensorflow_version 1.x

import numpy as np
import tensorflow as tf
assert tf.__version__.startswith('1')

import lucid.modelzoo.vision_models as models
import lucid.optvis.render as render
from lucid.misc.io import show, load
from lucid.misc.io.showing import _image_url
import lucid.scratch.web.svelte as lucid_svelte


/tools/node/bin/svelte -> /tools/node/lib/node_modules/svelte-cli/bin.js
/tools/node/lib
└─┬ svelte-cli@2.2.0 
  └── svelte@1.64.1 


In [3]:
googlenet = models.InceptionV1()
googlenet.load_graphdef()

def get_acts(layer, img):
  with tf.Graph().as_default(), tf.Session():
    t_input = tf.placeholder(tf.float32, [224, 224, 3])
    T = render.import_model(googlenet, t_input, t_input)
    acts = T(layer).eval({t_input: img})[0]
  return acts

def many_cossim(us,v):
  us_mags = np.sqrt((us*us).sum(-1)) + 1e-4
  v_mag = np.sqrt((v*v).sum()) + 1e-4
  return (us*v).sum(-1) / us_mags / v_mag

def cossim_grid(acts1, acts2):
  cossims = [[many_cossim(acts2, v)
   for v in row] for row in acts1]
  return np.asarray(cossims)

In [4]:
img = load("https://storage.googleapis.com/lucid-static/building-blocks/examples/dog_cat.png")
acts = get_acts("mixed4d", img)
print acts.shape


(14, 14, 528)

In [5]:
cossims = cossim_grid(acts, acts)
print cossims.shape
print ""
cossims_flat = np.hstack(np.hstack(cossims))
show(cossims_flat)


(14, 14, 14, 14)


In [6]:
%%html_define_svelte CossimOverlay

<div class="container" style="width: {{size}}px; height: {{size}}px; ">
  <div class="image" style="background-image: url({{image_url}}); z-index: -10; width: {{size}}px; height: {{size}}px;"></div>
  <div class="overlay" style="z-index: 10; width: {{size}}px; height: {{size}}px; left: {{size/2-N/2}}px; top:{{size/2-N/2}}px">
    <div class="overlay-inner" style="background-image: url({{masks_url}}); width: {{N}}px; height: {{N}}px; transform: scale({{size/N}}); background-position: {{-pos[0]*N}}px {{-pos[1]*N}}px; opacity: 0.7;">
    </div>
  </div>
  <div class="event-catcher" style="z-index: 20; width: {{size}}px; height: {{size}}px;" on:mousemove="set({pos: [Math.floor(N*event.offsetX/size), Math.floor(N*event.offsetY/size)]})"></div>
</div>

<div>{{pos}}</div>

<style>
  .container {
    position: relative;
  }
  .image, .overlay, .event-catcher {
    position: absolute;
    left: 0px;
    top: 0px;
  }
  .overlay-inner {
    image-rendering: pixelated;
  }
</style>

<script>

  
  export default {
    data () {
      return {
        image_url: undefined,
        size: undefined,
        N: undefined,
        masks_url: undefined,
        pos: [0,0]
      };
    },
    computed: {
    },
    helpers: {}
  };
</script>


Trying to build svelte component from html...
svelte compile --format iife /tmp/svelte_UDSbHn/CossimOverlay_ecf230.html > /tmp/svelte_UDSbHn/CossimOverlay_ecf230.js
svelte version 1.64.1
compiling ../tmp/svelte_UDSbHn/CossimOverlay_ecf230.html...


In [7]:
lucid_svelte.CossimOverlay({
    "image_url": _image_url(img),
    "masks_url": _image_url(cossims_flat),
    "size": 224,
    "N": cossims.shape[0]
})



In [60]:
%%html_define_svelte CossimOverlayMulti


{{#each range(n_images) as n_img}}
<div class="container" style="width: {{size}}px; height: {{size}}px; float: left; margin: 5px;">
  <div class="image" style="background-image: url({{image_urls[n_img]}}); z-index: -10; width: {{size}}px; height: {{size}}px;"></div>
  <div class="overlay" style="z-index: 10; width: {{size}}px; height: {{size}}px; left: {{size/2-N/2}}px; top:{{size/2-N/2}}px">
    <div class="overlay-inner" style="width: {{N}}px; height: {{N}}px; transform: scale({{size/N}}); background-image: url({{(pos == undefined)? '' : masks_urls[pos[0]][n_img]}});  background-position: {{(pos == undefined)? '' : -pos[1]*N}}px {{(pos == undefined)? '' : -pos[2]*N}}px; opacity: 0.7;">
    </div>
  </div>
  <div class="event-catcher" 
       style="z-index: 20; width: {{size}}px; height: {{size}}px;"
       on:mousemove="set({pos: [n_img, Math.floor(N*event.offsetX/size), Math.floor(N*event.offsetY/size)]})"
       on:mouseout="set({pos: undefined})"
       ></div>
</div>
{{/each}}

<br style="clear: both;">

<div>{{pos}}</div>

<style>
  .container {
    position: relative;
  }
  .image, .overlay, .event-catcher {
    position: absolute;
    left: 0px;
    top: 0px;
  }
  .overlay-inner {
    image-rendering: pixelated;
  }
</style>

<script>

  function range(n){
    return Array(n).fill().map((_, i) => i);
  }
  
  
  export default {
    data () {
      return {
        image_urls: undefined,
        size: undefined,
        N: undefined,
        n_images: undefined,
        masks_urls: undefined,
        pos: undefined,
      };
    },
    computed: {
    },
    helpers: {range}
  };
</script>


Trying to build svelte component from html...
svelte compile --format iife /tmp/svelte_UDSbHn/CossimOverlayMulti_202f54d.html > /tmp/svelte_UDSbHn/CossimOverlayMulti_202f54d.js
svelte version 1.64.1
compiling ../tmp/svelte_UDSbHn/CossimOverlayMulti_202f54d.html...


In [0]:
img_urls = ["https://storage.googleapis.com/lucid-static/building-blocks/examples/dog_cat.png",
            "https://storage.googleapis.com/lucid-static/building-blocks/examples/flowers.png",
            "https://storage.googleapis.com/lucid-static/building-blocks/examples/pig.jpeg"]
imgs = [load(url) for url in img_urls]
acts_ = [get_acts("mixed4d", img) for img in imgs]
grids = [[np.hstack(np.hstack(cossim_grid(acts1, acts2)))
          for acts2 in acts_] for acts1 in acts_]

In [61]:
lucid_svelte.CossimOverlayMulti({
    "image_urls": [_image_url(img) for img in imgs],
    "masks_urls": [[_image_url(cossims_flat) for cossims_flat in row] for row in grids],
    "size": 224,
    "N": cossims.shape[0],
    "n_images": len(imgs),
})



In [0]:


In [67]:
img_urls = ["https://storage.googleapis.com/lucid-static/building-blocks/examples/dog_cat.png",
            "https://storage.googleapis.com/lucid-static/building-blocks/examples/flowers.png",
            "https://storage.googleapis.com/lucid-static/building-blocks/examples/pig.jpeg"]
imgs = [load(url) for url in img_urls]
acts_ = [get_acts("mixed4a", img) for img in imgs]
grids = [[np.hstack(np.hstack(cossim_grid(acts1, acts2)))
          for acts2 in acts_] for acts1 in acts_]

In [68]:
lucid_svelte.CossimOverlayMulti({
    "image_urls": [_image_url(img) for img in imgs],
    "masks_urls": [[_image_url(cossims_flat) for cossims_flat in row] for row in grids],
    "size": 224,
    "N": acts_[0].shape[1],
    "n_images": len(imgs),
})



In [0]: