In [0]:
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
In [2]:
!pip install --quiet lucid==0.0.5
!npm install -g svelte-cli@2.2.0
#tensorflow_version only works on colab
%tensorflow_version 1.x
import numpy as np
import tensorflow as tf
assert tf.__version__.startswith('1')
import lucid.modelzoo.vision_models as models
import lucid.optvis.render as render
from lucid.misc.io import show, load
from lucid.misc.io.showing import _image_url
import lucid.scratch.web.svelte as lucid_svelte
In [3]:
googlenet = models.InceptionV1()
googlenet.load_graphdef()
def get_acts(layer, img):
with tf.Graph().as_default(), tf.Session():
t_input = tf.placeholder(tf.float32, [224, 224, 3])
T = render.import_model(googlenet, t_input, t_input)
acts = T(layer).eval({t_input: img})[0]
return acts
def many_cossim(us,v):
us_mags = np.sqrt((us*us).sum(-1)) + 1e-4
v_mag = np.sqrt((v*v).sum()) + 1e-4
return (us*v).sum(-1) / us_mags / v_mag
def cossim_grid(acts1, acts2):
cossims = [[many_cossim(acts2, v)
for v in row] for row in acts1]
return np.asarray(cossims)
In [4]:
img = load("https://storage.googleapis.com/lucid-static/building-blocks/examples/dog_cat.png")
acts = get_acts("mixed4d", img)
print acts.shape
In [5]:
cossims = cossim_grid(acts, acts)
print cossims.shape
print ""
cossims_flat = np.hstack(np.hstack(cossims))
show(cossims_flat)
In [6]:
%%html_define_svelte CossimOverlay
<div class="container" style="width: {{size}}px; height: {{size}}px; ">
<div class="image" style="background-image: url({{image_url}}); z-index: -10; width: {{size}}px; height: {{size}}px;"></div>
<div class="overlay" style="z-index: 10; width: {{size}}px; height: {{size}}px; left: {{size/2-N/2}}px; top:{{size/2-N/2}}px">
<div class="overlay-inner" style="background-image: url({{masks_url}}); width: {{N}}px; height: {{N}}px; transform: scale({{size/N}}); background-position: {{-pos[0]*N}}px {{-pos[1]*N}}px; opacity: 0.7;">
</div>
</div>
<div class="event-catcher" style="z-index: 20; width: {{size}}px; height: {{size}}px;" on:mousemove="set({pos: [Math.floor(N*event.offsetX/size), Math.floor(N*event.offsetY/size)]})"></div>
</div>
<div>{{pos}}</div>
<style>
.container {
position: relative;
}
.image, .overlay, .event-catcher {
position: absolute;
left: 0px;
top: 0px;
}
.overlay-inner {
image-rendering: pixelated;
}
</style>
<script>
export default {
data () {
return {
image_url: undefined,
size: undefined,
N: undefined,
masks_url: undefined,
pos: [0,0]
};
},
computed: {
},
helpers: {}
};
</script>
In [7]:
lucid_svelte.CossimOverlay({
"image_url": _image_url(img),
"masks_url": _image_url(cossims_flat),
"size": 224,
"N": cossims.shape[0]
})
In [60]:
%%html_define_svelte CossimOverlayMulti
{{#each range(n_images) as n_img}}
<div class="container" style="width: {{size}}px; height: {{size}}px; float: left; margin: 5px;">
<div class="image" style="background-image: url({{image_urls[n_img]}}); z-index: -10; width: {{size}}px; height: {{size}}px;"></div>
<div class="overlay" style="z-index: 10; width: {{size}}px; height: {{size}}px; left: {{size/2-N/2}}px; top:{{size/2-N/2}}px">
<div class="overlay-inner" style="width: {{N}}px; height: {{N}}px; transform: scale({{size/N}}); background-image: url({{(pos == undefined)? '' : masks_urls[pos[0]][n_img]}}); background-position: {{(pos == undefined)? '' : -pos[1]*N}}px {{(pos == undefined)? '' : -pos[2]*N}}px; opacity: 0.7;">
</div>
</div>
<div class="event-catcher"
style="z-index: 20; width: {{size}}px; height: {{size}}px;"
on:mousemove="set({pos: [n_img, Math.floor(N*event.offsetX/size), Math.floor(N*event.offsetY/size)]})"
on:mouseout="set({pos: undefined})"
></div>
</div>
{{/each}}
<br style="clear: both;">
<div>{{pos}}</div>
<style>
.container {
position: relative;
}
.image, .overlay, .event-catcher {
position: absolute;
left: 0px;
top: 0px;
}
.overlay-inner {
image-rendering: pixelated;
}
</style>
<script>
function range(n){
return Array(n).fill().map((_, i) => i);
}
export default {
data () {
return {
image_urls: undefined,
size: undefined,
N: undefined,
n_images: undefined,
masks_urls: undefined,
pos: undefined,
};
},
computed: {
},
helpers: {range}
};
</script>
In [0]:
img_urls = ["https://storage.googleapis.com/lucid-static/building-blocks/examples/dog_cat.png",
"https://storage.googleapis.com/lucid-static/building-blocks/examples/flowers.png",
"https://storage.googleapis.com/lucid-static/building-blocks/examples/pig.jpeg"]
imgs = [load(url) for url in img_urls]
acts_ = [get_acts("mixed4d", img) for img in imgs]
grids = [[np.hstack(np.hstack(cossim_grid(acts1, acts2)))
for acts2 in acts_] for acts1 in acts_]
In [61]:
lucid_svelte.CossimOverlayMulti({
"image_urls": [_image_url(img) for img in imgs],
"masks_urls": [[_image_url(cossims_flat) for cossims_flat in row] for row in grids],
"size": 224,
"N": cossims.shape[0],
"n_images": len(imgs),
})
In [0]:
In [67]:
img_urls = ["https://storage.googleapis.com/lucid-static/building-blocks/examples/dog_cat.png",
"https://storage.googleapis.com/lucid-static/building-blocks/examples/flowers.png",
"https://storage.googleapis.com/lucid-static/building-blocks/examples/pig.jpeg"]
imgs = [load(url) for url in img_urls]
acts_ = [get_acts("mixed4a", img) for img in imgs]
grids = [[np.hstack(np.hstack(cossim_grid(acts1, acts2)))
for acts2 in acts_] for acts1 in acts_]
In [68]:
lucid_svelte.CossimOverlayMulti({
"image_urls": [_image_url(img) for img in imgs],
"masks_urls": [[_image_url(cossims_flat) for cossims_flat in row] for row in grids],
"size": 224,
"N": acts_[0].shape[1],
"n_images": len(imgs),
})
In [0]: