Masking Pipeline

"This notebook corresponds to version {{ version }} of the pipeline tool: https://github.com/NSLS-II/pipelines"

Version--Dec 20,2015

NSLS2 data retrieval imports


In [1]:
from databroker import DataBroker as db, get_images, get_table, get_events
from filestore.api import register_handler, deregister_handler
from filestore.retrieve import _h_registry, _HANDLER_CACHE

In [2]:
#hdr = db[{{ uid }}]

In [3]:
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib.colors import LogNorm

In [4]:
from eiger_io.pims_reader import EigerImages as Images

In [5]:
%matplotlib notebook

In [6]:
#cd /XF11ID/analysis/Analysis_Pipelines/Develop/


/XF11ID/analysis/Analysis_Pipelines/Develop

In [9]:
from chxtools import develop as dev
from chxtools import Time_Correlation_Functions as tcf
from chxtools import speckle as spc


/home/yuzhang/.conda/envs/user_analysis/lib/python3.4/site-packages/IPython/html.py:14: ShimWarning: The `IPython.html` package has been deprecated. You should import from `notebook` instead. `IPython.html.widgets` has moved to `ipywidgets`.
  "`IPython.html.widgets` has moved to `ipywidgets`.", ShimWarning)

In [10]:
#%run develop.py
#%run Time_Correlation_Functions.py
#%run speckle.py

In [29]:
import xray_vision
import xray_vision.mpl_plotting as mpl_plot  
from xray_vision.mpl_plotting import speckle
from xray_vision.mask.manual_mask import ManualMask

import skbeam.core.roi as roi

import skbeam.core.correlation as corr
import skbeam.core.utils as utils

Users put

  • uid for Bluesky Scan
  • filename for acquiring data directly by EigerSofteare

In [12]:
BlueScan = True
DirectAcq = False
detector = 'eiger_4M_cam_img_image_lightfield'  #for 4M
detector = 'eiger4m_image'  #for 4M

In [13]:
if BlueScan:
    uid = '10882a8e-7404-470a-81bb-1fad24dc3e9d'
    
    
else:
    uid = '/XF11ID/data/2015/11/23/d01ab510-3cf3-4719-bee3_795_master.h5'

Get data from the uid


In [16]:
if BlueScan:
    hdr = db[uid]
    ev, = get_events(  hdr, [detector] )
    imgs = ev['data'][detector]
else:    
    imgs =  Images(uid)
print (imgs)
Nimg=len(imgs)


hdf5 path = /XF11ID/data/2016/02/04/7d2403d3-4316-4d69-b925_249_master.h5
<Frames>
Length: 10000 frames
Frame Shape: 2167 x 2070
Pixel Datatype: uint32

Get data path


In [17]:
from datetime import datetime
dt = datetime.fromtimestamp(hdr['start'].time)
path ='/XF11ID/analysis' + '/%s/%s/%s/' % (dt.year, dt.month, dt.day)
path


Out[17]:
'/XF11ID/analysis/2016/2/4/'

Load ManuMaks or Not


In [19]:
load_ManuMask = False
ManuMask =  None

show the pixel mask


In [20]:
fig, ax = plt.subplots()
im = ax.imshow(imgs.md['pixel_mask'], vmin=0,vmax=1,cmap='viridis',origin='lower')
ax.set_title( 'pixel_mask')
fig.colorbar( im )
plt.show()


show one image with pixel_mask


In [21]:
fig, ax = plt.subplots()
im = ax.imshow(imgs[0]*imgs.md['pixel_mask'], vmin=0,vmax=1,cmap='viridis',origin='lower')
ax.set_title( 'imgs_0')
fig.colorbar( im )


Out[21]:
<matplotlib.colorbar.Colorbar at 0x7fe075434630>

Check bad images by waterfall plot


In [24]:
masker =dev.Masker( imgs, imgs.md['pixel_mask'])

In [23]:
#one option: sum of each frame intensity, this code is slow
#bin_img = 1
#imgsum = [np.sum(img* mask) for img in imgs[::bin_img ]]

In [25]:
imgs.shape


Out[25]:
(2167, 2070)

Users put interested retangle here


In [26]:
rectangle = [ [0, 1500, 2050, 1], ]  # corner y, x, length, width

In [27]:
roi_data = np.array((  rectangle ), dtype=np.int64)  
rect_array = roi.rectangles(roi_data, shape=imgs.shape) * imgs.md['pixel_mask']


---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-27-46caf35822e8> in <module>()
      1 roi_data = np.array((  rectangle ), dtype=np.int64)
----> 2 rect_array = roi.rectangles(roi_data, shape=imgs.shape) * imgs.md['pixel_mask']

NameError: name 'roi' is not defined

In [28]:
fig, axes = plt.subplots( )
axes.set_title("ROI")
im,im_label = show_label_array_on_image(axes, masker[0], rect_array, imshow_cmap='viridis', 
                cmap='Paired',    vmin=0.0001, vmax= 1, log_img=False, origin="lower")
#im = axes.imshow(rect_array, norm=LogNorm(0.001,1.0),cmap='viridis',origin='lower')

axes.set_xlim(  rectangle[0][1] -100, rectangle[0][1] +100  )
axes.set_ylim(  rectangle[0][0] -0, rectangle[0][0] + rectangle[0][2] )

fig.colorbar(im )
plt.show()


---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-28-eebe9f2d3587> in <module>()
      1 fig, axes = plt.subplots( )
      2 axes.set_title("ROI")
----> 3 im,im_label = show_label_array_on_image(axes, masker[0], rect_array, imshow_cmap='viridis', 
      4                 cmap='Paired',    vmin=0.0001, vmax= 1, log_img=False, origin="lower")
      5 #im = axes.imshow(rect_array, norm=LogNorm(0.001,1.0),cmap='viridis',origin='lower')

NameError: name 'show_label_array_on_image' is not defined

In [25]:
qind, pixelist = roi.extract_label_indices(   rect_array  )
noqs = len( np.unique(qind) )
nopr = np.bincount(qind, minlength=(noqs+1))[1:]

In [26]:
t0 = time.time()
data_pixel =   Get_Pixel_Array( imgs, pixelist).get_data()
mean_inten = get_mean_intensity( data_pixel, qind)

run_time(t0)


Total time: 4.84 min

In [27]:
bad_frame = np.where( mean_inten[1] == mean_inten[1].max())[0]
print ('The bad frame number is:  %s'%bad_frame)


The bad frame number is:  [1245]

In [28]:
fig, axes = plt.subplots(  )
axes.plot( mean_inten[1], '-go'  ) 
ax.set_ylabel('Intensity')
ax.set_xlabel('Frame')
ax.set_title('Kymograph_sum') 
plt.show()


Show waterfall


In [29]:
if False:
    fig, ax = plt.subplots( )
    ax.set_ylabel('Pixel')
    ax.set_xlabel('Frame')
    ax.set_title('Kymograph')
    ax.imshow(kymo.T, cmap='viridis', vmin=0.0001, vmax= 1, aspect=6.0)
    
    plt.show()

save the kymo_sum


In [30]:
np.save( path +  str(uid)+"_kymo_sum",mean_inten[1] ) 
data_pixel=[]

Remove hotspots in the image


In [31]:
hot= np.array([imgs[i].max() for i in np.random.random_integers(0, 200-1, 5)]).mean()

In [32]:
hot


Out[32]:
4294967295.0

In [33]:
mask_rh  = RemoveHotSpots( masker, hot)._get_mask(Ns=0,Ne=10)

In [34]:
makser = Masker( imgs, imgs.md['pixel_mask']*mask_rh)

In [36]:
# Plot the result
fig, ax = plt.subplots()
im = ax.imshow(makser[0],  cmap='viridis',origin='lower', 
              norm= LogNorm( vmin=0.001, vmax=1 ) )

fig.colorbar(im)
ax.set_title("Remove_hot_pixel")
plt.show()



In [37]:
(imgs[100]*imgs.md['pixel_mask']).max()


Out[37]:
4294967295

In [38]:
(imgs[100]*imgs.md['pixel_mask'] *  mask_rh ).max()


Out[38]:
14487

Import all the required packages to create manual mask

xray-vision - plotting helper functions for X-ray science https://github.com/Nikea/xray-vision


In [39]:
import xray_vision
import xray_vision.mpl_plotting as mpl_plot  
from xray_vision.mpl_plotting import speckle
from xray_vision.mask.manual_mask import ManualMask

Get the Averaged Image Data


In [40]:
good_start = 0
good_end= 7600

avg_img = np.average(imgs[good_start:good_end: 10], axis=0) #*imgs.md['pixel_mask']  #*mask_rh

In [41]:
avg_img =avg_img*imgs.md['pixel_mask']*mask_rh

In [42]:
# Plot the result
fig, ax = plt.subplots()
im = ax.imshow(avg_img,  cmap='viridis',origin='lower', 
              norm= LogNorm( vmin=0.001, vmax=1e1 ) )

fig.colorbar(im)
ax.set_title("Averaged Data")
plt.show()


save ave_img data


In [43]:
np.save( path +  str(uid)+"_avg_img",avg_img )

In [44]:
fig, ax = plt.subplots()
m = ManualMask(ax, avg_img,
               cmap='viridis',origin='lower',
               norm= LogNorm( vmin=0.001, vmax=1e0 ) )
plt.show()



In [45]:
#np.save( new_path + 'mask_SiO2',m.mask)

In [46]:
if load_ManuMask:
    
    new_mask = np.load( new_path +'mask_SiO2.npy' )
    
else:
    new_mask = m.mask

In [47]:
fig, ax = plt.subplots()
im=ax.imshow(~new_mask,origin='lower' ,vmin=0, vmax=1,cmap='viridis')
fig.colorbar(im)
plt.show()


Combine the hand-drawn mask and the pixel mask and hot pixel mask


In [48]:
mask = (~new_mask)*imgs.md['pixel_mask']*mask_rh

fig, ax = plt.subplots()
im=ax.imshow(mask, origin='lower' ,vmin=0, vmax=1,cmap='viridis')
fig.colorbar(im)
plt.show()


save the combined mask to use in data analysis


In [49]:
np.save(path + str(uid)+"_mask", mask )

The End!