XPCS Pipeline for GiSAXS

"This notebook corresponds to version {{ version }} of the pipeline tool: https://github.com/NSLS-II/pipelines"

This notebook begins with a raw time-series of images and ends with $g_2(t)$ for a range of $q$, fit to an exponential or stretched exponential, and a two-time correlation functoin.

Overview

  • Setup: load packages/setup path
  • Load Metadata & Image Data
  • Apply Mask
  • Clean Data: shutter open/bad frames
  • Get Q-Map
  • Get 1D curve
  • Define Q-ROI (qr, qz)
  • Check beam damage
  • One-time Correlation
  • Fitting
  • Two-time Correlation The important scientific code is imported from the chxanalys and scikit-beam project. Refer to chxanalys and scikit-beam for additional documentation and citation information.

CHX Olog NoteBook

CHX Olog (https://logbook.nsls2.bnl.gov/11-ID/)

Setup

Import packages for I/O, visualization, and analysis.


In [4]:
from chxanalys.chx_libs import (np, roi, time, datetime, os, get_events, 
                                getpass, db, get_images,LogNorm, plt,tqdm, utils, Model)

from chxanalys.chx_generic_functions import (get_detector, get_fields, get_sid_filenames,  
     load_data, load_mask,get_fields, reverse_updown, ring_edges,get_avg_img,check_shutter_open,
    apply_mask, show_img,check_ROI_intensity,run_time, plot1D, get_each_frame_intensity,                                             
    create_hot_pixel_mask,show_ROI_on_image,create_time_slice,save_lists, 
                        save_arrays, psave_obj,pload_obj, get_non_uniform_edges )
 

from chxanalys.XPCS_SAXS import (get_circular_average,save_lists,get_ring_mask, get_each_ring_mean_intensity,
                                 plot_qIq_with_ROI,save_saxs_g2,plot_saxs_g2,fit_saxs_g2,cal_g2,
                                create_hot_pixel_mask,get_circular_average,get_t_iq,save_saxs_g2,
                                plot_saxs_g2,fit_saxs_g2,fit_q2_rate,plot_saxs_two_g2,fit_q_rate,
                                circular_average,plot_saxs_g4, get_t_iqc,multi_uids_saxs_xpcs_analysis)


from chxanalys.Two_Time_Correlation_Function import (show_C12, get_one_time_from_two_time,
                                                get_four_time_from_two_time,rotate_g12q_to_rectangle)

from chxanalys.chx_compress_analysis import ( compress_eigerdata, read_compressed_eigerdata,
                                             Multifile,get_avg_imgc, get_each_frame_intensityc,
                get_each_ring_mean_intensityc, mean_intensityc,cal_waterfallc,plot_waterfallc)

from chxanalys.SAXS import fit_form_factor
from chxanalys.chx_correlationc import ( cal_g2c,Get_Pixel_Arrayc,auto_two_Arrayc,get_pixelist_interp_iq,)
from chxanalys.chx_correlationp import (cal_g2p, auto_two_Arrayp)

from chxanalys.Create_Report import (create_pdf_report, 
                                create_multi_pdf_reports_for_uids,create_one_pdf_reports_for_uids)


from chxanalys.XPCS_GiSAXS import (get_qedge,get_qmap_label,get_qr_tick_label, get_reflected_angles,
    convert_gisaxs_pixel_to_q, show_qzr_map, get_1d_qr, get_qzrmap, show_qzr_roi,get_each_box_mean_intensity,
    save_gisaxs_g2,plot_gisaxs_g2, fit_gisaxs_g2,plot_gisaxs_two_g2,plot_qr_1d_with_ROI,fit_qr_qz_rate,
                                  multi_uids_gisaxs_xpcs_analysis,plot_gisaxs_g4)

%matplotlib notebook

In [5]:
plt.rcParams.update({'figure.max_open_warning': 0})

In [6]:
#%reset

In [7]:
#%%javascript
#var nb = IPython.notebook;
#var kernel = IPython.notebook.kernel;
#var command = "NOTEBOOK_FULL_PATH = '" + nb.base_url + nb.notebook_path + "'";
#kernel.execute(command);

In [8]:
#print("NOTEBOOK_FULL_PATH:\n", NOTEBOOK_FULL_PATH)

Make a directory for saving results


In [9]:
CYCLE = '2016_3'

username = getpass.getuser()
username = 'manisen'

date_path = datetime.now().strftime('%Y/%m/%d')  # e.g., '2016/03/01'
data_dir = os.path.join('/XF11ID/analysis/', CYCLE, username, 'Results/')



##Or define data_dir here, e.g.,#data_dir = '/XF11ID/analysis/2016_2/rheadric/test/'


os.makedirs(data_dir, exist_ok=True)
print('Results from this analysis will be stashed in the directory %s' % data_dir)


Results from this analysis will be stashed in the directory /XF11ID/analysis/2016_3/manisen/Results/

Load Metadata & Image Data

Print detector, scan-id, uid, datapath of data collected.

Change these lines


In [10]:
uid = '862a66'  #count : 1 ['862a66'] (scan num: 9854) (Measurement: XPCS series .1s & .9s 500 frames )
#uid = '7bfdc3' #count : 1 ['7bfdc3'] (scan num: 9747) (Measurement: XPCS series 10Hz 1000 frames )
       
#uid = '862a66' #(scan num: 9854) (Measurement: XPCS series .1s & .9s 500 frames ) #sample medadata is grong here ...

#uid = 'ecf092'

In [11]:
data_dir = os.path.join('/XF11ID/analysis/', CYCLE, username, 'Results/%s/'%uid)

os.makedirs(data_dir, exist_ok=True)
print('Results from this analysis will be stashed in the directory %s' % data_dir)


Results from this analysis will be stashed in the directory /XF11ID/analysis/2016_3/manisen/Results/862a66/

Don't Change these lines below here


In [12]:
detector = get_detector( db[uid ] )
print ('Detector is:  %s'%detector  )
sud = get_sid_filenames(db[uid])
print ('scan_id, full-uid, data path are:  %s--%s--%s'%(sud[0], sud[1], sud[2][0] ))


Detector is:  image
scan_id, full-uid, data path are:  9854--862a668a-37b6-4b80-abb7-dd569a372283--/XF11ID/data/2016/11/17/940a9a4d-a16a-44c1-96d3_8990

In [13]:
imgs = load_data( uid, detector  )
Nimg = len(imgs)
md = imgs.md


hdf5 path = /XF11ID/data/2016/11/17/940a9a4d-a16a-44c1-96d3_8990_master.h5

In [14]:
#md

In [15]:
try:
    md['Measurement']= db[uid]['start']['Measurement']
    md['sample']=db[uid]['start']['sample']  
    #md['sample']= 'SiO2 Colloidal'  #change the sample name if the md['sample'] is wrong
    print( 'The sample is %s' %md['sample'])
    
except:
    md['Measurement']= 'Measurement'
    md['sample']='sample'


The sample is PSP4VPTAADS30NMPSP4VP24ANN

In [16]:
print( 'The data are: %s' %imgs )


The data are: <Frames>
Length: 500 frames
Frame Shape: 2167 x 2070
Pixel Datatype: uint32

Overwrite Some Metadata Due to Wrong Input


In [17]:
print( 'The Metadata are: \n%s' %md )


The Metadata are: 
{'x_pixel_size': 7.5000004e-05, 'count_time': 0.1, 'sample': 'PSP4VPTAADS30NMPSP4VP24ANN', 'beam_center_y': 346.0, 'detector_distance': 4.8899999, 'frame_time': 0.10001, 'y_pixel_size': 7.5000004e-05, 'beam_center_x': 1477.0, 'incident_wavelength': 1.2852099, 'Measurement': 'XPCS series .1s & .9s 500 frames', 'pixel_mask': array([[0, 0, 0, ..., 0, 0, 4],
       [0, 0, 0, ..., 0, 0, 0],
       [0, 0, 0, ..., 0, 0, 0],
       ..., 
       [0, 0, 0, ..., 0, 0, 0],
       [0, 0, 0, ..., 0, 0, 0],
       [0, 0, 0, ..., 0, 0, 0]], dtype=uint32)}

In [18]:
#db[uid]['start']['acquire period']

In [19]:
# The physical size of the pixels
dpix = md['x_pixel_size'] * 1000.  #in mm, eiger 4m is 0.075 mm
lambda_ =md['incident_wavelength']    # wavelegth of the X-rays in Angstroms
Ldet = md['detector_distance']  # detector to sample distance (mm), currently, *1000 for saxs, *1 for gisaxs
exposuretime= md['count_time']
try:
    acquisition_period = db[uid]['start']['acquire period']
except:    
    acquisition_period = md['frame_time']

print( 'The sample is %s'%(  md['sample']  ))
print( 'Exposuretime=%s sec, Acquisition_period=%s sec'%( exposuretime, acquisition_period  ))

timeperframe = acquisition_period#for g2
#timeperframe = exposuretime#for visiblitly
#timeperframe = 2  ## manual overwrite!!!! we apparently writing the wrong metadata....


The sample is PSP4VPTAADS30NMPSP4VP24ANN
Exposuretime=0.1 sec, Acquisition_period=1.0 sec

In [20]:
setup_pargs=dict(uid=uid, dpix= dpix, Ldet=Ldet, lambda_= lambda_, 
        timeperframe=timeperframe,  path= data_dir)

In [21]:
setup_pargs


Out[21]:
{'Ldet': 4.8899999,
 'dpix': 0.075000003562308848,
 'lambda_': 1.2852099,
 'path': '/XF11ID/analysis/2016_3/manisen/Results/862a66/',
 'timeperframe': 1.0,
 'uid': '862a66'}

Apply Mask

  • load and plot mask if exist
  • otherwise create a mask using Mask pipeline
  • Reverse the mask in y-direction due to the coordination difference between python and Eiger software
  • Reverse images in y-direction
  • Apply the mask

Change these lines


In [22]:
mask_path = '/XF11ID/analysis/2016_3/masks/'
#mask_name = 'July30_mask.npy' #>= 160 C use this one
mask_name =  'Nov16_4M-GiSAXS_mask.npy'

In [23]:
mask = load_mask(mask_path, mask_name, plot_ =  True, image_name = 'uid= %s-mask'%uid )
mask = np.array(mask, dtype = np.int32)



In [24]:
md['mask'] = mask
md['mask_file']= mask_path + mask_name 
md['NOTEBOOK_FULL_PATH'] = None #NOTEBOOK_FULL_PATH

In [25]:
maskr = mask[::-1,:]
imgsr = reverse_updown( imgs )
imgsra = apply_mask( imgsr, maskr )

In [26]:
#show_img( imgsra[0],  vmin=0.1, vmax=100, logs=True, image_name= 'uid= %s'%uid)

In [27]:
#show_img( imgsa[0],  vmin= .01, vmax=50, logs= True, image_name= 'uid= %s'%uid)

In [28]:
avg_imgr =  get_avg_img( imgsra, sampling = int(Nimg/3), plot_ = True, uid =uid)



In [30]:
#show_img( avg_imgr,  vmin= .01, vmax=500, logs= True, image_name= 'uid= %s'%uid)

In [ ]:

Determine Compress Or Not

  • For sake of simplicity, we always compress data

In [31]:
print (len( np.where(avg_imgr)[0] ) / ( imgsra[0].size))
compress =  len( np.where(avg_imgr)[0] ) / ( imgsra[0].size) < 1  #if the photon ocupation < 0.1, do compress

print (compress)
#compress = False


0.17658866305964077
True

Compress Data

  • Generate a compressed data with filename
  • Replace old mask with a new mask with removed hot pixels
  • Do average image
  • Do each image sum
  • Find badframe_list for where image sum above bad_pixel_threshold
  • Check shutter open frame to get good time series

In [32]:
good_start = 10  #make the good_start at least 2

In [33]:
if compress:
    filename = '/XF11ID/analysis/Compressed_Data' +'/uid_%s.cmp'%sud[1]
    maskr, avg_imgr, imgsum, bad_frame_list = compress_eigerdata(imgsr, maskr, md, filename, 
                     force_compress= False, bad_pixel_threshold= 5e10,nobytes=4, para_compress=True)    
    
    min_inten = 0    
    good_start = max(good_start, np.where( np.array(imgsum) > min_inten )[0][0] )    
    print ('The good_start frame number is: %s '%good_start)
    
    FD = Multifile(filename, good_start, len(imgsr))
    
    plot1D( y = imgsum[ np.array( [i for i in np.arange( len(imgsum)) if i not in bad_frame_list])],
           title ='Uid= %s--imgsum'%uid, xlabel='Frame', ylabel='Total_Intensity', legend=''   )


Averaging images:   2%|▏         | 12/500 [00:00<00:04, 118.06it/s]
Using already created compressed file with filename as :/XF11ID/analysis/Compressed_Data/uid_862a668a-37b6-4b80-abb7-dd569a372283.cmp.
Averaging images: 100%|██████████| 500/500 [00:04<00:00, 116.77it/s]
Get each frame intensity: 100%|██████████| 500/500 [00:00<00:00, 531.62it/s]
No bad frames are involved.
The good_start frame number is: 10 

/opt/conda_envs/analysis/lib/python3.5/site-packages/matplotlib/axes/_axes.py:531: UserWarning: No labelled objects found. Use label='...' kwarg on individual plots.
  warnings.warn("No labelled objects found. "

In [34]:
#%system ls -lh {sud[2][0]+"*"}|tail -2 ; ls -lh {filename}

In [35]:
bad_pixel_threshold= 8.6e15  #if re-define a bad pixel threshold
bad_pixel_low_threshold= 1e7 #if re-define a bad pixel low threshold

In [36]:
if bad_pixel_threshold<1e14:
    mask, avg_img, imgsum, bad_frame_list = compress_eigerdata(imgs, mask, md, filename, 
                    force_compress=False, bad_pixel_threshold= bad_pixel_threshold,
                bad_pixel_low_threshold=bad_pixel_low_threshold, nobytes=4)
    min_inten = 10
    good_start = max(good_start, np.where( np.array(imgsum) > min_inten )[0][0] )   
    
    print ('The good_start frame number is: %s '%good_start)

Define a good time series by defining a good start and good end


In [37]:
if False:
        good_start =0  #0
        min_inten = 10
        good_start =  max(good_start, np.where( np.array(imgsum) > min_inten )[0][0])    
        good_end =     len(imgsr)
        filename = '/XF11ID/analysis/Compressed_Data' +'/uid_%s.cmp'%sud[1] 
        FD = Multifile(filename, good_start, good_end)
        avg_img= get_avg_imgc( FD,  beg= None,end=None, plot_=False )
        imgsum,bad_frame_list = get_each_frame_intensityc( FD, bad_pixel_threshold= bad_pixel_threshold,
                    bad_pixel_low_threshold=bad_pixel_low_threshold, plot_=False )

In [38]:
if not compress:   
    #sampling = 1   #sampling should be one    
    sampling = 100  #sampling should be one     
    
    good_start = check_shutter_open( imgsra,  min_inten=5, time_edge = [0,10], plot_ = False )
    print ('The good_start frame number is: %s '%good_start)
    good_series = max(good_start, apply_mask( imgsra[good_start:], maskr ))
    avg_imgr =  get_avg_img( good_series, sampling = sampling, plot_ = False, uid =uid)
    imgsum, bad_frame_list = get_each_frame_intensity(good_series ,sampling = sampling, 
                                bad_pixel_threshold=1.2e8,  plot_ = False, uid=uid)

In [39]:
#print ('The bad frame list is: %s'% bad_frame_list)
print ('The number of bad frames is : %s '%len(bad_frame_list))
print ('The good_start frame number is: %s '%good_start)
md['good_start'] = good_start
md['bad_frame_list'] = bad_frame_list


The number of bad frames is : 0 
The good_start frame number is: 10 

In [40]:
imgsum_y = imgsum[ np.array( [i for i in np.arange( len(imgsum)) if i not in bad_frame_list])]
imgsum_x = np.arange( len( imgsum_y))
save_lists(  [imgsum_x, imgsum_y], label=['Frame', 'Total_Intensity'], filename='uid=%s-imgsum'%uid, path= data_dir  )

In [41]:
plot1D( y = imgsum_y, title ='uid=%s--img-sum-t'%uid, xlabel='Frame',
       ylabel='Total_Intensity', legend='imgsum', save=True, path=data_dir)


Plot intensity average image


In [42]:
#avg_img = get_avg_imgc( FD,  beg=0,end=10000,sampling = 1, plot_ = False )
show_img( avg_imgr,  vmin=0.01, vmax= 60.0, logs=True, image_name= 'uid=%s--img-avg-'%uid,
        save=True, path=data_dir) 
md['avg_img'] = avg_imgr


Get Q-Map (Qz and Qr)

  • Users put incident-Beam and Reflection_Beam Centers here!!!

Change these lines


In [43]:
inc_x0 = 1473
inc_y0 = 372

refl_x0 = 1473
refl_y0 = 730

Don't Change these lines below here


In [44]:
alphaf,thetaf, alphai, phi = get_reflected_angles( inc_x0, inc_y0,refl_x0 , refl_y0, Lsd=Ldet )
qx, qy, qr, qz = convert_gisaxs_pixel_to_q( inc_x0, inc_y0,refl_x0,refl_y0, lamda=lambda_, Lsd=Ldet )


The incident_angle (alphai) is: 0.157298186294
The incident_angle (alphai) is: 0.157298186294

In [45]:
ticks = show_qzr_map(  qr,qz, inc_x0, data = avg_imgr, Nzline=10,  Nrline=10   )


Define Q-ROI

  • User provide the interested Qz and Qr here for XPCS analysis

Change these lines


In [80]:
qz_start = 0.03
qz_end = 0.04

qz_num= 1
qz_width = (qz_end - qz_start)/(qz_num +1)

qr_start =  0.002
qr_end = 0.07
qr_num = 10
qr_width = ( qr_end- qr_start)/(qr_num+5)

Qr = [qr_start , qr_end, qr_width, qr_num]
Qz=  [qz_start,   qz_end,  qz_width , qz_num ]

Don't Change these lines below here

  • Create label array (Qz, Qr, Qzr boxes)

In [81]:
qr_edge, qr_center = get_qedge(qr_start, qr_end, qr_width, qr_num )
qz_edge, qz_center = get_qedge(qz_start, qz_end, qz_width, qz_num )
qz_center1 =   qz_center     
label_array_qz = get_qmap_label(qz, qz_edge)
label_array_qr = get_qmap_label(qr, qr_edge)

label_array_qzr, qzc, qrc = get_qzrmap(label_array_qz, label_array_qr, 
                                       qz_center, qr_center)

labels_qzr, indices_qzr = roi.extract_label_indices(label_array_qzr)


labels_qz, indices_qz = roi.extract_label_indices(label_array_qz)
labels_qr, indices_qr = roi.extract_label_indices(label_array_qr)

num_qz = len(np.unique(labels_qz))
num_qr = len(np.unique(labels_qr))
num_qzr = len(np.unique(labels_qzr))
  • Extract the labeled array

In [82]:
boxes = label_array_qzr 
box_maskr_ = boxes*maskr

The second ROI


In [83]:
qz_start = 0.04
qz_end = 0.050

qz_num= 1
qz_width = (qz_end - qz_start)/(qz_num +1)

qr_start =  0.002
qr_end = 0.064
qr_num = 10
qr_width = ( qr_end- qr_start)/(qr_num+5)

Qr = [qr_start , qr_end, qr_width, qr_num]
Qz=  [qz_start,   qz_end,  qz_width , qz_num ]

In [84]:
qr_edge, qr_center = get_qedge(qr_start, qr_end, qr_width, qr_num )
qz_edge, qz_center = get_qedge(qz_start, qz_end, qz_width, qz_num )
qz_center2 =   qz_center   
label_array_qz = get_qmap_label(qz, qz_edge)
label_array_qr = get_qmap_label(qr, qr_edge)

label_array_qzr, qzc, qrc = get_qzrmap(label_array_qz, label_array_qr, 
                                       qz_center, qr_center)

labels_qzr, indices_qzr = roi.extract_label_indices(label_array_qzr)


labels_qz, indices_qz = roi.extract_label_indices(label_array_qz)
labels_qr, indices_qr = roi.extract_label_indices(label_array_qr)

num_qz = len(np.unique(labels_qz))
num_qr = len(np.unique(labels_qr))
num_qzr = len(np.unique(labels_qzr))

In [85]:
qz_center = qz_center1 +    qz_center2

In [87]:
#qz_center

In [51]:
boxes2 = label_array_qzr 
box_maskr2 = boxes2*maskr
#box_maskr2[np.where( box_maskr2 )] += np.max( box_maskr  )

w= np.where( box_maskr2 )
box_maskr =  box_maskr_.copy()
box_maskr[w]  = box_maskr2[w] + np.max( box_maskr_  ) 
qind, pixelist = roi.extract_label_indices(box_maskr)
noqs = len(np.unique(qind))

In [52]:
md['ring_mask'] = box_maskr
md['qr_center']= qr_center
md['qr_edge'] = qr_edge
md['qz_center']= qz_center
md['qz_edge'] = qz_edge
md['beam_center_x'] = inc_x0
md['beam_center_y']=  inc_y0
md['refl_center_x'] = refl_x0
md['refl_center_y']=  refl_y0
md['incident angle'] = alphai*180/np.pi
md['data_dir'] = data_dir

psave_obj(  md, data_dir + 'uid=%s-md'%uid ) #save the setup parameters

In [53]:
show_qzr_roi( avg_imgr, box_maskr, inc_x0, ticks, alpha=0.5, save=True, path=data_dir, uid=uid   )



In [ ]:

  • Number of pixels in each q box

In [54]:
nopr = np.bincount(qind, minlength=(noqs+1))[1:]
nopr


Out[54]:
array([8003, 8018, 8020, 8014, 7326, 8020, 4398, 3934, 3938, 3948, 7370,
       7370, 7370, 7504, 7370, 7370, 7370, 3752, 3685, 3685])
  • Plot average image with interested Q-regions (boxes)

Get 1D Curve (Q||-intensity)

  • Users put cuts here for static analysis

In [55]:
qr_1d = get_1d_qr( avg_imgr, Qr, Qz, qr, qz, inc_x0,  None,  True, ticks, .8,
                  save= True, setup_pargs=setup_pargs )


The qr_edge is:  [ 0.002       0.00613333  0.00842963  0.01256296  0.01485926  0.01899259
  0.02128889  0.02542222  0.02771852  0.03185185  0.03414815  0.03828148
  0.04057778  0.04471111  0.04700741  0.05114074  0.05343704  0.05757037
  0.05986667  0.064     ]
The qr_center is:  [ 0.00406667  0.0104963   0.01692593  0.02335556  0.02978519  0.03621481
  0.04264444  0.04907407  0.0555037   0.06193333]
The qz_edge is:  [ 0.04   0.045]
The qz_center is:  [0.042499999999999996]
The qr_1d is saved in /XF11ID/analysis/2016_3/manisen/Results/862a66/ with filename as uid=862a66--qr_1d.csv

In [56]:
fig,ax = plt.subplots()
ax.loglog( np.array( qr_1d['qr0'] ),  np.array(qr_1d['0.0246']),    ls='-', label = 'qr0' )
ax.loglog( np.array( qr_1d['qr1'] ),  np.array(qr_1d['0.0355']),    ls='-', label = 'qr1' )
ax.loglog( np.array( qr_1d['qr2'] ),  np.array(qr_1d['0.0464']),    ls='-', label = 'qr2' )
ax.legend( loc='best')
#plot1D( x = np.array( qr_1d['qr0'] ), y =  np.array(qr_1d['0.0246']), logxy=True, ax=ax, legend='qr0' )
#plot1D( x = np.array( qr_1d['qr1'] ), y =  np.array(qr_1d['0.0355']), logxy=True, ax=ax, legend='qr1' )


---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
/opt/conda_envs/analysis/lib/python3.5/site-packages/pandas/indexes/base.py in get_loc(self, key, method, tolerance)
   1944             try:
-> 1945                 return self._engine.get_loc(key)
   1946             except KeyError:

pandas/index.pyx in pandas.index.IndexEngine.get_loc (pandas/index.c:4154)()

pandas/index.pyx in pandas.index.IndexEngine.get_loc (pandas/index.c:4018)()

pandas/hashtable.pyx in pandas.hashtable.PyObjectHashTable.get_item (pandas/hashtable.c:12368)()

pandas/hashtable.pyx in pandas.hashtable.PyObjectHashTable.get_item (pandas/hashtable.c:12322)()

KeyError: '0.0246'

During handling of the above exception, another exception occurred:

KeyError                                  Traceback (most recent call last)
<ipython-input-56-4a2363b1c7d4> in <module>()
      1 fig,ax = plt.subplots()
----> 2 ax.loglog( np.array( qr_1d['qr0'] ),  np.array(qr_1d['0.0246']),    ls='-', label = 'qr0' )
      3 ax.loglog( np.array( qr_1d['qr1'] ),  np.array(qr_1d['0.0355']),    ls='-', label = 'qr1' )
      4 ax.loglog( np.array( qr_1d['qr2'] ),  np.array(qr_1d['0.0464']),    ls='-', label = 'qr2' )
      5 ax.legend( loc='best')

/opt/conda_envs/analysis/lib/python3.5/site-packages/pandas/core/frame.py in __getitem__(self, key)
   1995             return self._getitem_multilevel(key)
   1996         else:
-> 1997             return self._getitem_column(key)
   1998 
   1999     def _getitem_column(self, key):

/opt/conda_envs/analysis/lib/python3.5/site-packages/pandas/core/frame.py in _getitem_column(self, key)
   2002         # get column
   2003         if self.columns.is_unique:
-> 2004             return self._get_item_cache(key)
   2005 
   2006         # duplicate columns & possible reduce dimensionality

/opt/conda_envs/analysis/lib/python3.5/site-packages/pandas/core/generic.py in _get_item_cache(self, item)
   1348         res = cache.get(item)
   1349         if res is None:
-> 1350             values = self._data.get(item)
   1351             res = self._box_item_values(item, values)
   1352             cache[item] = res

/opt/conda_envs/analysis/lib/python3.5/site-packages/pandas/core/internals.py in get(self, item, fastpath)
   3288 
   3289             if not isnull(item):
-> 3290                 loc = self.items.get_loc(item)
   3291             else:
   3292                 indexer = np.arange(len(self.items))[isnull(self.items)]

/opt/conda_envs/analysis/lib/python3.5/site-packages/pandas/indexes/base.py in get_loc(self, key, method, tolerance)
   1945                 return self._engine.get_loc(key)
   1946             except KeyError:
-> 1947                 return self._engine.get_loc(self._maybe_cast_indexer(key))
   1948 
   1949         indexer = self.get_indexer([key], method=method, tolerance=tolerance)

pandas/index.pyx in pandas.index.IndexEngine.get_loc (pandas/index.c:4154)()

pandas/index.pyx in pandas.index.IndexEngine.get_loc (pandas/index.c:4018)()

pandas/hashtable.pyx in pandas.hashtable.PyObjectHashTable.get_item (pandas/hashtable.c:12368)()

pandas/hashtable.pyx in pandas.hashtable.PyObjectHashTable.get_item (pandas/hashtable.c:12322)()

KeyError: '0.0246'

In [ ]:


In [57]:
plot_qr_1d_with_ROI( qr_1d, qr_center, loglog=False, save=True, setup_pargs=setup_pargs )


  • Check one box intensity

In [58]:
roi_inten = check_ROI_intensity( avg_imgr, box_maskr, ring_number= 4, uid =uid )


Check beam damage

  • do a waterfall analysis

In [59]:
if compress:
    qindex = 4
    wat = cal_waterfallc( FD, box_maskr, qindex= qindex, save =True, path=data_dir, uid=uid)


Get waterfall for q index=4: 100%|██████████| 490/490 [00:01<00:00, 313.64it/s]

In [60]:
if compress:
    plot_waterfallc( wat, qindex, aspect=None,  vmax= 5, uid=uid, save =True, 
                    path=data_dir, beg= FD.beg)


  • check mean intensity of each box as a function of time

In [61]:
if compress:
    times, mean_int_sets = get_each_ring_mean_intensityc(FD, box_maskr,
                        timeperframe = None, plot_ = True, uid = uid , save=True, path=data_dir  )
    ring_avg = np.average( mean_int_sets, axis=0)
    
else:
    
    mean_int_sets = get_each_ring_mean_intensity(good_series, box_maskr, sampling = sampling,
                                timeperframe = md['frame_time']*sampling, plot_ = True, uid = uid  )


Get ROI intensity of each frame: 100%|██████████| 490/490 [00:01<00:00, 290.92it/s]

In [62]:
#plot1D( x= range( len(mean_int_sets)), y= mean_int_sets[:,1])

One time Correlation

Note : Enter the number of buffers for Muliti tau one time correlation number of buffers has to be even. More details in https://github.com/scikit-beam/scikit-beam/blob/master/skbeam/core/correlation.py

if define another good_series


In [63]:
if False:
    if compress:
        good_end =  4000 #len(imgs)
        FD = Multifile(filename, good_start,good_end  )
    else:
        good_start = 1
        good_end =  1000
        good_series = apply_mask( imgs[good_start:good_end-1], mask )

In [64]:
lag_steps = None

In [65]:
bad_frame_list


Out[65]:
array([], dtype=int64)

In [66]:
para_cal = True   #if True to use the parallel calculation

In [67]:
t0 = time.time()
if compress:
    if para_cal:
        g2, lag_steps  =cal_g2p( FD,  box_maskr, bad_frame_list, good_start, num_buf = 8, 
                        imgsum= None, norm=None )
    else:
        g2, lag_steps  =cal_g2c( FD,  box_maskr, bad_frame_list, good_start, num_buf = 8, 
                        imgsum= None, norm=None )        
else:
    bad_image_process = False
    if  len(bad_frame_list):
        bad_image_process = True
    print( bad_image_process  )    

    g2, lag_steps  =cal_g2( good_series,  box_maskr, bad_image_process,
                       bad_frame_list, good_start, num_buf = 8 )
run_time(t0)


In this g2 calculation, the buf and lev number are: 8--8--
490 frames will be processed...
100%|██████████| 20/20 [00:04<00:00,  4.60it/s]
G2 calculation DONE!
Total time: 0.16 min

In [68]:
lag_steps


Out[68]:
array([  0,   1,   2,   3,   4,   5,   6,   7,   8,  10,  12,  14,  16,
        20,  24,  28,  32,  40,  48,  56,  64,  80,  96, 112, 128, 160,
       192, 224, 256, 320, 384])

In [88]:
taus = lag_steps * timeperframe
res_pargs = dict(taus=taus, qz_center= qz_center, qr_center=qr_center,  path=data_dir, uid=uid +'_twoROI' )

In [70]:
timeperframe


Out[70]:
1.0

In [90]:
save_gisaxs_g2(  g2, res_pargs )


The correlation function of uid= 862a66_twoROI is saved with filename as /XF11ID/analysis/2016_3/manisen/Results/862a66/g2-862a66_twoROI-20161117-1706-.csv

Plot the one time correlation functions


In [91]:
plot_gisaxs_g2( g2, taus,  vlim=[0.95, 1.3], res_pargs=res_pargs, one_plot=True)


Fit g2


In [92]:
fit= True

In [93]:
if fit:
    fit_result = fit_gisaxs_g2( g2, res_pargs, function = 'stretched',  vlim=[0.95, 1.3], 
                fit_variables={'baseline':True, 'beta':True, 'alpha':False,'relaxation_rate':True},
                guess_values={'baseline':1.229,'beta':0.05,'alpha':1.0,'relaxation_rate':0.01},
                              one_plot= True)



In [94]:
psave_obj( fit_result, data_dir + 'uid=%s-g2-fit-para'%uid )

Plot the relaxation time versus qr for different qz


In [95]:
fit_qr_qz_rate(  qr_center, qz_center, fit_result, power_variable= False,
           uid=uid, path= data_dir )


The fitted diffusion coefficient D0 is:  2.231e+00   A^2S-1
The fitted diffusion coefficient D0 is:  7.926e+00   A^2S-1
Out[95]:
array([ 2.23112681,  7.92643378])

For two-time


In [96]:
#if compress:
#    FD = Multifile(filename, 0, Nimg)

In [97]:
run_two_time =  True

In [98]:
para_cal = False #True

In [99]:
if run_two_time:
    
    if compress:
        norm = None
        data_pixel =   Get_Pixel_Arrayc( FD, pixelist,  norm=norm ).get_data()
        if para_cal:
            g12b = auto_two_Arrayp(  data_pixel,  box_maskr, index = None   )    
        else:    
            g12b = auto_two_Arrayc(  data_pixel,  box_maskr, index = None   ) 
           
        if lag_steps is None:
            num_bufs=8
            noframes = FD.end - FD.beg
            num_levels = int(np.log( noframes/(num_bufs-1))/np.log(2) +1) +1
            tot_channels, lag_steps, dict_lag = multi_tau_lags(num_levels, num_bufs)
            max_taus= lag_steps.max()
    else:
        qind, pixelist = roi.extract_label_indices(   box_maskr  )
        t0 = time.time()
        data_pixel =   Get_Pixel_Array( good_series , pixelist).get_data()
        run_time(t0)
        g12b = auto_two_Array( good_series, box_maskr, data_pixel = data_pixel )


100%|██████████| 490/490 [00:03<00:00, 158.88it/s]
100%|██████████| 20/20 [00:00<00:00, 23.45it/s]

In [100]:
if run_two_time:
    show_C12(g12b, q_ind= 1, N1=0, N2=100, vmin=1.01, vmax=1.2,
             timeperframe=timeperframe,save=True, path= data_dir, uid = uid )



In [101]:
if run_two_time:
    if lag_steps is None:
        num_bufs=8
        noframes = FD.end - FD.beg
        num_levels = int(np.log( noframes/(num_bufs-1))/np.log(2) +1) +1
        tot_channels, lag_steps, dict_lag = multi_tau_lags(num_levels, num_bufs)
        max_taus= lag_steps.max()
    
    max_taus= lag_steps.max()
    t0=time.time()
    g2b = get_one_time_from_two_time(g12b)[:max_taus]
    run_time(t0)
    taus2 = np.arange( g2b.shape[0])[:max_taus] *timeperframe

    res_pargs2 = dict(taus=taus2, qz_center=qz_center, qr_center=qr_center,
             path=data_dir, uid=uid + 'g2_from_two-time'        )        
    save_gisaxs_g2(  g2b, res_pargs2,  taus=np.arange( g2b.shape[0]) *timeperframe,
                 filename='g2_from_two-time')


Total time: 0.00 min
The correlation function of uid= 862a66g2_from_two-time is saved with filename as /XF11ID/analysis/2016_3/manisen/Results/862a66/g2_from_two-time

In [102]:
if run_two_time:
    plot_gisaxs_two_g2( g2, taus, 
                 g2b, np.arange( g2b.shape[0]) *timeperframe,
                 res_pargs=res_pargs, vlim=[.9, 1.3],one_plot= True, uid =uid )


Four Time Correlation


In [103]:
run_four_time = False

In [104]:
if run_four_time:
    t0=time.time()
    g4 = get_four_time_from_two_time(g12b, g2=g2b)[:max_taus]
    run_time(t0)

In [105]:
if run_four_time:
    taus4 = np.arange( g4.shape[0])*timeperframe
    res_pargs4 = dict(taus=taus4, qz_center=qz_center, qr_center=qr_center, path=data_dir, uid=uid )
    save_gisaxs_g2(   g4,  res_pargs4,   filename='uid=%s--g4.csv' % (uid) )

In [106]:
if run_four_time:
        plot_gisaxs_g4( g4, taus4,  vlim=[0.95, 1.05], res_pargs=res_pargs, one_plot= True)

Create a PDF report


In [107]:
create_report = True

In [108]:
pdf_out_dir = os.path.join('/XF11ID/analysis/', CYCLE, username, 'Results/')

In [109]:
if create_report:
    c= create_pdf_report(  data_dir, uid, pdf_out_dir,
                    filename= "XPCS_Analysis_Report_for_uid=%s.pdf"%uid, report_type='gisaxs')
    
    #Page one: Meta-data/Iq-Q/ROI
    c.report_header(page=1)
    c.report_meta( top=730)
    c.report_static( top=560, iq_fit =None )
    c.report_ROI( top= 300)
    
    #Page Two: img~t/iq~t/waterfall/mean~t/g2/rate~q
    c.new_page()
    c.report_header(page=2)
    c.report_time_analysis( top= 720)
    c.report_one_time( top= 350)  
    
    #Page Three: two-time/two g2
    if run_two_time:
        c.new_page()
        c.report_header(page=3)
        c.report_two_time(  top= 720 )      

    if run_four_time:
        c.new_page()
        c.report_header(page=4)
        c.report_four_time(  top= 720 ) 
        
        
    c.save_page()
    c.done()


****************************************
The pdf report is created with filename as: /XF11ID/analysis/2016_3/manisen/Results/XPCS_Analysis_Report_for_uid=862a66.pdf
****************************************

The End!

Attach the PDF report to Olog


In [90]:
from chxanalys.chx_olog import LogEntry,Attachment, update_olog_uid, update_olog_id

In [91]:
os.environ['HTTPS_PROXY'] = 'https://proxy:8888'
os.environ['no_proxy'] = 'cs.nsls2.local,localhost,127.0.0.1'

In [92]:
c.filename


Out[92]:
'/XF11ID/analysis/2016_3/manisen/Results/XPCS_Analysis_Report_for_uid=862a66.pdf'

In [ ]:


In [ ]:
filename = c.filename
atch=[  Attachment(open(filename, 'rb')) ] 

update_olog_uid( uid=uid, text='Add XPCS Analysis PDF Report', attachments= atch )


Olog Password (username = manisen):········

Attach the Analysis Notebook to Olog


In [ ]:
#NOTEBOOK_FULL_PATH

In [ ]:
#note_path = '2016_2/yuzhang/August/XPCS_GiSAXS_Single_Run_Sep.ipynb'

In [ ]:
#filename = '/XF11ID/analysis/'+ note_path #NOTEBOOK_FULL_PATH

#atch=[  Attachment(open(filename, 'rb')) ] 
#update_olog_uid( uid=uid, text='Add XPCS Analysis notebook', attachments= atch )

In [ ]:
uid

In [ ]: