In [1]:
pd.options.display.max_columns = 100
In [18]:
# l1adarkfnames = io.get_current_science_fnames('l1a', 'periapse*uvdark', env='production')
l1adarkfnames = !ls /Users/klay6683/Dropbox/data/iuvs/periapse_muvdarks_v02_r01/*.fits*
In [19]:
len(l1adarkfnames)
Out[19]:
In [4]:
from ipywidgets import interact, interactive, fixed
In [5]:
from iuvs import io
In [13]:
def f(i):
l1a = io.L1AReader(l1adarkfnames[i])
print('n_dims:', l1a.n_dims)
i = 0
if l1a.n_dims == 3:
i = 1
print(l1a.img.shape[i])
print(l1a.Binning['SPABINWIDTH'].shape[1])
In [22]:
interact(f, i=(0,len(l1adarkfnames)-1));
In [7]:
l1a = io.L1AReader(l1adarkfnames[0])
In [8]:
l1a.Integration
Out[8]:
In [9]:
l1a.n_dims
Out[9]:
In [10]:
l1a.img.shape
Out[10]:
In [ ]:
In [20]:
df = pd.DataFrame([io.ScienceFilename(str(i)).as_series() for i in l1adarkfnames])
In [21]:
df.columns
Out[21]:
In [22]:
df['phase'].value_counts()
Out[22]:
In [24]:
from ipyparallel import Client
c = Client()
In [25]:
dview = c.direct_view()
lview = c.load_balanced_view()
In [57]:
def check_for_issues(p):
from iuvs import exceptions, io
d = {}
d['fname'] = p.name
try:
l1a = io.L1AReader(p)
except exceptions.DimensionsError:
d['dims'] = False
d['kind'] = l1a.img_header['BIN_TBL'][:3]
d['n_unique_spabins'] = l1a.n_unique_spabins
d['n_unique_spebins'] = l1a.n_unique_spebins
return d
In [58]:
check_for_issues(df.p[4])
Out[58]:
In [59]:
doing = df.p
results = lview.map_async(check_for_issues, doing)
In [14]:
from iuvs.multitools import nb_progress_display
In [71]:
nb_progress_display(results, doing)
In [72]:
results.progress
Out[72]:
In [73]:
resultdf = pd.DataFrame(results.result)
In [77]:
resultdf.describe()
Out[77]:
In [74]:
for col in resultdf.columns:
if col == 'fname': continue
print(col)
print(resultdf[col].value_counts(dropna=False))
In [78]:
resultdf['phase'] = resultdf.fname.map(lambda x: io.ScienceFilename(x).phase)
In [90]:
subdf = resultdf[(resultdf.n_unique_spabins>1) | (resultdf.n_unique_spebins>1)]
In [91]:
subdf[subdf.kind=='LIN'].info()
In [79]:
def myfilter(fname, d):
from iuvs import exceptions, io
okay = True
try:
l1a = io.L1AReader(fname)
except exceptions.DimensionsError:
d['error'] = 'dims'
l1a = None
okay = False
return d, l1a, False
if l1a.img_header['BIN_TBL'].startswith('NON LINEAR'):
d['error'] = 'nonlinear'
okay = False
elif l1a.spatial_size != 7 or l1a.spectral_size != 256:
d['error'] = 'binning'
okay = False
return d, l1a, okay
In [80]:
def get_primary_dark(fname):
from iuvs import exceptions, io
import numpy as np
d = dict(fname=fname)
d, l1a, okay = myfilter(fname, d)
if okay is not True:
return d
if l1a.n_dims == 2:
return l1a.primary_img_dn_s[np.newaxis,...]
else:
return l1a.primary_img_dn_s
In [81]:
from iuvs.multitools import *
In [82]:
from ipywidgets import HTML
In [83]:
container = []
errors = []
prog = IntProgress(min=0, max=len(df.p))
display(prog)
t = HTML()
display(t)
for i,fname in enumerate(df.p.iloc[:]):
prog.value = i
t.value = fname.name
ret = get_primary_dark(fname)
if isinstance(ret, dict):
errors.append(ret)
else:
container.append(ret)
In [30]:
periapse_darks = np.concatenate(container, axis=0)
In [31]:
periapse_darks.shape
Out[31]:
In [32]:
panel = pd.Panel(periapse_darks)
In [33]:
panel.to_hdf('/Users/klay6683/Dropbox/data/iuvs/periapse_muvdarks_v02_r01/periapse_darks.hdf', 'panel')
In [72]:
def create_summary_df(fname):
from iuvs import exceptions, io
import numpy as np
d = dict(fname=fname)
d, l1a, okay = myfilter(fname, d)
if not okay:
return d
main_header = io.get_header_df(l1a.hdulist[0])
integration = l1a.Integration
if main_header.loc[0, 'NAXIS'] == 2:
main_header.loc[0, 'NAXIS3'] = np.nan
avgtuple = None
elif main_header.loc[0, 'NAXIS'] == 3:
avgtuple = (1,2)
else:
d['error'] = 'axes'
return d
lenint = len(integration)
if lenint > 1:
main_header = main_header.append([main_header]*(lenint-1), ignore_index=True)
joined = pd.concat([integration, main_header], axis=1)
for col in l1a.Observation.names[:-3]:
val = l1a.Observation[col][0]
if col == 'COLLECTION_ID':
val = val[0]
joined[col] = val
# savepath = io.save_to_hdf(joined.sort_index(axis=1), fname, 'l1a_dark_scans')
d['success'] = True
# return d
return joined
In [84]:
dview.push({'myfilter':myfilter})
Out[84]:
In [85]:
create_summary_df(l1adarkfnames[0])
Out[85]:
In [94]:
doing = l1adarkfnames
results = lview.map_async(create_summary_df, doing)
In [95]:
nb_progress_display(results, doing)
In [97]:
dfs = [df for df in results.result if type(df)!=dict]
In [98]:
results_df = pd.concat(dfs)
In [100]:
results_df.head()
Out[100]:
In [62]:
results_df.loc[results_df.ANC_SVN_REVISION == '', 'ANC_SVN_REVISION'] = 0
In [63]:
results_df = results_df.convert_objects(convert_numeric=True)
In [64]:
results_df.to_hdf('/home/klay6683/output/l1a_dark_scans/results_df.h5', 'df')
In [277]:
import glob
h5fnames = glob.glob("/home/klay6683/output/l1a_dark_scans/*.h5")
In [278]:
len(h5fnames)
Out[278]:
In [279]:
def chunker(seq, size):
return (seq[pos:pos + size] for pos in range(0, len(seq), size))
In [280]:
dfs = []
for i,chunk in enumerate(chunker(h5fnames, 200)):
print("Chunk {}".format(i))
frames = []
for fname in chunk:
frames.append(pd.read_hdf(fname, 'df'))
dfs.append(pd.concat(frames, ignore_index=True))
In [282]:
superdf = pd.concat(dfs, ignore_index=True)
In [283]:
superdf.info()
In [284]:
from iuvs import calib
In [285]:
superdf.DET_TEMP = superdf.DET_TEMP.map(calib.convert_det_temp_to_C)
superdf.CASE_TEMP = superdf.CASE_TEMP.map(calib.convert_case_temp_to_C)
In [286]:
superdf.to_hdf('/home/klay6683/to_keep/l1a_dark_scan.h5','df')
In [287]:
from iuvs import meta
In [103]:
from iuvs import calib
In [104]:
results_df.DET_TEMP = results_df.DET_TEMP.map(calib.convert_det_temp_to_C)
In [105]:
results_df.CASE_TEMP = results_df.CASE_TEMP.map(calib.convert_case_temp_to_C)
In [107]:
results_df.to_hdf('/Users/klay6683/Dropbox/data/iuvs/periapse_muvdarks_v02_r01/results_df.hdf',
'df')
In [ ]: