In [1]:
from __future__ import print_function, division
In [2]:
# This changes the current directory to the base saga directory - make sure to run this first!
# This is necessary to be able to import the py files and use the right directories,
# while keeping all the notebooks in their own directory.
import os
import sys
if 'saga_base_dir' not in locals():
saga_base_dir = os.path.abspath('..')
if saga_base_dir not in sys.path:
os.chdir(saga_base_dir)
In [3]:
%matplotlib inline
from matplotlib import pyplot as plt
from matplotlib import rcParams
rcParams['figure.figsize'] = (16, 8)
rcParams['image.interpolation'] = 'none'
rcParams['image.origin'] = 'lower'
In [4]:
import re
import urllib2
import datetime
from glob import glob
import numpy as np
from collections import Counter, OrderedDict
from astropy import units as u
from astropy.coordinates import SkyCoord, AltAz, EarthLocation, get_sun
from astropy.time import Time
from astropy.utils.data import get_readable_fileobj
from astropy.io import fits
from astropy.table import Table, QTable, Column, MaskedColumn
from astropy import table
from IPython import display
In [5]:
for module in ['hosts', 'targeting', 'aat']:
if module in globals():
reload(globals()[module])
else:
globals()[module] = __import__(module)
#g = targeting.get_gama() #re-caches the gama catalog
In [6]:
hsd = hosts.get_saga_hosts_from_google(clientsecretjsonorfn='client_secrets.json', useobservingsummary=False)
hsd = dict([(h.name, h) for h in hsd])
In [7]:
def catalog_to_sc(cat, hostordist = None):
if hasattr(hostordist, 'dist'):
distance = hostordist.dist
else:
distance = hostordist
return SkyCoord(cat['ra']*u.deg, cat['dec']*u.deg, distance=distance)
In [8]:
nmstoget = ('Dune', 'AnaK', 'Odyssey', 'Gilgamesh', 'OBrother', 'Narnia', 'Catch22')
hosts_to_target = [h for h in hsd.values() if h.name in nmstoget]
assert len(hosts_to_target)==len(nmstoget)
new_targets = [hosts.NSAHost(145729), hosts.NSAHost(21709)]
hosts_to_target.extend(new_targets)
In [9]:
# now set to the latest base catalogs
for h in hsd.values() + hosts_to_target:
h.fnsdss = 'SAGADropbox/base_catalogs/base_sql_nsa{0}.fits.gz'.format(h.nsaid)
h._cached_sdss = None
In [10]:
# make sure the catalogs are loaded
for h in hosts_to_target:
h.get_sdss_catalog()
hosts_to_target
Out[10]:
In [11]:
# these are the already-observed objects
# we use "h" for this only as a convenience because the method adds some useful bits that aren't in the raw catalogs
spectra = h.load_and_reprocess_sdss_catalog('SAGADropbox/data/saga_spectra_clean.fits.gz')
x = base catalog on Dropbox
; CORRECT FOR EXTINCTION
r = x.r - x.extinction_r
i = x.i - x.extinction_i
g = x.g - x.extinction_g
; DEFINE GRI CRITERIA, ERRORS ARE SUBTRACTED IN QUADRATURE
cgr = (g - r) - 2.*sqrt(x.g_err^2 + x.r_err^2)
cri = (r - i) - 2.*sqrt(x.r_err^2 + x.i_err^2)
; SELECT TARGETS
qtarget = where(x.rhost_kpc le 300 and $ ; INSIDE RVIR
x.fibermag_r le 23 and $ ; FIBERMAG CUT
x.remove eq -1 and $ ; SHRED LIST
x.phot_sg eq 'GALAXY'and $ ; GALAXIES ONLY
cgr le 0.85 and cri le 0.55 and $ ; GRI CUT
x.zquality lt 3) ; NO SPECTRAThe scheme below is:
7: r<300 kpc , M_r<20, "good" colorcuts
6: r<300 kpc , 20<M_r<20.75, "good" colorcuts
5: r<300 kpc , M_r<20, "ok" colorcuts
4: r<300 kpc , 20<M_r<20.75, "ok" colorcuts
3: 450>r>300 kpc , M_r<20, "good" colorcuts
2: 450>r>300 kpc , 20<M_r<20.75, " good" colorcuts
1: everything else with r<21 (*not* M_r)
In [681]:
for h in hosts_to_target:
print('On host', h.name)
cat = h.get_sdss_catalog()
r0 = cat['r'] - cat['Ar']
cutbyrawmagcut = (r0<20.75)&(cat['r']>21)
if np.sum(cutbyrawmagcut)>0:
print('WARNING: ', np.sum(cutbyrawmagcut), 'objects have rmags in selection but are extincted too much for r>21')
pris = np.ones(len(cat))
# remove list
tokeep = cat['REMOVE']==-1
pris[~tokeep] = -cat['REMOVE'][~tokeep] # sets the REMOVE objects to -their remove value
remmskval = np.min(pris)-1
# remove anything in the remove list online but not in the catalog as remove
pris[~targeting.remove_targets_with_remlist(cat, h, maskonly=True, verbose='warning')&(pris>-1)] = remmskval
if np.sum(pris==remmskval) > 0:
print('Removed', np.sum(pris==remmskval), 'due to online remove list. Remmsk val:', remmskval)
photgood = ((cat['r'] < 21.) & # this is to cut the numbers a bit - more stringent mag cuts below
(cat['fibermag_r']<23) &
(cat['phot_sg']=='GALAXY'))
nearish = (cat['RHOST_KPC']<450) # again, to limit the numbers
pris[~photgood|~nearish] = 0
# pris ~-100 are removed due to already-observed
nospec = cat['ZQUALITY']<3
pris[~nospec] = -100-cat['ZQUALITY'][~nospec]
brighter = r0 < 20
fainter = (r0 < 20.75)&~brighter
goodcolors = targeting.colorcut_mask(cat, {'g-r': (None, 0.85, 2), 'r-i': (None, 0.55, 2)},deredden=True)
okcolors = targeting.colorcut_mask(cat, {'g-r': (None, 1.2), 'r-i': (None, 0.7)}, deredden=False)
inside = cat['RHOST_KPC']<300
pris[(pris>0)&~inside&fainter&goodcolors] = np.max(pris) + 1
pris[(pris>0)&~inside&brighter&goodcolors] = np.max(pris) + 1
pris[(pris>0)&inside&fainter&okcolors] = np.max(pris) + 1
pris[(pris>0)&inside&brighter&okcolors] = np.max(pris) + 1
pris[(pris>0)&inside&fainter&goodcolors] = np.max(pris) + 1
pris[(pris>0)&inside&brighter&goodcolors] = np.max(pris) + 1
#everything left is in pri 1
# this *shouldn't* be necessary, as ZQUALITY should be in the base catalog.
# But as a sanity check we look to see if anything in the spectral catalog is still being included
spec_this_host = spectra[spectra['HOST_NSAID']==h.nsaid]
spec_this_host = spec_this_host[np.in1d(spec_this_host['OBJID'], cat['OBJID'])]
zq = cat['ZQUALITY'].copy()
for i, zqi in zip(spec_this_host['OBJID'], spec_this_host['ZQUALITY']):
zq[cat['OBJID']==i] = zqi
if np.any(pris[zq>2]>=0):
print('POSSIBLE PROBLEM: Found some objects in spectrum list that are *not* claimed '
'as having spectra in the base catalogs. Setting them to -11x:', dict(Counter(pris[pris<-110])))
pris[zq>2] = -110 - zq[zq>2]
#de-duplicate
if len(np.unique(cat['OBJID'])) != len(cat):
_, idxs = np.unique(cat['OBJID'], return_index=True)
msk = np.ones_like(cat, dtype=bool)
msk[idxs] = 0
pris[msk] = -1000
print('WARNING: some duplicate objid found. Setting', np.sum(pris==-1000), 'dupes to pri=-1000')
cat['aat_pris'] = pris
if 'aat_pris_unjiggered' in cat.colnames:
cat.remove_column('aat_pris_unjiggered')
#informational
counter = Counter(pris)
print('Rank counts:')
for k in reversed(sorted(counter)):
print(int(k), ':', counter[k])
Note: the 2 objects in OBrother with the warning have r~21, and are definitely not interesting targets
The re-jiggered scheme is:
7: r<300 kpc , M_r<20, "good" colorcuts, ML>.001
6: r<300 kpc , 20<M_r<20.75, "good" colorcuts, ML>.001
5: r<300 kpc , M_r<20, "good" colorcuts, ML<.001
4: r<300 kpc , 20<M_r<20.75, "good" colorcuts, ML<.001
<3 : same as above
In [682]:
for h in new_targets:
print(h.name)
cat = h.get_sdss_catalog()
if 'aat_pris_unjiggered' in cat.colnames:
print('Already rejiggered', h, 'skipping (but still reporting stats)...')
pris = cat['aat_pris']
print(dict(Counter(pris[pris>0])))
print(dict(Counter(pris[pris<=0])))
continue
p = cat['PROBABILITY_CLASS1']
if np.all(p<0):
print('WARNING: host', h, 'does not have probs, so not re-jiggering based on ML probs')
continue
pris = np.array(cat['aat_pris_unjiggered' if 'aat_pris_unjiggered' in cat.colnames else 'aat_pris'])
goodcolorbright = pris == 7
goodcolorfaint = pris == 6
okcolor = (pris==5) | (pris==4)
pris[okcolor] = -0.75
pris[goodcolorfaint] = 4
pris[goodcolorbright] = 5
pris[goodcolorfaint&(p>.001)] = 6
pris[goodcolorbright&(p>.001)] = 7
print(dict(Counter(pris[pris>0])))
print(dict(Counter(pris[pris<=0])))
cat['aat_pris_unjiggered'] = cat['aat_pris']
cat['aat_pris'] = pris
In [499]:
special_objids = []
In [500]:
possible_anak_sat = SkyCoord(353.7229, 1.2064, unit=u.deg)
spectrascs = SkyCoord(spectra['ra'], spectra['dec'], unit=u.deg)
row = spectra[np.argsort(possible_anak_sat.separation(spectrascs))[0]]
special_objids.append(row['OBJID'])
row
Out[500]:
The above object was put as SAT==2, but is at very close z to AnaK...? r~=17.6 in DECALS. Marla says this is because it's too far from the host, so not worth special-izing
In [501]:
del special_objids[special_objids.index(row['OBJID'])]
In [502]:
possible_anak_sat = SkyCoord(354.286, 0.211, unit=u.deg)
anakcat = hsd['AnaK'].get_sdss_catalog()
anakscs = SkyCoord(anakcat['ra'], anakcat['dec'], unit=u.deg)
seps = possible_anak_sat.separation(anakscs)
closest = np.argsort(seps)[0]
print(seps[closest].arcsec)
row = anakcat[closest]
special_objids.append(row['OBJID'])
row
Out[502]:
This is not actually quite on the center-of-light in DECALS... but it might be an HII region?
Marla wants:RA = 354.527/DEC = 0.533964
In [503]:
possible_anak_sat = SkyCoord(354.527, 0.533964, unit=u.deg)
seps = possible_anak_sat.separation(anakscs)
closest = np.argsort(seps)[0]
print(seps[closest].arcsec)
row = anakcat[closest]
special_objids.append(row['OBJID'])
row
Out[503]:
That one seems good
In [750]:
def find_risa_objs(stringfromrisa, h):
risa_tab = Table.read(stringfromrisa, delimiter='\t', format='ascii', names=['unk', 'ra', 'dec', 'mag', 'cand'])
risa_sc = SkyCoord(risa_tab['ra'], risa_tab['dec'], unit=u.deg)
cat = h.get_sdss_catalog()
catsc = SkyCoord(cat['ra'], cat['dec'], unit=u.deg)
idx, d2d, _ = risa_sc.match_to_catalog_sky(catsc)
assertmsg = '{} matched of {}: {}'.format(np.sum(d2d < .1*u.arcsec),len(d2d), d2d[d2d > .1*u.arcsec])
assert np.all(d2d < .1*u.arcsec), assertmsg
return cat['OBJID'][idx]
In [698]:
risa_145729="""
-1 224.56410 -1.0730193 19.7973 true
-1 224.59164 -1.1261357 17.9597 true
-1 224.58605 -1.1340797 19.9805 true
-1 224.65696 -1.1021396 17.4972 true
-1 224.54478 -1.1645862 19.4057 true
-1 224.50349 -1.1783027 17.7666 true
-1 224.61258 -1.2283750 20.1190 true
-1 224.66071 -1.2407656 20.3448 true
-1 224.58210 -1.2891033 20.1278 true
-1 224.68795 -0.82928863 18.7276 true
-1 224.46354 -0.85993860 20.6228 true
-1 224.43907 -1.3290346 20.2419 true
-1 224.27041 -1.0663297 19.6382 true
-1 224.92796 -1.0868430 19.9441 true
-1 224.95218 -1.2046019 20.2506 true
-1 224.98659 -1.0996963 19.2848 true
-1 224.95028 -1.2134533 19.1667 true
-1 224.56810 -0.71035594 19.5400 true
-1 224.56710 -0.71155324 18.3361 true
-1 224.63475 -0.76637428 20.5220 true
-1 224.79342 -0.82424335 19.5245 true
-1 224.26293 -1.3190454 19.7427 true
-1 224.34037 -1.3851494 19.5061 true
-1 224.67776 -1.3717603 18.9769 true
-1 224.30819 -0.89372642 19.5476 true
-1 224.95888 -1.0081097 19.6524 true
-1 225.01145 -1.2106150 19.6745 true
-1 224.27946 -0.80572367 19.0886 true
-1 224.44473 -0.64135326 18.2103 true
-1 224.59702 -0.60626247 19.4427 true
-1 224.98059 -1.2775413 20.7447 true
-1 224.25056 -1.3830278 20.6792 true
-1 224.03729 -1.0589027 20.7401 true
-1 224.94320 -0.87332390 19.9586 true
-1 224.12169 -1.2418469 18.6920 true
-1 225.09967 -1.2117895 19.8792 true
-1 224.28313 -0.67401930 20.1558 true
-1 224.18769 -0.79627184 19.9399 true
-1 224.23305 -0.67032897 20.2131 true
-1 225.00922 -0.80628957 20.5866 true
-1 224.32848 -1.5812675 18.2125 true
-1 224.27623 -1.5292467 18.4006 true
-1 224.70055 -1.6463751 18.5479 true
-1 225.06682 -1.2727903 20.5982 true
-1 224.89664 -1.5217602 19.0338 true
-1 225.02588 -1.4044669 20.2629 true
-1 224.98083 -1.4368200 20.4261 true
-1 225.07035 -0.95870445 19.6174 true
-1 224.14144 -0.71374995 20.3682 true
-1 224.18156 -0.65458846 19.8804 true
-1 224.03748 -0.86010033 20.2428 true
-1 224.29784 -1.5985485 19.0072 true
-1 224.30080 -1.5957333 20.6291 true
-1 224.65269 -1.6814901 20.2405 true
-1 224.18598 -1.4982058 19.6720 true
-1 225.18215 -0.98714751 20.2422 true
"""[1:-1]
special_objids.extend(find_risa_objs(risa_145729, [h for h in hosts_to_target if h.nsaid==145729][0])
In [752]:
risa_obrother="""
1 335.97281 -3.4295662 17.7332 true
1 335.91314 -3.5404510 19.7043 true
-1 335.77781 -3.4459616 20.1615 true
-1 335.81490 -3.6025596 20.5123 true
1 336.04145 -3.2204987 20.5081 true
-1 336.09493 -3.4649021 19.7341 true
1 335.99401 -3.7007769 20.5931 true
1 336.12273 -3.5101925 20.4868 true
1 335.72556 -3.1674595 20.6372 true
1 335.84376 -3.0261104 20.7444 true
-1 336.23396 -3.1875586 20.7117 true
1 335.55249 -3.6052065 20.0624 true
1 335.65592 -3.6558837 20.5213 true
-1 335.57983 -3.6963397 19.3788 true
-1 336.30042 -3.4636766 19.8654 true
"""[1:-1]
special_objids.extend(find_risa_objs(risa_obrother, hsd['OBrother']))
In [773]:
risa_catch22="""
-1 348.62149 4.5071708 17.7932 true
-1 348.73347 4.5865011 19.4766 true
-1 348.67493 4.6123235 20.3472 true
-1 348.72361 4.4495699 20.6729 true
-1 348.72881 4.4323062 19.4804 true
-1 348.55899 4.5900220 20.2576 true
-1 348.64485 4.4040044 17.6392 true
-1 348.59640 4.3492465 20.6181 true
-1 348.68132 4.3095517 20.6868 true
-1 348.68817 4.3020869 20.7035 true
-1 348.89822 4.4892740 18.9281 true
-1 348.43132 4.7245873 19.3515 true
-1 348.51966 4.7464873 19.3880 true
-1 348.39920 4.5666321 18.2252 true
-1 348.99115 4.5918658 20.7471 true
-1 348.31622 4.3290159 19.6409 true
-1 348.87290 4.8064919 18.9324 true
-1 348.63961 4.2011104 20.5785 true
-1 348.98746 4.3261837 18.3678 true
-1 348.40085 4.8681252 17.4557 true
-1 348.64976 4.8775596 18.4048 true
-1 348.82739 4.9078081 19.3681 true
-1 348.68761 4.9226928 17.8673 true
-1 348.85846 4.8346772 19.6157 true
"""[1:-1]
special_objids.extend(find_risa_objs(risa_catch22, hsd['Catch22']))
In [ ]:
risa_narnia="""
"""[1:-1]
special_objids.extend(find_risa_objs(risa_narnia, hsd['Narnia']))
In [774]:
for soid in special_objids:
for h in hosts_to_target:
cat = h.get_sdss_catalog()
if soid in cat['OBJID']:
print('Found oid', soid, 'in', h.name, 'so setting to pri=9')
pris = cat['aat_pris']
if np.any(pris[np.in1d(cat['OBJID'], soid)]<1):
print('WARNING: ', np.sum(pris[np.in1d(cat['OBJID'], soid)]<1), 'special objects in',
h.name, 'are pri<0... skipping')
cat['aat_pris'][np.in1d(cat['OBJID'], soid)&(pris>0)] = 9
break
else:
print('Could not find oid', soid, 'anywhere')
For some hosts we already have sky positions from the last run, so copy those over
In [597]:
# commented because it should only be run once
!ls aat_targets_jun2015/*_sky.dat
#!cp aat_targets_jun2015/*_sky.dat aat_targets_jul2016/
For the remainder, generate and visually inspect them one at a time. Edit the file to remove any that are not good sky positions
In [599]:
#Identify sky regions for each host and write out to separate files -
from os.path import exists
for h in hosts_to_target:
outfn = 'aat_targets_jul2016/' + h.name.replace(' ','_') + '_sky.dat'
if exists(outfn):
print(outfn, 'exists, not overwriting')
else:
try:
h.get_usnob_catalog()
usnocat = None
except IOError:
# this is currently hanging as the usno server is down...
#print('Downloading USNO B1 catalog for', h.name)
#h.usnob_environs_query(dl=True)
#usnocat = None
usnocat = False
aat.select_sky_positions(h, nsky=100, outfn=outfn, rad=1*u.deg, usnocat=usnocat)
aat.imagelist_fld_targets(outfn, ttype='sky', n=np.inf)
!subl $outfn
raw_input('Wrote ' + outfn + '. Press enter to continue onto the next one once '
'you remove bad entries from this file.')
In [776]:
for h in hosts_to_target:
cat = h.get_sdss_catalog()
pris = cat['aat_pris']
guides = aat.select_guide_stars_sdss(cat)
calibs = aat.select_flux_stars(cat, onlyoutside=300*u.kpc)
skyradec = 'aat_targets_jul2016/{0}_sky.dat'.format(h.name)
aat.produce_master_fld(h, datetime.date(2016, 7, 28), cat[pris>0], pris=pris[pris>0].astype(int),
guidestars=guides, fluxstars=calibs,skyradec=skyradec,
outfn='aat_targets_jul2016/{}_master.fld'.format(h.name),
randomizeorder=True, inclhost=False)
We sub-sample from the master lists generated above for individual configurations.
In [12]:
def do_subsampling(h, finum, n_in_pri, **kwargs):
assert h in hosts_to_target
n_in_pri = dict(n_in_pri) #copy just in case
for i in range(9):
if i+1 not in n_in_pri:
n_in_pri[i+1] = np.inf
fnbase = 'aat_targets_jul2016/' + h.name
fnmaster = fnbase + '_master.fld'
fnconfig = kwargs.pop('fnconfig', fnbase + '_{0}.fld'.format(finum))
print('Writing', fnconfig, 'from master', fnmaster)
kwargs.setdefault('nflux', 5)
kwargs.setdefault('nguides', 30)
kwargs.setdefault('fieldname', str(finum))
if 'listorem' not in kwargs:
listorem = []
for i in range(1, finum):
globpat = fnbase + '_' + str(i) + '*.lis'
listorem.extend(glob(globpat))
assertmsg = 'Only got ' + str(len(listorem)) +' .lis file(s):'+str(listorem)+', expected ' + str(finum-1)
assert len(listorem) == (finum-1), assertmsg
kwargs['listorem'] = listorem
aat.subsample_from_master_fld(fnmaster, fnconfig, n_in_pri, **kwargs)
return fnconfig
This function parses a dump of all the image headers from 'fitsheader ccd_?/.fits'
In [90]:
def make_logtab(logs, skip_res=[]):
"""
Generates a table with the AAT observing logs.
``logs`` can be a filename or list of lines from fitsheader
"""
logtab = Table()
logtab['num'] = [1]
logtab['ccd'] = [2]
logtab['UTDATE'] = ['2015:06:19']
logtab['UTSTART'] = ['08:56:19']
logtab['OBJECT'] = ['a'*100]
logtab['TOTALEXP'] = ['a'*20]
logtab['RUNCMD'] = ['a'*20]
logtab['GRATID'] = ['a'*20]
logtab['SOURCE'] = ['plate 0 ']
logtab['MEANRA'] = [1.1]
logtab['MEANDEC'] = [1.1]
logtab['CFG_FILE'] = ['Odyssey_1p0xxxxxxxxxx.sds']
logtab = logtab[1:]
def add_logcol(accumulated_lines):
l = accumulated_lines[0]
hdunum = int(l.split()[2])
hdupath = l.split()[4][:-1]
if hdunum != 0:
return
for rex in skip_res:
if re.match(rex, hdupath):
print('Skipping', hdupath, 'because it has', rex)
return
items = {'num': int(hdupath.split('/')[-1].split('.')[0][5:]),
'ccd': int(hdupath.split('/')[-2][4:])}
for l in accumulated_lines[1:]:
if '=' in l:
nm, data = l.split('=')
nm = nm.strip()
data = data.split('/')[:2]
if len(data)> 1:
comment = data[1]
data = data[0].replace("'", '').strip()
if nm in logtab.colnames:
items[nm] = data
logtab.add_row(items)
accumulated_lines = None
if isinstance(logs, basestring):
with open(logs) as f:
loglines = list(f)
else:
loglines = logs
for l in loglines:
if l.startswith('# HDU'):
if accumulated_lines is not None:
add_logcol(accumulated_lines)
accumulated_lines = [l]
elif l.strip() == '':
continue
elif accumulated_lines is not None:
accumulated_lines.append(l)
return logtab
Out[90]:
In [14]:
sshtarget = 'visitor3@aatlxa'
In [267]:
fnconfig = do_subsampling(hsd['Dune'], 1, {1:100, 2:100, 3:300})
!scp $fnconfig "$sshtarget:configure/"
In [268]:
fnconfig = do_subsampling(hsd['Odyssey'], 1, {1:200, 2:200, 3:300})
!scp $fnconfig "$sshtarget:configure/"
In [269]:
fnconfig = do_subsampling(hsd['OBrother'], 1, {1:100, 2:100, 3:300})
!scp $fnconfig "$sshtarget:configure/"
In [270]:
fnconfig = do_subsampling(hsd['Catch22'], 1, {1:100, 2:50, 3:100})
!scp $fnconfig "$sshtarget:configure/"
In [509]:
fnconfig = do_subsampling(hsd['AnaK'], 1, {1:100, 2:100, 3:300})
!scp $fnconfig "$sshtarget:configure/"
In [547]:
# use this to get the log data from the remote machine
rem_fitsheader_loc = '/home/visitor3/miniconda2/bin/fitsheader'
datadir = '/data_lxy/aatobs/OptDet_data/160728'
fitsheader_output = !ssh $sshtarget "$rem_fitsheader_loc $datadir/ccd_?/*.fits"
In [783]:
# use this to use the local file to make the log
fitsheader_output = !fitsheader aat_data/28jul2016/ccd_?/*.fits
In [784]:
logtab = make_logtab(fitsheader_output, skip_res=['.*temp.*'])
logtab[logtab['ccd']==1].show_in_notebook(display_length=20)
Out[784]:
In [785]:
comments_jul28 = """
* At start of night weather clear, but high humidity and possible fog
* The Dune_1 observations have the blue side centered 20 angstroms further to the red than the remaining
observations. The initial setup provides a bit more overlap but after inspecting a bit we concluded that it
would be better to tryto make sure 3727 falls into the image at the blue side
* start of Dune_1 : seeing ~1.5-2"
* start of Odyssey_1 : seeing ~2-3"
* start of OBrother_1: seeing ~4-5"
* near end of OBrother_1: seeing ~2.5". Sudden improvement in weather, humidity fell/fog disappeared,
* start of Catch22_1: seeing ~3"
* start of AnaK_1: seeing ~1.9"
"""[1:-1]
display.Markdown(comments_jul28)
Out[785]:
In [787]:
logfn = 'aat_data/28jul2016/log_jul28'
logtab.write(logfn,format='ascii')
with open(logfn, 'a') as f:
f.write('\nComments:\n')
f.write(comments_jul28)
Note: master catalogs changed due to Risa's custom targets, so some added pri=9 objects appear even in repeats
In [587]:
fnconfig = do_subsampling(hsd['Dune'], 2, {1:100, 2:100, 3:300})
!scp $fnconfig "$sshtarget:configure/"
In [588]:
fnconfig = do_subsampling(hsd['Gilgamesh'], 1, {1:100, 2:100, 3:300})
!scp $fnconfig "$sshtarget:configure/"
In [770]:
fnconfig = do_subsampling(hsd['OBrother'], 2, {1:100, 2:100, 3:300})
!scp $fnconfig "$sshtarget:configure/"
In [778]:
fnconfig = do_subsampling(hsd['Catch22'], 2, {1:50, 2:50, 3:200})
!scp $fnconfig "$sshtarget:configure/"
In [24]:
fnconfig = do_subsampling(hsd['Narnia'], 1, {1:100, 2:100, 3:300})
!scp $fnconfig "$sshtarget:configure/"
In [145]:
# use this to use the local file to make the log
fitsheader_output = !fitsheader aat_data/29jul2016/ccd_?/*.fits
In [146]:
logtab = make_logtab(fitsheader_output, skip_res=['.*temp.*'])
logtab[logtab['ccd']==1].show_in_notebook(display_length=20)
Out[146]:
In [147]:
comments_jul29 = """
* start of Dune_2 : seeing ~1.3"
* start of Gilgamesh_1 : seeing ~1.6"
* start of Odyssey_1a : seeing ~1.4"
* start of OBrother_2 : seeing ~1.7"
* temperature rose dramatically near end of OBrother_2 - might lead to variable seeing
* start of Catch22_2: seeing ~2.3"
* start of Narnia_1: seeing ~1.5"
"""[1:-1]
display.Markdown(comments_jul29)
Out[147]:
In [148]:
logfn = 'aat_data/29jul2016/log_jul29'
logtab.write(logfn,format='ascii')
with open(logfn, 'a') as f:
f.write('\nComments:\n')
f.write(comments_jul29)
These nights are by the team that's doing the time-trade. They will be primarily second-half, but with a little time in the first half
In [56]:
hs = [h for h in hosts_to_target if h.nsaid==145729]
assert len(hs) == 1
fnconfig = do_subsampling(hs[0], 1, {1:25, 2:25, 3:50, 4:100})
!scp $fnconfig "$sshtarget:configure/"
NSA 145729 is being treated a bit oddly because there might be two back-to-back-ish observations. So we're making one config in advance for both the p0 and p1 configurations, and making fld's for config 2 assuming either of those.
In [130]:
hs = [h for h in hosts_to_target if h.nsaid==145729]
assert len(hs) == 1
fnconfig = do_subsampling(hs[0], 2, {1:25, 2:25, 3:50, 4:100},
listorem=['aat_targets_jul2016/NSA145729_1_p0.lis'],
fnconfig='aat_targets_jul2016/NSA145729_2_after1p0.fld')
!scp $fnconfig "$sshtarget:configure/"
In [134]:
hs = [h for h in hosts_to_target if h.nsaid==145729]
assert len(hs) == 1
fnconfig = do_subsampling(hs[0], 2, {1:25, 2:25, 3:50, 4:100},
listorem=['aat_targets_jul2016/NSA145729_1_p1.lis'],
fnconfig='aat_targets_jul2016/NSA145729_2_after1p1.fld')
!scp $fnconfig "$sshtarget:configure/"
In [57]:
fnconfig = do_subsampling(hsd['Catch22'], 3, {1:100, 2:100, 3:300)
!scp $fnconfig "$sshtarget:configure/"
In [58]:
fnconfig = do_subsampling(hsd['Narnia'], 2, {1:100, 2:100, 3:300})
!scp $fnconfig "$sshtarget:configure/"
In [55]:
hs = [h for h in hosts_to_target if h.nsaid==21709]
assert len(hs) == 1
fnconfig = do_subsampling(hs[0], 1, {1:25, 2:25, 3:50, 4:100})
!scp $fnconfig "$sshtarget:configure/"
In [155]:
# fitsheader run on the output downloaded to another computer
with open('/Users/erik/tmp/fitsheader_jul30') as f:
fitsheader_output = list(f)
In [158]:
logtab = make_logtab(fitsheader_output, skip_res=['.*temp.*'])
logtab[logtab['ccd']==1].show_in_notebook(display_length=20)
logfn = 'aat_data/30jul2016/log_jul30'
logtab.write(logfn,format='ascii')
Note that this assumes the .lis files were generated after the configuration was made. If need be these can be re-created from the .sds files by starting up configure and exporting the allocated fibers list.
In [132]:
!scp "$sshtarget:configure/*.sds" aat_targets_jul2016/
In [133]:
# *_* because otherwise the collision matrix list comes through too
!scp "$sshtarget:configure/*_*.lis" aat_targets_jul2016/
In [54]:
listab, lisscs, lisheader = aat.load_lis_file('aat_targets_jul2016/Narnia_1_p1.lis')
guidemsk = listab['codes']=='F'
names = ['{0[ids]}_f#={0[fibnums]}'.format(row) for row in listab[guidemsk]]
print(targeting.sampled_imagelist(lisscs[guidemsk], None, names=names))
In [61]:
import astroplan
In [8]:
#if online
ufo = urllib2.urlopen('https://docs.google.com/spreadsheet/ccc?key=1b3k2eyFjHFDtmHce1xi6JKuj3ATOWYduTBFftx5oPp8&output=csv')
hosttab = QTable.read(ufo.read(), format='csv')
ufo.close()
In [9]:
#if offline
#hosttab = Table.read('SAGADropbox/hosts/host_catalog_flag0.csv')
In [10]:
hostscs = SkyCoord(u.Quantity(hosttab['RA'], u.deg),
u.Quantity(hosttab['Dec'], u.deg),
distance=u.Quantity(hosttab['distance'], u.Mpc))
In [11]:
allspec = Table.read('/Users/erik/Dropbox/SAGA/data/allspectaken_v5.fits.gz')
In [12]:
sagaobsed = allspec[~((allspec['TELNAME']=='NSA')|
(allspec['TELNAME']=='SDSS')|
(allspec['TELNAME']=='GAMA'))]
sagaobsed_nsaids = np.unique(sagaobsed['HOST_NSAID'])
In [13]:
#UTC time from 8:35-19:35 is AAT 18 deg window
nighttimes = Time('2016-7-28 8:45:00') + np.arange(12)*u.hour
aao = EarthLocation(lon='149:3:57.9', lat='-31:16:37.3')
aao_frame = AltAz(obstime=nighttimes, location=aao)
In [14]:
sunaao = get_sun(nighttimes).transform_to(aao_frame)
np.max(sunaao.alt.value)
Out[14]:
In [15]:
seczs = []
for sc in hostscs:
az = sc.transform_to(aao_frame)
seczs.append(az.secz)
seczs = np.array(seczs)
hrsvis = np.sum((1<=seczs)&(seczs<1.75),axis=1)
visenough = hrsvis>2
In [16]:
aaoobs = astroplan.Observer(aao, 'Australia/NSW')
midnight = aaoobs.midnight(Time('2016-7-28'))
In [121]:
up_times = {}
hoststoshow = hosttab[visenough]
hosts_to_target_nsaids = []
for h in hosts_to_target:
hosts_to_target_nsaids.append(h.nsaid)
if h.nsaid not in hoststoshow['NSAID']:
print('adding', h)
hoststoshow.add_row(None)
hoststoshow['NSAID'][-1] = h.nsaid
hoststoshow['RA'][-1] = h.ra
hoststoshow['Dec'][-1] = h.dec
with open('aat_targets_jul2016/aattargs_iobserve.dat', 'w') as f:
for host in hoststoshow:
already_obs = host['NSAID'] in sagaobsed_nsaids
name = 'NSA'+str(host['NSAID'])
for nm, val in hsd.items():
if val.nsaid == host['NSAID']:
name = nm
if nm.startswith('NSA'):
name = name+'_obsed'
break
f.write(name.replace(' ','_'))
if already_obs:
f.write('-observed')
f.write(' ')
f.write(str(host['RA']) + ' ')
f.write(str(host['Dec']) + '\n')
targ = astroplan.FixedTarget(SkyCoord(host['RA'], host['Dec'], unit=u.deg), name)
tpl = (name, host['NSAID'], host['RA'], host['Dec'])
transit = aaoobs.target_meridian_transit_time(midnight, targ)
up_times[transit.jd] = tpl
timestoplot = transit + np.linspace(-6, 6, 100)*u.hour
taa = aaoobs.altaz(timestoplot, targ)
msk = taa.secz >=1
color = 'g' if already_obs else 'k'
color = 'r' if host['NSAID'] in hosts_to_target_nsaids else color
plt.plot(timestoplot.plot_date[msk], taa.secz[msk], c=color)
plt.text(transit.plot_date, aaoobs.altaz(transit, targ).secz, name, ha='center', color=color)
t0 = aaoobs.sun_rise_time(midnight, 'previous')
t1 = aaoobs.sun_set_time(midnight, 'previous')
t2 = aaoobs.twilight_evening_civil(midnight, 'previous')
t3 = aaoobs.twilight_evening_nautical(midnight, 'previous')
t4 = aaoobs.twilight_evening_astronomical(midnight, 'previous')
t5 = aaoobs.twilight_morning_astronomical(midnight, 'next')
t6 = aaoobs.twilight_morning_nautical(midnight, 'next')
t7 = aaoobs.twilight_morning_civil(midnight, 'next')
t8 = aaoobs.sun_rise_time(midnight, 'next')
t9 = aaoobs.sun_set_time(midnight, 'next')
plt.fill_between([t0.plot_date,t1.plot_date],1,3, lw=0, facecolor='y', alpha=.9)
plt.fill_between([t1.plot_date,t2.plot_date],1,3, lw=0, facecolor='y', alpha=.75)
plt.fill_between([t2.plot_date,t3.plot_date],1,3, lw=0, facecolor='y', alpha=.5)
plt.fill_between([t3.plot_date,t4.plot_date],1,3, lw=0, facecolor='y', alpha=.25)
plt.fill_between([t5.plot_date,t6.plot_date],1,3, lw=0, facecolor='y', alpha=.25)
plt.fill_between([t6.plot_date,t7.plot_date],1,3, lw=0, facecolor='y', alpha=.5)
plt.fill_between([t7.plot_date,t8.plot_date],1,3, lw=0, facecolor='y', alpha=.75)
plt.fill_between([t8.plot_date,t9.plot_date],1,3, lw=0, facecolor='y', alpha=.9)
plt.axvline(midnight.plot_date, ls=':', c='k')
plt.gca().xaxis_date(aaoobs.timezone)
plt.xlim(t1.plot_date-.05, t8.plot_date+.05)
plt.axhline(1/np.cos(65*u.deg), c='k', ls='--') #~AAT limit
plt.ylim(2.5,1)
plt.tight_layout()
In [18]:
#NSA urls
for jd in sorted(up_times):
name, nsaid, ra, dec = up_times[jd]
print(name, 'http://www.nsatlas.org/getAtlas.html?search=nsaid&nsaID={}&submit_form=Submit'.format(nsaid))
In [19]:
# DECALS URLs
for jd in sorted(up_times):
name, nsaid, ra, dec = up_times[jd]
print(name, 'http://legacysurvey.org/viewer?ra={}&dec={}&zoom=8'.format(ra, dec))
Notes:
In [440]:
sagaobsed[sagaobsed['HOST_NSAID']==150307].show_in_notebook()
Out[440]:
Note: NSA150307 was called "Iliad" when it was observed on WIYN
In [227]:
for h in hosts_to_target:
fnbase = 'aat_targets_jul2016/test_runs/' + h.name
finum = 1
fnmaster = fnbase.replace('test_runs/', '') + '_master.fld'
fnconfig = fnbase + '_test_{0}.fld'.format(finum)
print('Writing', fnconfig, 'from master', fnmaster)
listorem = [fnbase + '_test_' + str(i) + '.lis' for i in range(1, finum)]
aat.subsample_from_master_fld(fnmaster, fnconfig,
{1:30, 2:30, 3:30, 4:np.inf, 5:np.inf, 6:np.inf, 7:np.inf, 8:np.inf, 9:np.inf}, #8 reserved for flux
nflux=5, nguides=30,
fieldname=str(finum), listorem=listorem)
print('')
In [175]:
for h in hosts_to_target:
fnbase = 'aat_targets_jul2016/test_runs/' + h.name
finum = 2
fnmaster = fnbase.replace('test_runs/', '') + '_master.fld'
fnconfig = fnbase + '_test_{0}.fld'.format(finum)
print('Writing', fnconfig, 'from master', fnmaster)
listorem = [fnbase + '_test_' + str(i) + '.lis' for i in range(1, finum)]
aat.subsample_from_master_fld(fnmaster, fnconfig,
{1:30, 2:30, 3:30, 4:np.inf, 5:np.inf, 6:np.inf, 7:np.inf, 8:np.inf, 9:np.inf}, #8 reserved for flux
nflux=5, nguides=30,
fieldname=str(finum), listorem=listorem)
print('')
In [188]:
for h in hosts_to_target:
fnbase = 'aat_targets_jul2016/test_runs/' + h.name
finum = 3
fnmaster = fnbase.replace('test_runs/', '') + '_master.fld'
fnconfig = fnbase + '_test_{0}.fld'.format(finum)
print('Writing', fnconfig, 'from master', fnmaster)
listorem = [fnbase + '_test_' + str(i) + '.lis' for i in range(1, finum)]
aat.subsample_from_master_fld(fnmaster, fnconfig,
{1:30, 2:30, 3:30, 4:np.inf, 5:np.inf, 6:np.inf, 7:np.inf, 8:np.inf, 9:np.inf}, #8 reserved for flux
nflux=5, nguides=30,
fieldname=str(finum), listorem=listorem)
print('')
Basically there after 2 observations of Narnia and OBrother. AnaK and Dune 1 or maybe 2, Gilg/Ody 1.
Guess: Catch 22 requires 2 or 3
NSA 145729 and maybe 165082?
In [231]:
h = hsd['Catch22']
fnbase = 'aat_targets_jul2016/test_runs/' + h.name
finum = 2
fnmaster = fnbase.replace('test_runs/', '') + '_master.fld'
fnconfig = fnbase + '_test_{0}.fld'.format(finum)
print('Writing', fnconfig, 'from master', fnmaster)
listorem = [fnbase + '_test_' + str(i) + '.lis' for i in range(1, finum)]
aat.subsample_from_master_fld(fnmaster, fnconfig,
{1:30, 2:30, 3:30, 4:np.inf, 5:np.inf, 6:np.inf, 7:np.inf, 8:np.inf, 9:np.inf}, #8 reserved for flux
nflux=5, nguides=30,
fieldname=str(finum), listorem=listorem)
In [232]:
h = hsd['Catch22']
fnbase = 'aat_targets_jul2016/test_runs/' + h.name
finum = 3
fnmaster = fnbase.replace('test_runs/', '') + '_master.fld'
fnconfig = fnbase + '_test_{0}.fld'.format(finum)
print('Writing', fnconfig, 'from master', fnmaster)
listorem = [fnbase + '_test_' + str(i) + '.lis' for i in range(1, finum)]
aat.subsample_from_master_fld(fnmaster, fnconfig,
{1:30, 2:30, 3:30, 4:np.inf, 5:np.inf, 6:np.inf, 7:np.inf, 8:np.inf, 9:np.inf}, #8 reserved for flux
nflux=5, nguides=30,
fieldname=str(finum), listorem=listorem)
In [235]:
h = hsd['Catch22']
fnbase = 'aat_targets_jul2016/test_runs/' + h.name
finum = 4
fnmaster = fnbase.replace('test_runs/', '') + '_master.fld'
fnconfig = fnbase + '_test_{0}.fld'.format(finum)
print('Writing', fnconfig, 'from master', fnmaster)
listorem = [fnbase + '_test_' + str(i) + '.lis' for i in range(1, finum)]
aat.subsample_from_master_fld(fnmaster, fnconfig,
{1:30, 2:30, 3:30, 4:np.inf, 5:np.inf, 6:np.inf, 7:np.inf, 8:np.inf, 9:np.inf}, #8 reserved for flux
nflux=5, nguides=30,
fieldname=str(finum), listorem=listorem)
In [91]:
logtab2015 = make_logtab('aat_targets_jul2016/alljun_2015_aaomega_headers')
In [92]:
logtab2015[(logtab2015['ccd']==1)].show_in_notebook(display_length=20)
Out[92]: