In [1]:
# This changes the current directory to the base saga directory - make sure to run this first!
# This is necessary to be able to import the py files and use the right directories,
# while keeping all the notebooks in their own directory.
import os
import sys

if 'saga_base_dir' not in locals():
    saga_base_dir = os.path.abspath('..')

if saga_base_dir not in sys.path:
    os.chdir(saga_base_dir)

In [2]:
from __future__ import print_function

import hosts
import targeting
import mmthecto
import numpy as np

from astropy import units as u
from astropy.coordinates import *
from astropy import table
from astropy.table import Table
from astropy.io import fits

In [3]:
%matplotlib inline
from matplotlib import pyplot as plt
from matplotlib import rcParams

rcParams['image.interpolation'] = 'none'
rcParams['figure.figsize'] = (16, 10)

In [4]:
hostlst = hosts.get_saga_hosts_from_google() #'named' hosts


Using cached version of google hosts list from file "hosts_dl.pkl2"

In [5]:
candidates = """
NSAID    RA   DEC Name
   150887      23.2455      4.53406  N/A
    150578.      23.0824      12.3229  N/A
    149781.      22.3942     -3.43167  OBrother
    150307.      22.9283     -5.49535  N/A
    61945.0      23.6087     0.297265  AnaK
    149977.      22.6248      10.5315  N/A
    153017.      23.9904      20.7498  N/A
    126115.    0.0663717      20.7524  N/A
    127226.     0.579656     -8.39644  N/A
    129387.      1.40109      12.9175  N/A
    129237.      1.36327      17.5922  N/A
    132339.      2.63655     -1.31876 Narnia
    130133.      1.65181     -7.51266  N/A
    131531.      2.25767      6.00248  N/A
    130625.      1.88703      4.19576  N/A
"""
candidates = Table.read(candidates,format='ascii')

hostdct = dict([(h.name, h) for h in hostlst])
candhosts = [hostdct[name] if name in hostdct else hosts.NSAHost(int(nsaid)) 
         for nsaid, name in zip(candidates['NSAID'], candidates['Name'])]
candhosts


Out[5]:
[<hosts.NSAHost object w/ name 'NSA150887'>,
 <hosts.NSAHost object w/ name 'NSA150578'>,
 <hosts.NSAHost object w/ name 'OBrother' AKA: ['NSA149781']>,
 <hosts.NSAHost object w/ name 'NSA150307'>,
 <hosts.NSAHost object w/ name 'AnaK' AKA: ['NGC7716', 'NSA61945']>,
 <hosts.NSAHost object w/ name 'NSA149977'>,
 <hosts.NSAHost object w/ name 'NSA153017'>,
 <hosts.NSAHost object w/ name 'NSA126115'>,
 <hosts.NSAHost object w/ name 'NSA127226'>,
 <hosts.NSAHost object w/ name 'NSA129387'>,
 <hosts.NSAHost object w/ name 'NSA129237'>,
 <hosts.NSAHost object w/ name 'Narnia' AKA: ['NGC1015', 'NSA132339']>,
 <hosts.NSAHost object w/ name 'NSA130133'>,
 <hosts.NSAHost object w/ name 'NSA131531'>,
 <hosts.NSAHost object w/ name 'NSA130625'>]

In [6]:
# now set to the latest base catalogs
for h in candhosts:
    h.fnsdss = 'catalogs/base_sql_nsa{0}.fits.gz'.format(h.nsaid)
    h._cached_sdss = None

In [7]:
# actually make sure they're loaded here
for h in candhosts:
    h.get_sdss_catalog()
h.get_sdss_catalog() # just to see


Out[7]:
<Table masked=False length=68227>
objIDradecflagsspecObjIDugrizu_errg_errr_erri_errz_errAuAgArAiAzfibermag_rfiber2mag_rexpRad_rsb_exp_rpetroR50_rpetroR90_rpetroMag_rsb_petro_rJJerrHHerrKKerrw1w1errw2w2errspec_zspec_z_errspec_z_warnphotozphotoz_errHOST_RAHOST_DECHOST_DISTHOST_VHOSTHOST_MKHOST_NSAIDHOST_FLAGHOST_SAGA_NAMERHOST_ARCMRHOST_KPCOBJ_NSAIDPCLASS_1SATSREMOVETELNAMEMASKNAMEZQUALITYSPEC_REPEATUBVRIrhostrhost_kpctypephot_sg
int64float64float64int64int64float32float32float32float32float32float32float32float32float32float32float32float32float32float32float32float32float32float32float64float32float32float32float64float32float32float32float32float32float32float32float32float32float32float32float32int32float32float32float64float64float64float64float64int64int64string384float64float64int64int64int64int64string32string384float64string384float32float32float32float32float32float64float64int16string48
123767862223968729328.29218937614.1654677952387097114444048024.886124.538422.964821.945621.31480.9661160.5658630.2382030.1351540.2807910.193710.142530.1033750.07838580.055576522.26122.97410.13222520.56680213240.6418641.3758422.985924.01858371229999.09999.09999.09999.09999.09999.017.94990.20725216.0030.175-1.0-1.0-10.5957910.08232228.30544.1957622.74691733.08.72613062501.9820026609613.114524378-1-1-1-1-1.025.444825.249923.633322.691921.30510.033019848276212.96201855523GALAXY
123767862223968753528.28085317844.17348908215281543964623616024.254923.137921.911421.785821.60710.9755350.2128210.09797780.1201670.3591270.1937230.1425390.1033810.0783910.055580222.295122.77680.21696420.58877813660.5905221.319522.047822.89947970379999.09999.09999.09999.09999.09999.019.49840.8485849999.09999.0-1.0-1.0-1-1.0-1.028.30544.1957622.74691733.08.72613062501.9857324142813.1392034243-1-1-1-1-1.024.151623.740822.428921.775221.31980.033070632014712.98195383036STAR
123767862223968755128.28923055074.1666156998281543964623104023.53823.673322.798922.260121.72350.6477140.3206250.2016270.1750920.3719330.1936680.1424990.1033520.0783690.055564622.950423.40560.0014911610.6619883934-9999.0-9999.027.000548.99569790599999.09999.09999.09999.09999.09999.017.52890.1373489999.09999.0-1.0-1.0-10.6378830.13766828.30544.1957622.74691733.08.72613062501.9984970286613.2236643743-1-1-1-1-1.023.73924.165923.163322.599521.6560.033291909191813.06881670293GALAXY
123767866143637581528.31787105284.22726178032281543964623616026.157222.157821.486820.018919.58651.650040.27290.2661160.1090580.298410.1960140.1442250.1046040.0793180.056237523.145223.51920.26470921.71037908093.992386.4714221.483926.48551619389999.09999.09999.09999.09999.09999.018.79340.4697929999.09999.0-1.0-1.0-10.95290.18447328.30544.1957622.74691733.08.72613062502.0320936016113.4459663078-1-1-1-1-1.024.202122.586821.762721.145219.4550.033880723416913.29995710213GALAXY
123767866143637582228.33792906844.18852269949281475245277440023.6423.18623.367121.556421.36180.9471160.2588170.5224180.1575880.5182120.1955760.1439030.104370.0791410.056111923.866524.15874.2508927.42103245640.8413041.1010723.562425.18263944879999.09999.09999.09999.09999.09999.017.65420.1575349999.09999.0-1.0-1.0-10.4657910.24836528.30544.1957622.74691733.08.72613062501.9943611526613.1962980939-1-1-1-1-1.022.668423.348323.272322.97321.08430.033272571525313.06122565683GALAXY
123767862223968729228.29176886824.1647939568535253360001296024.885422.432220.839620.158319.7941.140230.1365690.04787440.03727070.09619610.1936630.1424960.103350.07836690.055563121.331621.87340.4972821.36716999560.7398491.428920.998122.33924659979999.09999.09999.09999.09999.09999.016.53480.057011616.0030.175-1.0-1.0-10.3458570.04428628.30544.1957622.74691733.08.72613062502.0291259719913.4263300847-1-1-1-1-1.024.449523.149721.516420.618419.62070.033805136554113.27028530263GALAXY
123767862223968726328.27859066244.1725615376868987912192022.43722.274921.903121.544121.05370.3017610.1114710.1077530.1097590.2585090.1935980.1424480.1033150.07834070.055544522.402922.88650.28426621.26447399930.7152762.1280521.944423.21224125939999.09999.09999.09999.09999.09999.017.90260.1976439999.09999.0-1.0-1.0-10.5180330.23051728.30544.1957622.74691733.08.72613062502.1239167752214.0535422304-1-1-1-1-1.022.071722.610322.048821.731220.95780.035373218062313.88583936073GALAXY
123767862223968735528.3176155984.1603599999768987912448024.652822.420221.032920.118119.63140.9886330.11230.04642210.02986790.06804620.1945320.1431350.1038130.07871840.055812321.414821.88940.13439618.67027149280.6027391.2909821.147222.04332746389999.09999.09999.09999.09999.09999.017.55030.1441169999.09999.0-1.0-1.0-1-1.0-1.028.30544.1957622.74691733.08.72613062502.2462625390514.8630801279-1-1-1-1-1.024.149623.073421.620420.775919.53320.037448726017214.70058485566STAR
123767862223968754028.28544412874.16744059371281709320864000023.637623.19922.604121.956823.0580.7658820.2467270.2052090.1585460.6630690.1935820.1424360.1033060.0783340.055539823.07723.48390.512723.14884147140.46970.62672523.341423.6959243029999.09999.09999.09999.09999.09999.018.75020.4266669999.09999.0-1.0-1.0-1-1.0-1.028.30544.1957622.74691733.08.72613062502.0768094950513.741842602-1-1-1-1-1.023.327623.604222.846922.38821.98490.034594103114113.57999596836STAR
123767866143637579028.27436556534.17756344018281543964623616023.045222.920422.209821.03220.53330.8699960.3035030.2848310.1620630.3930950.1936990.1425220.1033690.07838160.055573523.165223.57710.45996222.92375309890.4501750.72617323.303423.56579012919999.09999.09999.09999.09999.09999.017.79190.1781079999.09999.0-1.0-1.0-10.7720220.09980828.30544.1957622.74691733.08.72613062502.1542383441214.2541741001-1-1-1-1-1.022.983423.361822.502921.912620.44250.035874985486314.08280933493GALAXY
..................................................................................................................................................................................................................
123767866251005182428.19789187115.1472318957268987912960022.83522.867922.292321.34821.01730.434580.185450.1897910.1362240.3097030.2328130.1713010.1242420.0942090.066795322.817523.27010.38463822.26348495330.6959141.9337222.490523.69868288769999.09999.09999.09999.09999.09999.017.51790.14659217.0010.42-1.0-1.0-10.7148250.15350328.30544.1957622.74691733.08.726130625057.4496442549380.115399837-1-1-1-1-1.022.735123.267122.526722.030820.82330.957482542474375.8620081473GALAXY
123767866251011728828.3660528395.18434052911475064453693712024.067321.571420.457420.318620.00591.873450.103330.06680430.09732490.2466760.2195710.1615590.1171760.08885080.062996321.594122.26882.4054324.35880355981.853625.2963820.410323.74582340179999.09999.09999.09999.09999.09999.017.1640.12208617.00.412-1.0-1.0-10.3537050.07156628.30544.1957622.74691733.08.726130625059.4257682552393.189137686-1-1-1-1-1.023.207322.13920.92620.319219.80090.990429148022388.7952750983GALAXY
123767866251005190128.23665285615.1710167367168987912192023.413121.80720.545120.115919.72680.6678750.07235580.04060740.0448980.09753060.2307160.1697580.1231230.09336040.066193720.9521.45890.37688120.43399375860.8519061.9869220.54922.19642340899999.09999.09999.09999.09999.09999.017.62850.16016816.9979999.0-1.0-1.0-1-1.0-1.028.30544.1957622.74691733.08.726130625058.6598300601388.121821481-1-1-1-1-1.023.10522.42121.07820.362419.56870.977658355025383.7820704826STAR
123767866251005176028.17903969485.10886184428158546838815488024.255721.812720.902720.383119.88711.572210.1032210.07750170.08189270.171120.2350480.1729460.1254350.09511360.067436721.660322.23230.93200722.86316457051.131292.3475321.051923.31522362919999.09999.09999.09999.09999.09999.017.11740.10316116.5329999.0-1.0-1.0-10.0777210.03472628.30544.1957622.74691733.08.726130625055.3054351766365.929494562-1-1-1-1-1.023.249222.316621.282520.706219.79460.921741950142361.8319551583GALAXY
123767866251005222828.19645911325.17059183035281543964623104022.856123.835923.091321.243821.3090.5824930.5428780.4989740.1641840.5400720.2343340.1724210.1250540.09482470.067231923.723824.10212.1204926.29226734991.47822.4647722.925925.76997248159999.09999.09999.09999.09999.09999.017.15150.100916.8869999.0-1.0-1.0-10.6481540.13671628.30544.1957622.74691733.08.726130625058.8520693103389.393645328-1-1-1-1-1.023.353224.28823.399222.691720.8720.980856108509385.037356083GALAXY
123767866251005136428.19721376785.18726645925105622137864464023.892324.876622.449220.730619.87290.8216750.6153490.194680.07011220.1000440.234650.1726530.1252220.09495230.067322322.941523.27670.0001973075.920286484530.5619921.0276722.79523.53906275269999.09999.09999.09999.09999.09999.016.58810.059152216.2170.202-1.0-1.0-1-1.0-1.028.30544.1957622.74691733.08.726130625059.8415908781395.940140938-1-1-1-1-1.025.810125.855423.489122.069320.00250.997348321978391.5114129936STAR
123767866251011698228.33370697955.10776644478316730484326744025.716124.869822.834421.580921.69030.6510580.581950.265010.1357850.440910.2154690.158540.1149860.08719070.061819322.669723.11270.024432616.76971117340.420140.82183723.288923.40136787629999.09999.09999.09999.09999.09999.017.36090.1263417.2289999.0-1.0-1.0-1-1.0-1.028.30544.1957622.74691733.08.726130625054.7465971912362.232246272-1-1-1-1-1.026.424725.725823.703822.525621.22610.912443917985358.1819909216STAR
123767866250998598528.10090610155.10763501063175990882046040024.506222.590721.735121.408821.91261.193350.148640.1206640.1460450.6549290.246980.1817260.1318030.09994190.070860121.857422.51220.52879422.34702084910.8196561.4443821.881423.44503684899999.09999.09999.09999.09999.09999.09999.09999.016.9840.506-1.0-1.0-10.2942650.1801828.30544.1957622.74691733.08.726130625056.0642111907370.949508831-1-1-1-1-1.023.70723.077522.091321.568221.20620.93436838814366.7884928823GALAXY
123767866250998657928.12080073075.16413373448281543964623104025.098223.347721.84321.076820.53021.203130.3199620.1485960.1231310.2507250.2421530.1781740.1292270.09798860.069475122.353622.92750.58884622.6884253080.8446191.2757422.193923.8226364459999.09999.09999.09999.09999.09999.017.04410.092761916.6659999.0-1.0-1.0-10.4142990.0605628.30544.1957622.74691733.08.726130625059.1431466447391.319363668-1-1-1-1-1.024.925524.037722.481521.608720.46880.985689958387386.9348951393GALAXY
123767866251005209828.16527226365.15719765697281543964758272025.385223.242721.703721.472320.91451.020120.2891760.128630.1741220.3330220.2364770.1739980.1261980.09569190.067846722.27922.78860.78178123.25845326091.108173.2668121.619523.83796562699999.09999.09999.09999.09999.09999.017.47950.13953816.4410.249-1.0-1.0-10.384370.09836528.30544.1957622.74691733.08.726130625058.2924947433385.691586949-1-1-1-1-1.025.053123.943322.357121.551320.860.971523519244381.3738263693GALAXY

In [8]:
mlpred = Table.read('catalogs/SAGA.ALL.objid_rescaledrobs_pred.Oct28_SDSS_nopreclean.csv.fits.gz')
mlpred['objID'] = mlpred['OBJID'] # used for matching
mlpred


Out[8]:
<Table masked=False length=4950412>
OBJIDRADECDERED_RPROBABILITY_CLASS_1RESCALED_PROBABILITY_CLASS_1BEST_GUESS_CLASSobjID
int64float64float64float64float64float64int64int64
123764594182435644040.2877964113-0.73564706094916.75940.001571683161970.00034585919898101237645941824356440
123764594182435644240.2745834514-0.73301406347817.45140.0003724938366918.19307625291e-0501237645941824356442
123764594182435648140.2721052363-0.64251025320416.79110.001785145754540.00035879402480901237645941824356481
123764594182435648340.2712777717-0.64072529300120.98940.0002404548112865.13915737065e-0501237645941824356483
123764594182435648440.2783697699-0.64121250744521.28840.0002708648099025.63874521068e-0501237645941824356484
123764594182435648540.2826647047-0.64289294706122.49263.61806080935e-050.001237645941824356485
123764594182435649740.2846730128-0.81718021579914.86280.0003631305597288.19307625291e-0501237645941824356497
123764594182435650040.2881452154-0.80504550894814.38260.0004886749827388.85746681048e-0501237645941824356500
123764594182435650140.2854134086-0.80410047343220.24540.0007633014669820.00016232650348401237645941824356501
123764594182435650240.2864321267-0.80552497416121.61140.001126225175560.00024519359920301237645941824356502
........................
12376802972817173381.4432989565121.63946437822.71047.69491488335e-059.35409943407e-0601237680297281717338
12376802972817173411.4447360295721.625112294922.88140.0001025784870331.57161308307e-0501237680297281717341
12376802972817173521.4632599319321.634553337622.59710.0002013463964885.13915737065e-0501237680297281717352
12376802972817173531.4632834494321.645956298722.90920.0001263621986112.22024770719e-0501237680297281717353
12376802972817173591.4802799969121.635139295422.54780.0001183507286521.57161308307e-0501237680297281717359
12376802972817819001.4985220800521.631728324318.11390.0005650325313860.00011316201751101237680297281781900
12376802972817822141.4973308890721.623954531721.40330.000301649239245.77062812064e-0501237680297281782214
12376802972817822271.5043843701621.626287124120.44810.0002287081377545.13915737065e-0501237680297281782227
12376802972817822711.514854296721.626998974822.12510.0001093436167261.57161308307e-0501237680297281782271
12376802972817826831.5089864409321.625326628622.52630.0001041810475841.57161308307e-0501237680297281782683

In [9]:
p_column_for_ranking = 'PROBABILITY_CLASS_1' #this is *just* SDSS

Testing w/ NSA 150887


In [9]:
n150887 = candhosts[0]
hosts_to_target = [n150887]

In [12]:
joined_catalogs = {}
for h in hosts_to_target:
    #filter on manual remove lists *and* REMOVE!=-1
    cat = targeting.remove_targets_with_remlist(h.get_sdss_catalog(), h, 'catalogs/TargetRemoveAug28_2015.csv')
    
    pre_joined = table.join(cat, mlpred, join_type='left', keys='objID')
    
    tokeep = pre_joined['REMOVE']==-1
    notobserved = pre_joined['ZQUALITY']<3
    print('removed', np.sum(~tokeep),'for remove list and', np.sum(~notobserved),'b/c observed')
    joined_catalogs[h] = pre_joined[tokeep&notobserved]
    highp = pre_joined[p_column_for_ranking]>0.01
    print('n w/ ML pred>0.01 and observed:', np.sum(highp&~notobserved), 'not yet obs:', np.sum(highp&notobserved))


Removed 4 objects for NSA150887
removed 250 for remove list and 722 b/c observed
n w/ ML pred>0.01 and observed: 7 not yet obs: 506

No redmapper


In [13]:
# compute ranks, which includes filtering on mag
rankdct = {}
for h in hosts_to_target:
    cat = joined_catalogs[h]
    
    highprob = cat[p_column_for_ranking].filled(0) > 0.5
    medprob = (cat[p_column_for_ranking].filled(0) > 0.05) & ~highprob
    lowprob = (cat[p_column_for_ranking].filled(0) > 0.01) & ~highprob & ~medprob
    inside = cat['RHOST_KPC']<300
    
    rankdct[h] = ranks = np.ones(len(cat), dtype=int)*-1 #the -1 will be re-set at the end
    
    # rank 1 reserved for flux stars
    ranks[inside&highprob] = 2
    ranks[~inside&highprob] = 3
    ranks[inside&medprob] = 4
    ranks[~inside&medprob] = 5
    ranks[inside&lowprob] = 6
    ranks[~inside&lowprob] = 7
    
    validtarget = (cat['r'] < 21.5) & (cat['fibermag_r']<23) & (cat['phot_sg']=='GALAXY')
    other = ranks < 0
    colorcutmsk = targeting.colorcut_mask(cat,  targeting.bossanova_color_cuts)
    
    ranks[validtarget&other&colorcutmsk] = 8
    ranks[validtarget&other&~colorcutmsk] = 9
    
#informational
for h, ranks in rankdct.items():
    cat = joined_catalogs[h]
    
    hranks = dict(enumerate(np.bincount(ranks[ranks>-1])))
    del hranks[0]
    del hranks[1]
    hranks[-1] = np.sum(ranks==-1)
    
    print(h.name,'rank counts:', hranks)
    
    nother =  np.sum(ranks==8)
    nprob = np.sum((ranks<8)&(ranks>0))
    nother_obs = 270-np.sum((ranks<8)&(ranks>0)&(cat['RHOST_ARCM']<30))
    print('fraction not "other": {0:.1%}, nprob_obs={1}\n'.format(nprob/(nprob+nother), nother_obs))


NSA150887 rank counts: {2: 3, 3: 3, 4: 52, 5: 84, 6: 80, 7: 251, 8: 9483, 9: 7581, -1: 51746}
fraction not "other": 4.8%, nprob_obs=120

Sanity-checks

are the ML-selected objects meeting the fibermag and r cuts?


In [14]:
for h in hosts_to_target:
    cat = joined_catalogs[h]
    ranks = rankdct[h]

    plt.figure()
    
    msk = (ranks>0)&(ranks<8)
    plt.subplot(1,2,1)
    plt.hist(cat[msk]['r'],bins=25, histtype='step')
    plt.axvline(21,c='r')
    plt.xlabel('r')
    plt.title(h.name, fontsize=24)
    
    plt.subplot(1,2,2)
    plt.hist(cat[msk]['fibermag_r'],bins=25, histtype='step')
    plt.axvline(23,c='r')
    plt.xlabel('fibermag_r')
    plt.title(h.name, fontsize=24)
    
    plt.tight_layout()


Check that new ML predictions are fairly complete


In [17]:
bins = np.linspace(15,25, 100)
notinml = cat['OBJID'].mask
plt.hist(cat['r'], color='b', bins=bins, histtype='step', label='all')
plt.hist(cat['r'][notinml], color='r', bins=bins, histtype='step', label='insdssnotML')
plt.legend(loc=0)
None


check compared to the old mlpred file


In [18]:
mlpred_old = Table.read('catalogs/SAGAobjid_PredictionAug28_SDSS_SDSSwise_v4removed_combinedPs.fits.gz')
mlpred_old['OBJID'].name = 'objID'

In [19]:
# 
joined_catalogs_old = {}
for h in hosts_to_target:
    #filter on manual remove lists *and* REMOVE!=-1
    cat = targeting.remove_targets_with_remlist(h.get_sdss_catalog(), h, 'catalogs/TargetRemoveAug28_2015.csv')
    
    pre_joined = table.join(cat, mlpred_old, join_type='left', keys='objID')
    
    tokeep = pre_joined['REMOVE']==-1
    notobserved = pre_joined['ZQUALITY']<3
    print('removed', np.sum(~tokeep),'for remove list and', np.sum(~notobserved),'b/c observed')
    joined_catalogs_old[h] = pre_joined[tokeep&notobserved]
    highp = pre_joined[p_column_for_ranking]>0.01
    print('n w/ ML pred>0.01 and observed:', np.sum(highp&~notobserved), 'not yet obs:', np.sum(highp&notobserved))


Removed 4 objects for NSA150887
removed 250 for remove list and 722 b/c observed
n w/ ML pred>0.01 and observed: 0 not yet obs: 124

In [20]:
cat = joined_catalogs_old[n150887]
bins = np.linspace(15,25, 100)
notinml = cat['PROBABILITY_CLASS_1'].mask
plt.hist(cat['r'], color='b', bins=bins, histtype='step', label='all')
plt.hist(cat['r'][notinml], color='r', bins=bins, histtype='step', label='insdssnotML')
plt.legend(loc=0)
None


Now actually generate a catalog


In [22]:
generated_cats = {}
for h in hosts_to_target:
    print('On host', h.name)
    sys.stdout.flush()

    cat = joined_catalogs[h]
    ranks = rankdct[h]
    
    fnout = 'mmthecto/{0}_nov2015.cat'.format(h.name)
    fluxfnout = fnout.replace('.cat', '.fluxstars')
    print('Going to write', fnout)
    
    msk = (cat['RHOST_ARCM']<40) & (ranks>0) & (ranks<9)
    generated_cats[h] = mmthecto.generate_catalog(h, cat[msk], ranks[msk], 
                                         repeatflux=3, removefluxdistance=1*u.arcmin,
                                         fnout=fnout, fluxfnout=fluxfnout)


On host NSA150887
Going to write mmthecto/NSA150887_nov2015.cat
Including 4503 targets
Found 73 Flux stars
Removing 34 Flux stars too close to program stars
Could not find psf mags so falling back on regular mags...
Found 255 guide stars

Have a look at the input catalog


In [68]:
gcat = generated_cats[h]

In [77]:
ranknum = gcat['rank'].copy()
ranknum[gcat['rank']==''] = '-1'
ranknum = ranknum.astype(int)
{r: np.sum(r==ranknum) for r in set(ranknum)}


Out[77]:
{-1: 255, 1: 117, 2: 3, 4: 52, 5: 14, 6: 80, 7: 79, 8: 4275}

In [122]:
msk = (ranknum>1)&(ranknum<6)
gcatm = gcat[msk]
targeting.sampled_imagelist(gcatm[np.argsort(gcatm['ra'])], None, None, names=gcat['rank'][msk])
None

In [155]:
#identify clumps of nearby objects
gcatmsc = SkyCoord(gcatm['ra'], gcatm['dec'], unit=u.deg)
seps = u.Quantity([c.separation(gcatmsc) for c in gcatmsc])

pairs = zip(*np.where((seps>0.1*u.arcsec)&(seps<10*u.arcsec)))
pairs = [pair for pair in pairs if pair[0]<pair[1]]

clumps = []
for p1, p2 in pairs:
    for clump in clumps:
        if p1 in clump:
            clump.append(p2)
            break
        elif p2 in clump:
            clump.append(p1)
            break
    else:
        clumps.append([p1, p2])
for cl in clumps:
    targeting.sampled_imagelist(gcatm[np.array(cl)], None, None, names=cl)
clumps


Out[155]:
[[5, 6, 7],
 [18, 20],
 [24, 25, 28, 27],
 [33, 34, 35, 35],
 [46, 47, 48],
 [49, 50],
 [51, 52],
 [58, 59],
 [65, 66]]

In [161]:
np.sum(joined_catalogs[n150887]['objID']==long(1237678777399509169))


Out[161]:
1

In [163]:
cat = joined_catalogs[n150887]
clumpcats = []
for clump in clumps:
    msk = np.zeros(len(cat), dtype=bool)
    for c in clump:
        msk[cat['objID']==long(gcatm[c]['object'])] = True
    clumpcats.append(cat[msk])
clumpcats[0]


Out[163]:
<Table masked=True length=3>
objIDradecflagsspecObjIDugrizu_errg_errr_erri_errz_errAuAgArAiAzfibermag_rfiber2mag_rexpRad_rsb_exp_rpetroR50_rpetroR90_rpetroMag_rsb_petro_rJJerrHHerrKKerrw1w1errw2w2errspec_zspec_z_errspec_z_warnphotozphotoz_errHOST_RAHOST_DECHOST_DISTHOST_VHOSTHOST_MKHOST_NSAIDHOST_FLAGHOST_SAGA_NAMERHOST_ARCMRHOST_KPCOBJ_NSAIDPCLASS_1SATSREMOVETELNAMEMASKNAMEZQUALITYSPEC_REPEATUBVRIrhostrhost_kpctypephot_sgOBJIDRADECDERED_RPROBABILITY_CLASS_1RESCALED_PROBABILITY_CLASS_1BEST_GUESS_CLASS
int64float64float64int64int64float32float32float32float32float32float32float32float32float32float32float32float32float32float32float32float32float32float32float64float32float32float32float64float32float32float32float32float32float32float32float32float32float32float32float32int32float32float32float64float64float64float64float64int64int64string384float64float64int64int64int64int64string32string384float64string384float32float32float32float32float32float64float64int16string48int64float64float64float64float64float64int64
1237678777399509169348.8754707154.6146339985135253360136976017.952116.847316.447316.178316.09340.06250120.009963240.01184090.01455120.05448840.3699240.2721860.1974120.1496920.10613319.907420.424714.464424.24426590211.12921.92816.567323.79505787239999.09999.09999.09999.09999.09999.014.62280.09080549999.09999.0-1.0-1.0-10.0567390.019698348.6834.5340637.0192689.08.327150887012.486005011134.453883338-1-1-1-1-1.017.158117.191616.605316.289215.74850.208173896381133.3384367923GALAXY1237678777399509169348.8754707154.6146339985116.24990.4738886919280.5054881398710
1237678777399509170348.8741990974.613261474721231558551928920022.315920.624520.091919.815719.85340.3920780.03841790.03807880.04411040.1743270.369570.2719260.1972240.1495490.10603220.3321.16250.9124221.88830043441.149931.7662720.273722.57251919469999.09999.09999.09999.09999.09999.016.11660.045888815.5740.149-1.0-1.0-10.0522250.046889348.6834.5340637.0192689.08.327150887012.3840809333133.356332238-1-1-1-1-1.021.35221.010220.307619.932619.43330.206475522261132.2506032353GALAXY1237678777399509170348.8741990974.6132614747219.89470.08912066953020.06312521662060
1237678777399509175348.8733124224.61612028637246290906743128022.912921.629421.56622.765622.96570.5758390.0780980.1212130.5153721.138750.3695640.2719220.197220.1495460.1060321.834522.43550.49307122.04452437171.080312.0390221.657723.82089002249999.09999.09999.09999.09999.09999.017.61240.1811799999.09999.0-1.0-1.0-10.0602480.040626348.6834.5340637.0192689.08.327150887012.4022149121133.551604736-1-1-1-1-1.021.749221.868221.577621.632522.44580.206776961875132.4436797333GALAXY1237678777399509175348.8733124224.6161202863721.36880.09846447267910.07391436690660

In [167]:
clumped = []
for clump in clumps:
    for i in clump:
        clumped.append(i)
        
nmult = len(clumped) - len(clumps)
len(gcatm), len(clumped), nmult, len(clumped)/len(gcatm), nmult/len(gcatm)


Out[167]:
(69, 24, 15, 0.34782608695652173, 0.21739130434782608)

Have a looksee at the configuration output by xfitfibs


In [63]:
coords, targets, ranks, fields = mmthecto.parse_cfg_file('mmthecto/NSA150887_nov2015.cfg')

In [65]:
msk = ranks == 2
targeting.sampled_imagelist(coords[msk], None, None, names=targets[msk])


Out[65]:
'name ra dec\n1277 348.696642083 4.639955\n2945 348.394260417 4.55277194444\n1278 348.727540833 4.71334444444'

Production run


In [39]:
hostnames_to_target = [150887, 150578, 'OBrother', 150307, 'AnaK', 149977, 153017, 129387, 129237, 'Narnia', 126115]
hosts_to_target = []
for hostname in hostnames_to_target:
    if isinstance(hostname, basestring):
        for host in candhosts:
            if host.name == hostname:
                hosts_to_target.append(host)
                break
        else:
            raise ValueError('could not find host named {0}'.format(hostname))
    else:
        for host in candhosts:
            if host.nsaid == hostname:
                hosts_to_target.append(host)
                break                
        else:
            raise ValueError('could not find host with nsa number {0}'.format(hostname))
hosts_to_target


Out[39]:
[<hosts.NSAHost object w/ name 'NSA150887'>,
 <hosts.NSAHost object w/ name 'NSA150578'>,
 <hosts.NSAHost object w/ name 'OBrother' AKA: ['NSA149781']>,
 <hosts.NSAHost object w/ name 'NSA150307'>,
 <hosts.NSAHost object w/ name 'AnaK' AKA: ['NGC7716', 'NSA61945']>,
 <hosts.NSAHost object w/ name 'NSA149977'>,
 <hosts.NSAHost object w/ name 'NSA153017'>,
 <hosts.NSAHost object w/ name 'NSA129387'>,
 <hosts.NSAHost object w/ name 'NSA129237'>,
 <hosts.NSAHost object w/ name 'Narnia' AKA: ['NGC1015', 'NSA132339']>,
 <hosts.NSAHost object w/ name 'NSA126115'>]

In [40]:
joined_catalogs = {}
for h in hosts_to_target:
    #filter on manual remove lists *and* REMOVE!=-1
    #cat = targeting.remove_targets_with_remlist(h.get_sdss_catalog(), h, 'catalogs/TargetRemoveNov8_2015.csv')
    cat = targeting.remove_targets_with_remlist(h.get_sdss_catalog(), h, 'catalogs/TargetRemoveNov11_2015.csv')
    
    pre_joined = table.join(cat, mlpred, join_type='left', keys='objID')
    
    tokeep = pre_joined['REMOVE']==-1
    notobserved = pre_joined['ZQUALITY']<3
    print('removed', np.sum(~tokeep),'for remove list and', np.sum(~notobserved),'b/c observed')
    joined_catalogs[h] = pre_joined[tokeep&notobserved]
    highp = pre_joined[p_column_for_ranking]>0.01
    print('n w/ ML pred>0.01 and observed:', np.sum(highp&~notobserved), 'not yet obs:', np.sum(highp&notobserved))


Removed 36 objects for NSA150887
removed 250 for remove list and 722 b/c observed
n w/ ML pred>0.01 and observed: 7 not yet obs: 474
Removed 9 objects for NSA150578
removed 58 for remove list and 47 b/c observed
n w/ ML pred>0.01 and observed: 6 not yet obs: 702
Removed 12 objects for OBrother
removed 280 for remove list and 411 b/c observed
n w/ ML pred>0.01 and observed: 14 not yet obs: 723
Removed 20 objects for NSA150307
removed 80 for remove list and 2 b/c observed
n w/ ML pred>0.01 and observed: 1 not yet obs: 355
Removed 38 objects for AnaK
removed 479 for remove list and 2060 b/c observed
n w/ ML pred>0.01 and observed: 72 not yet obs: 778
Removed 11 objects for NSA149977
removed 52 for remove list and 225 b/c observed
n w/ ML pred>0.01 and observed: 5 not yet obs: 505
Removed 24 objects for NSA153017
removed 89 for remove list and 3 b/c observed
n w/ ML pred>0.01 and observed: 2 not yet obs: 447
Removed 21 objects for NSA129387
removed 282 for remove list and 317 b/c observed
n w/ ML pred>0.01 and observed: 12 not yet obs: 598
Removed 30 objects for NSA129237
removed 218 for remove list and 532 b/c observed
n w/ ML pred>0.01 and observed: 9 not yet obs: 958
Removed 15 objects for Narnia
removed 469 for remove list and 1185 b/c observed
n w/ ML pred>0.01 and observed: 70 not yet obs: 779
Removed 21 objects for NSA126115
removed 52 for remove list and 5 b/c observed
n w/ ML pred>0.01 and observed: 4 not yet obs: 441

Includes Redmapper


In [41]:
redmappertargs = table.Table.read('catalogs/redmagic_targets_0828.fit')
redmappertargs


Out[41]:
<Table masked=False length=631>
OBJIDRADECRMAG
int64float64float64float32
1237678776862507491348.5797776054.3278351404419.8827
1237678776862507396348.5248722724.3710496266420.2401
1237679005021766016348.4086321434.4864746187420.4682
1237679005021766095348.4341886954.5141557261620.0641
1237679005021766194348.4679628264.5172302080619.6637
1237678776862441902348.3756649024.3029249898420.5198
1237679005021700367348.3455920874.410483310918.9509
1237679005021701029348.3920771474.4384921914319.3736
1237679005021765999348.4078589654.4197279147119.4642
1237679005021766015348.4092094864.4703605164620.134
............
12376795043195132531.5804017444520.523302398119.6759
12376802473519351471.5978578456120.61336039819.9404
12376802473519353631.5672215343120.613519999919.7791
12376802473519354171.5893188504520.60142020619.9253
12376802468149337601.371879119420.291729315920.0632
12376802468149988931.3959075458620.319034508119.0755
12376802468149335571.3027878118320.257884137319.3816
12376802468149334421.2528969199620.238066210920.7478
12376802473515422090.62827009413620.637392107520.5838
12376795048559907170.60183053611820.792932545620.4004

In [42]:
# compute ranks, which includes filtering on mag
rankdct = {}
for h in hosts_to_target:
    cat = joined_catalogs[h]
    
    highprob = cat[p_column_for_ranking].filled(0) > 0.5
    medprob = (cat[p_column_for_ranking].filled(0) > 0.05) & ~highprob
    lowprob = (cat[p_column_for_ranking].filled(0) > 0.01) & ~highprob & ~medprob
    inside = cat['RHOST_KPC']<300
    
    rankdct[h] = ranks = np.ones(len(cat), dtype=int)*-1 #the -1 will be re-set at the end
    
    # rank 1 reserved for flux stars
    ranks[inside&highprob] = 2
    ranks[~inside&highprob] = 3
    ranks[inside&medprob] = 4
    ranks[~inside&medprob] = 5
    ranks[inside&lowprob] = 6
    ranks[~inside&lowprob] = 7
    
    validtarget = (cat['r'] < 21.5) & (cat['fibermag_r']<23) & (cat['phot_sg']=='GALAXY')
    other = ranks < 0
    colorcutmsk = targeting.colorcut_mask(cat,  targeting.bossanova_color_cuts)
    
    ranks[validtarget&other&colorcutmsk] = 9 #other incl. color cuts
    ranks[validtarget&other&~colorcutmsk] = 10 #other not incl. color cuts
    redmappertarg = np.in1d(cat['objID'], redmappertargs['OBJID'])
    rg7 = ranks>7
    ranks[validtarget&redmappertarg&rg7] = 8 #redmapper
    
#informational
for h, ranks in rankdct.items():
    cat = joined_catalogs[h]
    
    hranks = dict(enumerate(np.bincount(ranks[ranks>-1])))
    del hranks[0]
    del hranks[1]
    hranks[-1] = np.sum(ranks==-1)
    
    print(h.name,'rank counts:', hranks)
    
    nother =  np.sum(ranks==8)
    nprob = np.sum((ranks<8)&(ranks>0))
    nother_obs = 270-np.sum((ranks<8)&(ranks>0)&(cat['RHOST_ARCM']<30))
    print('fraction not "other": {0:.1%}, nprob_obs={1}\n'.format(nprob/(nprob+nother), nother_obs))


NSA129237 rank counts: {2: 1, 3: 5, 4: 62, 5: 273, 6: 108, 7: 494, 8: 35, 9: 6879, 10: 4764, -1: 45505}
fraction not "other": 96.4%, nprob_obs=56

NSA126115 rank counts: {2: 0, 3: 0, 4: 27, 5: 94, 6: 73, 7: 240, 8: 62, 9: 8320, 10: 5905, -1: 42900}
fraction not "other": 87.5%, nprob_obs=185

NSA150307 rank counts: {2: 0, 3: 3, 4: 16, 5: 58, 6: 50, 7: 210, 8: 24, 9: 8229, 10: 5351, -1: 42612}
fraction not "other": 93.4%, nprob_obs=184

AnaK rank counts: {2: 0, 3: 1, 4: 33, 5: 171, 6: 127, 7: 433, 8: 75, 9: 9868, 10: 5341, -1: 60359}
fraction not "other": 91.1%, nprob_obs=106

NSA153017 rank counts: {2: 0, 3: 0, 4: 44, 5: 38, 6: 191, 7: 165, 8: 66, 9: 9011, 10: 6426, -1: 46178}
fraction not "other": 86.9%, nprob_obs=191

NSA129387 rank counts: {2: 0, 3: 2, 4: 36, 5: 105, 6: 96, 7: 346, 8: 49, 9: 10017, 10: 6065, -1: 53318}
fraction not "other": 92.3%, nprob_obs=108

Narnia rank counts: {2: 0, 3: 2, 4: 39, 5: 180, 6: 66, 7: 443, 8: 63, 9: 10738, 10: 5803, -1: 56191}
fraction not "other": 92.1%, nprob_obs=136

OBrother rank counts: {2: 0, 3: 2, 4: 19, 5: 163, 6: 84, 7: 440, 8: 66, 9: 9093, 10: 6090, -1: 63281}
fraction not "other": 91.5%, nprob_obs=133

NSA149977 rank counts: {2: 0, 3: 0, 4: 14, 5: 85, 6: 67, 7: 331, 8: 38, 9: 6470, 10: 4561, -1: 51761}
fraction not "other": 92.9%, nprob_obs=161

NSA150578 rank counts: {2: 0, 3: 3, 4: 30, 5: 125, 6: 128, 7: 416, 8: 46, 9: 7388, 10: 6605, -1: 45888}
fraction not "other": 93.9%, nprob_obs=121

NSA150887 rank counts: {2: 2, 3: 3, 4: 32, 5: 84, 6: 69, 7: 251, 8: 94, 9: 9483, 10: 7487, -1: 51746}
fraction not "other": 82.4%, nprob_obs=152

Visual Removal of shreds


In [28]:
def clump_finder(sc, clumpsep=10*u.arcsec):
    """
    Find clumps within a certain distance in a SkyCoord catalog
    """
    sc1 = SkyCoord(sc.ra.reshape(1, len(sc)), sc.dec.reshape(1, len(sc)))
    sc2 = SkyCoord(sc.ra.reshape(len(sc), 1), sc.dec.reshape(len(sc), 1))
    seps = sc1.separation(sc2)
    #seps = u.Quantity([c.separation(sc) for c in sc])

    pairs = zip(*np.where((seps>0.1*u.arcsec)&(seps<clumpsep)))
    pairs = [pair for pair in pairs if pair[0]<pair[1]]

    clumps = []
    for p1, p2 in pairs:
        for clump in clumps:
            if p1 in clump:
                clump.append(p2)
                break
            elif p2 in clump:
                clump.append(p1)
                break
        else:
            clumps.append([p1, p2])
    return clumps

clumpdct = {}
for h in hosts_to_target:
    cat = joined_catalogs[h]
    ranks = rankdct[h]
    
    #msk = (1<ranks)&(ranks<8)
    msk = (ranks==2)|(ranks==4)|(ranks==6)
    
    catm = cat[msk]
    highp_scs = SkyCoord(catm['ra'], catm['dec'], unit=u.deg)
    clumpdct[h] = (clump_finder(highp_scs), msk)

In [30]:
#now we go through each and show the clumps 
for h in hosts_to_target:
    cat = joined_catalogs[h]
    clumps, clumpmsk = clumpdct[h]
    
    clumpnum = []
    clumpra = []
    clumpdec = []
    clumpobjid = []
    for i, c in enumerate(clumps):
        print('Clump#', i)
        for idx in c:
            clumpnum.append(i)
            clumpra.append(cat[clumpmsk][idx]['ra'])
            clumpdec.append(cat[clumpmsk][idx]['dec'])
            clumpobjid.append(cat[clumpmsk][idx]['objID'])
            if h.name.startswith('NSA'):
                nmstr = ''
            else:
                nmstr = h.name
            print('{0}\t{1}\t{2}\t{3}\t{4}\t{5}'.format(nmstr, h.nsaid, clumpobjid[-1], clumpra[-1], clumpdec[-1],'shred galaxy <semiauto-found>'))
            
        
    targeting.sampled_imagelist(clumpra, clumpdec, n=np.inf, names=clumpnum)
        
    res = raw_input('{} <enter to continue>'.format(h.name, h.nsaid))
    if res == 'x':
        break


NSA150887 <enter to continue>
NSA150578 <enter to continue>
OBrother <enter to continue>
NSA150307 <enter to continue>
Clump# 0
AnaK	61945	1237663277925335666	354.474453404	0.598619137594	shred galaxy <semiauto-found>
AnaK	61945	1237663277925335668	354.473556524	0.596307668716	shred galaxy <semiauto-found>
Clump# 1
AnaK	61945	1237663277925335849	354.526252686	0.536511405586	shred galaxy <semiauto-found>
AnaK	61945	1237663277925335850	354.527253118	0.533963323664	shred galaxy <semiauto-found>
AnaK <enter to continue>
NSA149977 <enter to continue>
Clump# 0
	153017	1237679504318857846	359.881179224	20.4716167358	shred galaxy <semiauto-found>
	153017	1237679504318857848	359.8815262	20.4690042853	shred galaxy <semiauto-found>
NSA153017 <enter to continue>
Clump# 0
	129387	1237678860091261057	21.1199772173	12.7913912476	shred galaxy <semiauto-found>
	129387	1237678860091261058	21.1193197583	12.7935316946	shred galaxy <semiauto-found>
NSA129387 <enter to continue>
Clump# 0
	129237	1237679169304723553	20.4180467989	17.385759242	shred galaxy <semiauto-found>
	129237	1237679169304723554	20.4190795227	17.3835685487	shred galaxy <semiauto-found>
NSA129237 <enter to continue>
Clump# 0
Narnia	132339	1237678890137616709	39.7498490775	-1.5948937078	shred galaxy <semiauto-found>
Narnia	132339	1237678890137616711	39.7489538331	-1.59642970008	shred galaxy <semiauto-found>
Narnia <enter to continue>
NSA126115 <enter to continue>

Keeping output of old version used to make rem list


In [61]:
#now we go through each and show the clumps 
#NOTE: the output should now be much smaller because the rem list is update
for h in hosts_to_target:
    cat = joined_catalogs[h]
    clumps, clumpmsk = clumpdct[h]
    
    clumpnum = []
    clumpra = []
    clumpdec = []
    clumpobjid = []
    for i, c in enumerate(clumps):
        print('Clump#', i)
        for idx in c:
            clumpnum.append(i)
            clumpra.append(cat[clumpmsk][idx]['ra'])
            clumpdec.append(cat[clumpmsk][idx]['dec'])
            clumpobjid.append(cat[clumpmsk][idx]['objID'])
            if h.name.startswith('NSA'):
                nmstr = ''
            else:
                nmstr = h.name
            print('{0}\t{1}\t{2}\t{3}\t{4}\t{5}'.format(nmstr, h.nsaid, clumpobjid[-1], clumpra[-1], clumpdec[-1],'shred galaxy <semiauto-found>'))
            
        
    targeting.sampled_imagelist(clumpra, clumpdec, n=np.inf, names=clumpnum)
        
    res = raw_input('{} <enter to continue>'.format(h.name, h.nsaid))
    if res == 'x':
        break


Clump# 0
	150887	1237678777399509169	348.875470715	4.61463399851	shred galaxy <semiauto-found>
	150887	1237678777399509170	348.874199097	4.61326147472	shred galaxy <semiauto-found>
	150887	1237678777399509171	348.874889667	4.61299644629	shred galaxy <semiauto-found>
	150887	1237678777399509175	348.873312422	4.61612028637	shred galaxy <semiauto-found>
	150887	1237678777399509177	348.875624665	4.61270778777	shred galaxy <semiauto-found>
	150887	1237678777399509171	348.874889667	4.61299644629	shred galaxy <semiauto-found>
	150887	1237678777399509173	348.871772577	4.6142060337	shred galaxy <semiauto-found>
	150887	1237678777399509174	348.87255077	4.61281086443	shred galaxy <semiauto-found>
	150887	1237678777399509177	348.875624665	4.61270778777	shred galaxy <semiauto-found>
	150887	1237678777399509174	348.87255077	4.61281086443	shred galaxy <semiauto-found>
	150887	1237678777399509177	348.875624665	4.61270778777	shred galaxy <semiauto-found>
	150887	1237678777399509174	348.87255077	4.61281086443	shred galaxy <semiauto-found>
	150887	1237678777399509175	348.873312422	4.61612028637	shred galaxy <semiauto-found>
	150887	1237678777399509179	348.869380157	4.61392946807	shred galaxy <semiauto-found>
Clump# 1
	150887	1237679005021700193	348.262759776	4.51102497818	shred galaxy <semiauto-found>
	150887	1237679005021700198	348.262752946	4.5115619791	shred galaxy <semiauto-found>
Clump# 2
	150887	1237679005021700287	348.392673002	4.55467286999	shred galaxy <semiauto-found>
	150887	1237679005021700288	348.394250267	4.55277230941	shred galaxy <semiauto-found>
	150887	1237679005021700293	348.395003897	4.55037932019	shred galaxy <semiauto-found>
	150887	1237679005021700292	348.393499392	4.5500727096	shred galaxy <semiauto-found>
Clump# 3
	150887	1237679005021765924	348.399199156	4.56663213131	shred galaxy <semiauto-found>
	150887	1237679005021765925	348.399470202	4.56707482085	shred galaxy <semiauto-found>
	150887	1237679005021765929	348.399741866	4.56784271094	shred galaxy <semiauto-found>
	150887	1237679005021765929	348.399741866	4.56784271094	shred galaxy <semiauto-found>
Clump# 4
	150887	1237679005021831462	348.620320878	4.37953179828	shred galaxy <semiauto-found>
	150887	1237679005021831464	348.620792614	4.37808731721	shred galaxy <semiauto-found>
Clump# 5
	150887	1237679005021962536	348.947720513	4.53281842475	shred galaxy <semiauto-found>
	150887	1237679005021962537	348.947099236	4.53092545954	shred galaxy <semiauto-found>
Clump# 6
	150887	1237679005022028541	349.065082463	4.49788962084	shred galaxy <semiauto-found>
	150887	1237679005022028542	349.06690645	4.49676632321	shred galaxy <semiauto-found>
Clump# 7
	150887	1237679005558636613	348.463135606	4.90493482583	shred galaxy <semiauto-found>
	150887	1237679005558636614	348.463627263	4.90502247936	shred galaxy <semiauto-found>
	150887	1237679005558636616	348.463484582	4.90748285897	shred galaxy <semiauto-found>
	150887	1237679005558636619	348.46120453	4.90438763983	shred galaxy <semiauto-found>
	150887	1237679005558636621	348.460703009	4.90446717938	shred galaxy <semiauto-found>
	150887	1237679005558636616	348.463484582	4.90748285897	shred galaxy <semiauto-found>
	150887	1237679005558636619	348.46120453	4.90438763983	shred galaxy <semiauto-found>
	150887	1237679005558636621	348.460703009	4.90446717938	shred galaxy <semiauto-found>
Clump# 8
	150887	1237679005558636865	348.479035772	4.8840183344	shred galaxy <semiauto-found>
	150887	1237679005558636869	348.477865827	4.88497463099	shred galaxy <semiauto-found>
Clump# 9
	150887	1237679005558637016	348.400848079	4.86812518008	shred galaxy <semiauto-found>
	150887	1237679005558637017	348.399643671	4.86673232256	shred galaxy <semiauto-found>
	150887	1237679005558637018	348.399782196	4.867484181	shred galaxy <semiauto-found>
	150887	1237679005558637018	348.399782196	4.867484181	shred galaxy <semiauto-found>
Clump# 10
	150887	1237679005558702452	348.649763333	4.87755963456	shred galaxy <semiauto-found>
	150887	1237679005558702453	348.650122048	4.87793328256	shred galaxy <semiauto-found>
Clump# 11
	150887	1237679005558767959	348.687609814	4.9226927841	shred galaxy <semiauto-found>
	150887	1237679005558767960	348.689507321	4.92204647702	shred galaxy <semiauto-found>
Clump# 12
	150887	1237679005558833479	348.895435695	4.83443530347	shred galaxy <semiauto-found>
	150887	1237679005558833483	348.895778528	4.83427597138	shred galaxy <semiauto-found>
Clump# 13
	150887	1237679005558833490	348.936664639	4.91567664312	shred galaxy <semiauto-found>
	150887	1237679005558833491	348.937161573	4.91584021414	shred galaxy <semiauto-found>
NSA150887 <enter to continue>
Clump# 0
	150578	1237678859539448025	346.143610763	12.2606082117	shred galaxy <semiauto-found>
	150578	1237678859539448026	346.143275929	12.2619884572	shred galaxy <semiauto-found>
Clump# 1
	150578	1237678859539710417	346.736889511	12.1714348191	shred galaxy <semiauto-found>
	150578	1237678859539710418	346.736475555	12.1711551982	shred galaxy <semiauto-found>
Clump# 2
	150578	1237678860076515632	346.681876851	12.5783212981	shred galaxy <semiauto-found>
	150578	1237678860076515633	346.682141667	12.5771753184	shred galaxy <semiauto-found>
Clump# 3
	150578	1237678920198979824	346.714555543	12.3464284582	shred galaxy <semiauto-found>
	150578	1237678920198979827	346.714002518	12.3466737188	shred galaxy <semiauto-found>
NSA150578 <enter to continue>
Clump# 0
OBrother	149781	1237679996084617517	335.97997618	-3.27054858631	shred galaxy <semiauto-found>
OBrother	149781	1237679996084617518	335.980436147	-3.26967074834	shred galaxy <semiauto-found>
Clump# 1
OBrother	149781	1237679996084617615	336.042463379	-3.30705542795	shred galaxy <semiauto-found>
OBrother	149781	1237679996084617622	336.043087481	-3.30861733053	shred galaxy <semiauto-found>
OBrother	149781	1237679996084617623	336.042569546	-3.30729495633	shred galaxy <semiauto-found>
OBrother	149781	1237679996084617623	336.042569546	-3.30729495633	shred galaxy <semiauto-found>
Clump# 2
OBrother	149781	1237679996084682961	336.130784555	-3.24599422746	shred galaxy <semiauto-found>
OBrother	149781	1237679996084682967	336.128210676	-3.24648987621	shred galaxy <semiauto-found>
Clump# 3
OBrother	149781	1237679996084748652	336.24029495	-3.34105229041	shred galaxy <semiauto-found>
OBrother	149781	1237679996084748653	336.239812694	-3.3414413667	shred galaxy <semiauto-found>
Clump# 4
OBrother	149781	1237680066954461497	336.233135666	-3.66379266792	shred galaxy <semiauto-found>
OBrother	149781	1237680066954461498	336.233927932	-3.66358515533	shred galaxy <semiauto-found>
OBrother <enter to continue>
Clump# 0
	150307	1237680064810516513	344.308630229	-5.69407939809	shred galaxy <semiauto-found>
	150307	1237680064810516520	344.306710996	-5.69546900002	shred galaxy <semiauto-found>
	150307	1237680064810516514	344.306853193	-5.69634327139	shred galaxy <semiauto-found>
Clump# 1
	150307	1237680064810516695	344.235258628	-5.53660088285	shred galaxy <semiauto-found>
	150307	1237680064810516699	344.233397057	-5.53556291763	shred galaxy <semiauto-found>
Clump# 2
	150307	1237680116342128690	343.828873761	-5.73589619098	shred galaxy <semiauto-found>
	150307	1237680116342128732	343.829435076	-5.73341847089	shred galaxy <semiauto-found>
Clump# 3
	150307	1237680116342128709	343.835391803	-5.72786056782	shred galaxy <semiauto-found>
	150307	1237680116342128739	343.834614583	-5.72772884505	shred galaxy <semiauto-found>
Clump# 4
	150307	1237680116342128742	343.834264962	-5.7220055619	shred galaxy <semiauto-found>
	150307	1237680116342128746	343.833677564	-5.72266055058	shred galaxy <semiauto-found>
Clump# 5
	150307	1237680116342128743	343.841934415	-5.72585195038	shred galaxy <semiauto-found>
	150307	1237680116342128750	343.842696274	-5.72754652012	shred galaxy <semiauto-found>
Clump# 6
	150307	1237680116342128744	343.836126308	-5.71889324999	shred galaxy <semiauto-found>
	150307	1237680116342128763	343.836246451	-5.71690212366	shred galaxy <semiauto-found>
	150307	1237680116342128767	343.834910393	-5.71782160736	shred galaxy <semiauto-found>
	150307	1237680116342128767	343.834910393	-5.71782160736	shred galaxy <semiauto-found>
Clump# 7
	150307	1237680116879130861	344.131710232	-5.31621882701	shred galaxy <semiauto-found>
	150307	1237680116879130863	344.132099502	-5.31560464722	shred galaxy <semiauto-found>
Clump# 8
	150307	1237680116879196393	344.260102892	-5.50009053777	shred galaxy <semiauto-found>
	150307	1237680116879196394	344.26112652	-5.4997614966	shred galaxy <semiauto-found>
NSA150307 <enter to continue>
Clump# 0
AnaK	61945	1237663277925335177	354.452155223	0.571743074265	shred galaxy <semiauto-found>
AnaK	61945	1237663277925335179	354.451710378	0.571253510325	shred galaxy <semiauto-found>
Clump# 1
AnaK	61945	1237663277925335666	354.474453404	0.598619137594	shred galaxy <semiauto-found>
AnaK	61945	1237663277925335668	354.473556524	0.596307668716	shred galaxy <semiauto-found>
Clump# 2
AnaK	61945	1237663277925335849	354.526252686	0.536511405586	shred galaxy <semiauto-found>
AnaK	61945	1237663277925335850	354.527253118	0.533963323664	shred galaxy <semiauto-found>
AnaK	61945	1237663277925335851	354.528240348	0.534306627776	shred galaxy <semiauto-found>
Clump# 3
AnaK	61945	1237663784194801801	354.017523976	0.0744706829001	shred galaxy <semiauto-found>
AnaK	61945	1237663784194801805	354.017501763	0.0755611531018	shred galaxy <semiauto-found>
Clump# 4
AnaK	61945	1237663784194801968	354.020622047	0.0721790045169	shred galaxy <semiauto-found>
AnaK	61945	1237663784194802608	354.022149447	0.0698653270433	shred galaxy <semiauto-found>
Clump# 5
AnaK	61945	1237663784195063812	354.613366663	0.201304756194	shred galaxy <semiauto-found>
AnaK	61945	1237663784195064879	354.614767727	0.199325294099	shred galaxy <semiauto-found>
Clump# 6
AnaK	61945	1237666408439612142	353.651205995	0.388295129826	shred galaxy <semiauto-found>
AnaK	61945	1237666408439612143	353.651621028	0.387283517283	shred galaxy <semiauto-found>
AnaK <enter to continue>
Clump# 0
	149977	1237678857925886377	339.383530295	10.313398898	shred galaxy <semiauto-found>
	149977	1237678857925886378	339.382812087	10.3130795172	shred galaxy <semiauto-found>
Clump# 1
	149977	1237678857926017517	339.685495242	10.4726036155	shred galaxy <semiauto-found>
	149977	1237678857926017519	339.685749129	10.4729804218	shred galaxy <semiauto-found>
Clump# 2
	149977	1237678858462626158	339.05887431	10.6516080995	shred galaxy <semiauto-found>
	149977	1237678858462626161	339.060107693	10.649892701	shred galaxy <semiauto-found>
Clump# 3
	149977	1237678858462757206	339.469112453	10.7553336146	shred galaxy <semiauto-found>
	149977	1237678858462757207	339.467882212	10.7531605977	shred galaxy <semiauto-found>
	149977	1237678858462757209	339.467280968	10.7522146571	shred galaxy <semiauto-found>
Clump# 4
	149977	1237678858462757431	339.351045429	10.8351990284	shred galaxy <semiauto-found>
	149977	1237678858462757432	339.349211541	10.8356721305	shred galaxy <semiauto-found>
Clump# 5
	149977	1237679010927476976	339.677347878	10.2111933345	shred galaxy <semiauto-found>
	149977	1237679010927476977	339.677574886	10.2094227729	shred galaxy <semiauto-found>
NSA149977 <enter to continue>
Clump# 0
	153017	1237679503782051902	0.0352644888843	19.9647649699	shred galaxy <semiauto-found>
	153017	1237679503782051904	0.0356566893112	19.9648745953	shred galaxy <semiauto-found>
Clump# 1
	153017	1237679504318857641	359.884255892	20.4790811116	shred galaxy <semiauto-found>
	153017	1237679504318857642	359.884309426	20.4795137184	shred galaxy <semiauto-found>
Clump# 2
	153017	1237679504318857846	359.881179224	20.4716167358	shred galaxy <semiauto-found>
	153017	1237679504318857848	359.8815262	20.4690042853	shred galaxy <semiauto-found>
Clump# 3
	153017	1237679504319185119	0.743337181228	20.3857416247	shred galaxy <semiauto-found>
	153017	1237679504319185121	0.744448670795	20.3864990704	shred galaxy <semiauto-found>
Clump# 4
	153017	1237679504855990416	0.500673133985	20.9872046114	shred galaxy <semiauto-found>
	153017	1237679504855990421	0.498317579732	20.9870444953	shred galaxy <semiauto-found>
	153017	1237679504855990422	0.500528286965	20.9875740247	shred galaxy <semiauto-found>
	153017	1237679504855990422	0.500528286965	20.9875740247	shred galaxy <semiauto-found>
Clump# 5
	153017	1237680246814736851	0.790376255819	20.3585106636	shred galaxy <semiauto-found>
	153017	1237680246814736852	0.789950039917	20.3591052771	shred galaxy <semiauto-found>
Clump# 6
	153017	1237680247351083137	359.463925664	20.697705675	shred galaxy <semiauto-found>
	153017	1237680247351083138	359.464152517	20.6973783299	shred galaxy <semiauto-found>
	153017	1237680247351083141	359.464556318	20.6969454017	shred galaxy <semiauto-found>
	153017	1237680247351083141	359.464556318	20.6969454017	shred galaxy <semiauto-found>
Clump# 7
	153017	1237680247351542037	0.599028432933	20.6829569705	shred galaxy <semiauto-found>
	153017	1237680247351542039	0.599304111176	20.6816607634	shred galaxy <semiauto-found>
Clump# 8
	153017	1237680247888019633	359.679552621	21.1476609229	shred galaxy <semiauto-found>
	153017	1237680247888019640	359.677529313	21.1477379847	shred galaxy <semiauto-found>
	153017	1237680247888019639	359.67679888	21.1494846944	shred galaxy <semiauto-found>
Clump# 9
	153017	1237680247888019634	359.672929531	21.1408115913	shred galaxy <semiauto-found>
	153017	1237680247888019636	359.674752501	21.1417033995	shred galaxy <semiauto-found>
Clump# 10
	153017	1237680247888085341	359.799624554	21.0120940079	shred galaxy <semiauto-found>
	153017	1237680247888085471	359.7982216	21.0110491474	shred galaxy <semiauto-found>
Clump# 11
	153017	1237680296744255765	359.953639542	21.1611585849	shred galaxy <semiauto-found>
	153017	1237680296744255771	359.953367762	21.1627259113	shred galaxy <semiauto-found>
	153017	1237680296744255773	359.952906628	21.1639405317	shred galaxy <semiauto-found>
	153017	1237680296744255774	359.952492231	21.1665559895	shred galaxy <semiauto-found>
NSA153017 <enter to continue>
Clump# 0
	129387	1237678860091261057	21.1199772173	12.7913912476	shred galaxy <semiauto-found>
	129387	1237678860091261058	21.1193197583	12.7935316946	shred galaxy <semiauto-found>
Clump# 1
	129387	1237678860091261161	21.2372307042	12.8553307298	shred galaxy <semiauto-found>
	129387	1237678860091261162	21.2378709934	12.8553464403	shred galaxy <semiauto-found>
Clump# 2
	129387	1237678920213528875	20.7039082382	12.5792548791	shred galaxy <semiauto-found>
	129387	1237678920213528876	20.7049319801	12.5792560281	shred galaxy <semiauto-found>
Clump# 3
	129387	1237678920213528916	20.7993195216	12.6301021648	shred galaxy <semiauto-found>
	129387	1237678920213528919	20.7998983449	12.6321441174	shred galaxy <semiauto-found>
Clump# 4
	129387	1237678920213594379	20.9856939115	12.649446754	shred galaxy <semiauto-found>
	129387	1237678920213594383	20.9841406978	12.6517735541	shred galaxy <semiauto-found>
Clump# 5
	129387	1237678920750465048	20.9009006536	12.9268280106	shred galaxy <semiauto-found>
	129387	1237678920750465049	20.9010245884	12.9272494364	shred galaxy <semiauto-found>
Clump# 6
	129387	1237678920750596202	21.2386400463	12.9029975868	shred galaxy <semiauto-found>
	129387	1237678920750596205	21.240782903	12.9029257073	shred galaxy <semiauto-found>
NSA129387 <enter to continue>
Clump# 0
	129237	1237678601850519687	20.139730757	17.3434985062	shred galaxy <semiauto-found>
	129237	1237678601850519688	20.1374937596	17.344519813	shred galaxy <semiauto-found>
Clump# 1
	129237	1237678601850650840	20.4153651474	17.2949101301	shred galaxy <semiauto-found>
	129237	1237678601850650843	20.4176668365	17.2948487244	shred galaxy <semiauto-found>
Clump# 2
	129237	1237678601850650900	20.3843676201	17.2666429342	shred galaxy <semiauto-found>
	129237	1237678601850651144	20.3864418251	17.2672320778	shred galaxy <semiauto-found>
Clump# 3
	129237	1237678601850716722	20.6519778236	17.3023550232	shred galaxy <semiauto-found>
	129237	1237678601850716724	20.6526973809	17.3014280585	shred galaxy <semiauto-found>
Clump# 4
	129237	1237678601850781900	20.7882226782	17.2879247202	shred galaxy <semiauto-found>
	129237	1237678601850781967	20.7883729416	17.2897524704	shred galaxy <semiauto-found>
Clump# 5
	129237	1237678601850781905	20.778102048	17.2922307107	shred galaxy <semiauto-found>
	129237	1237678601850781907	20.7798775661	17.290332519	shred galaxy <semiauto-found>
Clump# 6
	129237	1237678602387325534	20.0791687479	17.7356497279	shred galaxy <semiauto-found>
	129237	1237678602387325535	20.0791964396	17.7351209401	shred galaxy <semiauto-found>
Clump# 7
	129237	1237678602387456102	20.2680762824	17.579001312	shred galaxy <semiauto-found>
	129237	1237678602387456108	20.2674686053	17.5773930846	shred galaxy <semiauto-found>
Clump# 8
	129237	1237678602387456304	20.3279058719	17.7533858543	shred galaxy <semiauto-found>
	129237	1237678602387456305	20.3284666123	17.7539390491	shred galaxy <semiauto-found>
Clump# 9
	129237	1237678602387521791	20.5363426909	17.5281428748	shred galaxy <semiauto-found>
	129237	1237679169304789021	20.535739652	17.5275968037	shred galaxy <semiauto-found>
Clump# 10
	129237	1237679169304658233	20.2565174987	17.366638985	shred galaxy <semiauto-found>
	129237	1237679169304658234	20.2573862631	17.3669641401	shred galaxy <semiauto-found>
Clump# 11
	129237	1237679169304723553	20.4180467989	17.385759242	shred galaxy <semiauto-found>
	129237	1237679169304723554	20.4190795227	17.3835685487	shred galaxy <semiauto-found>
	129237	1237679169304723556	20.4193567215	17.3821970939	shred galaxy <semiauto-found>
Clump# 12
	129237	1237679169304723898	20.4486251215	17.3472958808	shred galaxy <semiauto-found>
	129237	1237679169304723899	20.4499280155	17.348045835	shred galaxy <semiauto-found>
Clump# 13
	129237	1237679169304855180	20.7601614677	17.3329583053	shred galaxy <semiauto-found>
	129237	1237679169304855184	20.7617786996	17.3325407316	shred galaxy <semiauto-found>
Clump# 14
	129237	1237679169841463798	20.1470869392	17.8420499141	shred galaxy <semiauto-found>
	129237	1237679169841463964	20.1470021461	17.8446289253	shred galaxy <semiauto-found>
Clump# 15
	129237	1237679169841725673	20.7768903825	17.8914501798	shred galaxy <semiauto-found>
	129237	1237679169841725674	20.7745992357	17.889759388	shred galaxy <semiauto-found>
NSA129237 <enter to continue>
Clump# 0
Narnia	132339	1237660024521097903	39.6563496491	-1.24509782837	shred galaxy <semiauto-found>
Narnia	132339	1237660024521097904	39.6538379293	-1.24549265738	shred galaxy <semiauto-found>
Clump# 1
Narnia	132339	1237678881037877350	39.4671357126	-1.66560234188	shred galaxy <semiauto-found>
Narnia	132339	1237678881037877364	39.4676879944	-1.6675854594	shred galaxy <semiauto-found>
Clump# 2
Narnia	132339	1237678881037877356	39.4692995141	-1.6622428003	shred galaxy <semiauto-found>
Narnia	132339	1237678881037877357	39.4701004746	-1.66242714076	shred galaxy <semiauto-found>
Clump# 3
Narnia	132339	1237678881037877358	39.4667640707	-1.66003424542	shred galaxy <semiauto-found>
Narnia	132339	1237678881037877361	39.4644531183	-1.65926288403	shred galaxy <semiauto-found>
Narnia	132339	1237678881037877362	39.4676998029	-1.65993365261	shred galaxy <semiauto-found>
Clump# 4
Narnia	132339	1237678881037943006	39.6240114966	-1.76010934055	shred galaxy <semiauto-found>
Narnia	132339	1237678881037943007	39.6243085214	-1.76035227571	shred galaxy <semiauto-found>
Clump# 5
Narnia	132339	1237678890137616709	39.7498490775	-1.5948937078	shred galaxy <semiauto-found>
Narnia	132339	1237678890137616711	39.7489538331	-1.59642970008	shred galaxy <semiauto-found>
Narnia <enter to continue>
Clump# 0
	126115	1237679504319185119	0.743337181228	20.3857416247	shred galaxy <semiauto-found>
	126115	1237679504319185121	0.744448670795	20.3864990704	shred galaxy <semiauto-found>
Clump# 1
	126115	1237679504319447189	1.42147036558	20.4790297383	shred galaxy <semiauto-found>
	126115	1237679504319447190	1.42244761771	20.4768027762	shred galaxy <semiauto-found>
	126115	1237679504319447191	1.42050157128	20.4809163743	shred galaxy <semiauto-found>
	126115	1237679504319447198	1.42174266319	20.478248075	shred galaxy <semiauto-found>
	126115	1237679504319447199	1.4194771981	20.4800145103	shred galaxy <semiauto-found>
	126115	1237679504319512686	1.42328040502	20.4784252344	shred galaxy <semiauto-found>
	126115	1237679504319447198	1.42174266319	20.478248075	shred galaxy <semiauto-found>
	126115	1237679504319512686	1.42328040502	20.4784252344	shred galaxy <semiauto-found>
	126115	1237679504319447194	1.41833711196	20.4793611138	shred galaxy <semiauto-found>
	126115	1237679504319447197	1.41783293265	20.4797089507	shred galaxy <semiauto-found>
	126115	1237679504319447199	1.4194771981	20.4800145103	shred galaxy <semiauto-found>
	126115	1237679504319512687	1.42255351217	20.4817143817	shred galaxy <semiauto-found>
	126115	1237679504319447196	1.41691702687	20.4784281918	shred galaxy <semiauto-found>
	126115	1237679504319447197	1.41783293265	20.4797089507	shred galaxy <semiauto-found>
	126115	1237679504319447199	1.4194771981	20.4800145103	shred galaxy <semiauto-found>
	126115	1237679504319447197	1.41783293265	20.4797089507	shred galaxy <semiauto-found>
	126115	1237679504319447199	1.4194771981	20.4800145103	shred galaxy <semiauto-found>
	126115	1237679504319447199	1.4194771981	20.4800145103	shred galaxy <semiauto-found>
	126115	1237679504319512686	1.42328040502	20.4784252344	shred galaxy <semiauto-found>
Clump# 2
	126115	1237679504855990416	0.500673133985	20.9872046114	shred galaxy <semiauto-found>
	126115	1237679504855990421	0.498317579732	20.9870444953	shred galaxy <semiauto-found>
	126115	1237679504855990422	0.500528286965	20.9875740247	shred galaxy <semiauto-found>
	126115	1237679504855990422	0.500528286965	20.9875740247	shred galaxy <semiauto-found>
Clump# 3
	126115	1237680246814736851	0.790376255819	20.3585106636	shred galaxy <semiauto-found>
	126115	1237680246814736852	0.789950039917	20.3591052771	shred galaxy <semiauto-found>
Clump# 4
	126115	1237680247351542037	0.599028432933	20.6829569705	shred galaxy <semiauto-found>
	126115	1237680247351542039	0.599304111176	20.6816607634	shred galaxy <semiauto-found>
NSA126115 <enter to continue>

Write out the catalog


In [43]:
generated_cats = {}
for h in hosts_to_target:
    print('On host', h.name)
    sys.stdout.flush()

    cat = joined_catalogs[h]
    ranks = rankdct[h]
    
    fnout = 'mmthecto/{0}_nov2015.cat'.format(h.name)
    fluxfnout = fnout.replace('.cat', '.fluxstars')
    print('Going to write', fnout, fluxfnout)
    msk = (cat['RHOST_ARCM']<40) & (ranks>0) & (ranks<10)
    generated_cats[h] = gcat = mmthecto.generate_catalog(h, cat[msk], ranks[msk], 
                                         repeatflux=3, removefluxdistance=20*u.arcsec,
                                         fnout=fnout, fluxfnout=fluxfnout)


On host NSA150887
Going to write mmthecto/NSA150887_nov2015.cat mmthecto/NSA150887_nov2015.fluxstars
Including 4565 targets
Found 73 Flux stars
Removing 12 Flux stars too close to program stars
Could not find psf mags so falling back on regular mags...
Found 255 guide stars
On host NSA150578
Going to write mmthecto/NSA150578_nov2015.cat mmthecto/NSA150578_nov2015.fluxstars
Including 3522 targets
Found 57 Flux stars
Removing 6 Flux stars too close to program stars
Could not find psf mags so falling back on regular mags...
Found 248 guide stars
On host OBrother
Going to write mmthecto/OBrother_nov2015.cat mmthecto/OBrother_nov2015.fluxstars
Including 4499 targets
Found 110 Flux stars
Removing 13 Flux stars too close to program stars
Could not find psf mags so falling back on regular mags...
Found 282 guide stars
On host NSA150307
Going to write mmthecto/NSA150307_nov2015.cat mmthecto/NSA150307_nov2015.fluxstars
Including 3696 targets
Found 99 Flux stars
Removing 8 Flux stars too close to program stars
Could not find psf mags so falling back on regular mags...
Found 295 guide stars
On host AnaK
Going to write mmthecto/AnaK_nov2015.cat mmthecto/AnaK_nov2015.fluxstars
Including 4872 targets
Found 79 Flux stars
Removing 15 Flux stars too close to program stars
Could not find psf mags so falling back on regular mags...
Found 72 guide stars
On host NSA149977
Going to write mmthecto/NSA149977_nov2015.cat mmthecto/NSA149977_nov2015.fluxstars
Including 3107 targets
Found 128 Flux stars
Removing 8 Flux stars too close to program stars
Could not find psf mags so falling back on regular mags...
Found 397 guide stars
On host NSA153017
Going to write mmthecto/NSA153017_nov2015.cat mmthecto/NSA153017_nov2015.fluxstars
Including 2653 targets
Found 64 Flux stars
Removing 3 Flux stars too close to program stars
Could not find psf mags so falling back on regular mags...
Found 292 guide stars
On host NSA129387
Going to write mmthecto/NSA129387_nov2015.cat mmthecto/NSA129387_nov2015.fluxstars
Including 4990 targets
Found 60 Flux stars
Removing 7 Flux stars too close to program stars
Could not find psf mags so falling back on regular mags...
Found 136 guide stars
On host NSA129237
Going to write mmthecto/NSA129237_nov2015.cat mmthecto/NSA129237_nov2015.fluxstars
Including 3349 targets
Found 49 Flux stars
Removing 4 Flux stars too close to program stars
Could not find psf mags so falling back on regular mags...
Found 155 guide stars
On host Narnia
Going to write mmthecto/Narnia_nov2015.cat mmthecto/Narnia_nov2015.fluxstars
Including 5099 targets
Found 46 Flux stars
Removing 11 Flux stars too close to program stars
Could not find psf mags so falling back on regular mags...
Found 107 guide stars
On host NSA126115
Going to write mmthecto/NSA126115_nov2015.cat mmthecto/NSA126115_nov2015.fluxstars
Including 3662 targets
Found 86 Flux stars
Removing 8 Flux stars too close to program stars
Could not find psf mags so falling back on regular mags...
Found 303 guide stars

Try to fix up 0-crossing of NSA153017


In [79]:
fncat = 'mmthecto/NSA153017_nov2015.cat'
fnflux = 'mmthecto/NSA153017_nov2015.fluxstars'

cat153017 = Table.read(fncat, format='ascii', data_start=2)
flux153017 = Table.read(fnflux, format='ascii')

catra = Angle(cat153017['ra'], u.hour).to(u.deg)
catdec = Angle(cat153017['dec'], u.deg)
fluxra = Angle(flux153017['RA'], u.deg)
fluxdec = Angle(flux153017['DEC'], u.deg)

msk = cat153017['type']=='guide'

plt.scatter(catra[msk], catdec[msk], alpha=.5,lw=0)
plt.scatter(catra[~msk], catdec[~msk], alpha=.5,lw=0, c='g')
plt.scatter(fluxra, fluxdec, alpha=.5,lw=0, c='r')


Out[79]:
<matplotlib.collections.PathCollection at 0x157e09b10>

In [89]:
msk = cat153017['type']=='guide'

plt.scatter(catra[msk].wrap_at(180*u.deg), catdec[msk].wrap_at(180*u.deg), alpha=.5,lw=0)
plt.scatter(catra[~msk].wrap_at(180*u.deg), catdec[~msk].wrap_at(180*u.deg), alpha=.5,lw=0, c='g')
plt.scatter(fluxra.wrap_at(180*u.deg), fluxdec.wrap_at(180*u.deg), alpha=.5,lw=0, c='r')


Out[89]:
<matplotlib.collections.PathCollection at 0x15a84f050>

Whoopsie - the problem is in RHOST_ARCM/RHOST_KPC - need to fix that up for the 0-crossing


In [93]:
hostnames_to_target = [153017]
hosts_to_target = []
for hostname in hostnames_to_target:
    if isinstance(hostname, basestring):
        for host in candhosts:
            if host.name == hostname:
                hosts_to_target.append(host)
                break
        else:
            raise ValueError('could not find host named {0}'.format(hostname))
    else:
        for host in candhosts:
            if host.nsaid == hostname:
                hosts_to_target.append(host)
                break                
        else:
            raise ValueError('could not find host with nsa number {0}'.format(hostname))
hosts_to_target


Out[93]:
[<hosts.NSAHost object w/ name 'NSA153017'>]

In [94]:
joined_catalogs = {}
for h in hosts_to_target:
    #filter on manual remove lists *and* REMOVE!=-1
    #cat = targeting.remove_targets_with_remlist(h.get_sdss_catalog(), h, 'catalogs/TargetRemoveNov8_2015.csv')
    cat = targeting.remove_targets_with_remlist(h.get_sdss_catalog(), h, 'catalogs/TargetRemoveNov11_2015.csv')
    
    pre_joined = table.join(cat, mlpred, join_type='left', keys='objID')
    
    tokeep = pre_joined['REMOVE']==-1
    notobserved = pre_joined['ZQUALITY']<3
    print('removed', np.sum(~tokeep),'for remove list and', np.sum(~notobserved),'b/c observed')
    joined_catalogs[h] = pre_joined[tokeep&notobserved]
    highp = pre_joined[p_column_for_ranking]>0.01
    print('n w/ ML pred>0.01 and observed:', np.sum(highp&~notobserved), 'not yet obs:', np.sum(highp&notobserved))


Removed 24 objects for NSA153017
removed 89 for remove list and 3 b/c observed
n w/ ML pred>0.01 and observed: 2 not yet obs: 447

In [96]:
redmappertargs = table.Table.read('catalogs/redmagic_targets_0828.fit')

In [118]:
# compute ranks, which includes filtering on mag
rankdct = {}
for h in hosts_to_target:
    cat = joined_catalogs[h]
    
    # need to fix up distance for 0-crossing
    hc3d = SkyCoord(h.coords.ra, h.coords.dec, h.dist)
    sc = SkyCoord(cat['ra']*u.deg, cat['dec']*u.deg, distance=h.dist)
    cat['RHOST_ARCM'] = sc.separation(hc3d).to(u.arcmin).value
    cat['RHOST_KPC'] = sc.separation_3d(hc3d).to(u.kpc).value
    
    highprob = cat[p_column_for_ranking].filled(0) > 0.5
    medprob = (cat[p_column_for_ranking].filled(0) > 0.05) & ~highprob
    lowprob = (cat[p_column_for_ranking].filled(0) > 0.01) & ~highprob & ~medprob
    inside = cat['RHOST_KPC']<300
    
    rankdct[h] = ranks = np.ones(len(cat), dtype=int)*-1 #the -1 will be re-set at the end
    
    # rank 1 reserved for flux stars
    ranks[inside&highprob] = 2
    ranks[~inside&highprob] = 3
    ranks[inside&medprob] = 4
    ranks[~inside&medprob] = 5
    ranks[inside&lowprob] = 6
    ranks[~inside&lowprob] = 7
    
    validtarget = (cat['r'] < 21.5) & (cat['fibermag_r']<23) & (cat['phot_sg']=='GALAXY')
    other = ranks < 0
    colorcutmsk = targeting.colorcut_mask(cat,  targeting.bossanova_color_cuts)
    
    ranks[validtarget&other&colorcutmsk] = 9 #other incl. color cuts
    ranks[validtarget&other&~colorcutmsk] = 10 #other not incl. color cuts
    redmappertarg = np.in1d(cat['objID'], redmappertargs['OBJID'])
    rg7 = ranks>7
    ranks[validtarget&redmappertarg&rg7] = 8 #redmapper
    
#informational
for h, ranks in rankdct.items():
    cat = joined_catalogs[h]
    
    hranks = dict(enumerate(np.bincount(ranks[ranks>-1])))
    del hranks[0]
    del hranks[1]
    hranks[-1] = np.sum(ranks==-1)
    
    print(h.name,'rank counts:', hranks)
    
    nother =  np.sum(ranks==8)
    nprob = np.sum((ranks<8)&(ranks>0))
    nother_obs = 270-np.sum((ranks<8)&(ranks>0)&(cat['RHOST_ARCM']<30))
    print('fraction not "other": {0:.1%}, nprob_obs={1}\n'.format(nprob/(nprob+nother), nother_obs))


NSA153017 rank counts: {2: 0, 3: 0, 4: 23, 5: 59, 6: 91, 7: 265, 8: 66, 9: 9011, 10: 6426, -1: 46178}
fraction not "other": 86.9%, nprob_obs=158


In [130]:
reload(mmthecto)
#write out the catalog
generated_cats = {}
for h in hosts_to_target:
    print('On host', h.name)
    sys.stdout.flush()

    cat = joined_catalogs[h]
    ranks = rankdct[h]
    
    fnout = 'mmthecto/{0}_nov2015_fixed.cat'.format(h.name)
    fluxfnout = fnout.replace('.cat', '.fluxstars')
    print('Going to write', fnout, fluxfnout)
    msk = (cat['RHOST_ARCM']<40) & (ranks>0) & (ranks<10)
    generated_cats[h] = gcat = mmthecto.generate_catalog(h, cat[msk], ranks[msk], 
                                         repeatflux=3, removefluxdistance=20*u.arcsec,
                                         fnout=fnout, fluxfnout=fluxfnout)


On host NSA153017
Going to write mmthecto/NSA153017_nov2015_fixed.cat mmthecto/NSA153017_nov2015_fixed.fluxstars
Including 4269 targets
Found 64 Flux stars
Removing 7 Flux stars too close to program stars
Could not find psf mags so falling back on regular mags...
Found 292 guide stars

In [131]:
reload(mmthecto)
#write out the catalog
generated_cats = {}
for h in hosts_to_target:
    print('On host', h.name)
    sys.stdout.flush()

    cat = joined_catalogs[h]
    ranks = rankdct[h]
    
    fnout = 'mmthecto/{0}_nov2015_fixed180.cat'.format(h.name)
    fluxfnout = fnout.replace('.cat', '.fluxstars')
    print('Going to write', fnout, fluxfnout)
    msk = (cat['RHOST_ARCM']<40) & (ranks>0) & (ranks<10)
    generated_cats[h] = gcat = mmthecto.generate_catalog(h, cat[msk], ranks[msk], 
                                         repeatflux=3, removefluxdistance=20*u.arcsec,
                                         fnout=fnout, fluxfnout=fluxfnout, wrapraat=180*u.deg)


On host NSA153017
Going to write mmthecto/NSA153017_nov2015_fixed180.cat mmthecto/NSA153017_nov2015_fixed180.fluxstars
Including 4269 targets
Found 64 Flux stars
Removing 7 Flux stars too close to program stars
Could not find psf mags so falling back on regular mags...
Found 292 guide stars

In [132]:
fncat = 'mmthecto/NSA153017_nov2015_fixed180.cat'
fnflux = 'mmthecto/NSA153017_nov2015_fixed180.fluxstars'

cat153017 = Table.read(fncat, format='ascii', data_start=2)
flux153017 = Table.read(fnflux, format='ascii')

catra = Angle(cat153017['ra'], u.hour).to(u.deg)
catdec = Angle(cat153017['dec'], u.deg)
fluxra = Angle(flux153017['RA'], u.deg)
fluxdec = Angle(flux153017['DEC'], u.deg)

msk = cat153017['type']=='guide'

plt.scatter(catra[msk], catdec[msk], alpha=.5,lw=0)
plt.scatter(catra[~msk], catdec[~msk], alpha=.5,lw=0, c='g')
plt.scatter(fluxra, fluxdec, alpha=.5,lw=0, c='r')


Out[132]:
<matplotlib.collections.PathCollection at 0x16358c090>