notebook.community

Edit and run



In [1]:

    
%pylab inline
from astropy.io import fits as pyfits
from astropy.table import Table,join
import numpy as np
from astropy.io.fits import Column
from datetime import datetime
import matplotlib.pyplot as plt
import requests

warnings.filterwarnings('ignore', category=RuntimeWarning, append=True)
warnings.filterwarnings('ignore', category=UserWarning, append=True);









    



Populating the interactive namespace from numpy and matplotlib



In [2]:

    
# Load data from Dropbox folder instead of clogging up Github

def download_from_dropbox(url):
    
    local_filename = "../{:}".format(url.split("/")[-1].split("?")[0])
    r = requests.get(url, stream=True)
    with open(local_filename, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024): 
            if chunk: # filter out keep-alive new chunks
                f.write(chunk)
                f.flush()
            
    return local_filename



In [15]:

    
# 3 tables:
# Table 1 - gz_hst_table_raw_and_weighted_votes.fits; contains all the vote data except for goods_full, but I hope to add that as soon as I get that data.
#votes_fname = download_from_dropbox("https://www.dropbox.com/s/drdl6mht4kh2d7m/gz_hst_table_raw_and_weighted_votes.fits?dl=1")
votes_fname = '/home/mel/Dropbox/gzhubble/hubble_files/catalog_debiasing_files/new_sb_method/gz_hst_table_raw_and_weighted_votes.fits'
votes_data = Table.read(votes_fname)

# Table 2 - lo_hi_limits.fits: just need low, high limits on p_features (2 columns). 
#lohi_fname = download_from_dropbox("https://www.dropbox.com/s/n1zxkqe3q9rx2ve/lo_hi_limits.fits?dl=1")
lohi_fname = '/home/mel/Dropbox/gzhubble/hubble_files/catalog_debiasing_files/new_sb_method/lo_hi_limits.fits'
lohi_data = Table.read(lohi_fname)

#Table 3 - gzh_t01_z_mu_categorized.fits, want z, mu, and category information (4 columns)
#meta_fname = download_from_dropbox("https://www.dropbox.com/s/hppphpzujretywn/gzh_t01_z_mu_categorized.fits?dl=1")
meta_fname = '/home/mel/Dropbox/gzhubble/hubble_files/catalog_debiasing_files/new_sb_method/gzh_t01_z_mu_categorized.fits'
meta_data = Table.read(meta_fname)



In [16]:

    
lohi_data.remove_column('Z_BEST')
lohi_data.remove_column('GZ_MU_I')
lohi_data.remove_column('OBJNO')
lohi_data.remove_column('survey_id')
lohi_data.remove_column('Table')
lohi_data.remove_column('t01_smooth_or_features_a02_features_or_disk_weighted_fraction')

meta_data.remove_column('OBJNO')
meta_data.remove_column('survey_id')
meta_data.remove_column('Table')
meta_data.remove_column('Correctable_Category')
meta_data.remove_column('t01_smooth_or_features_a02_features_or_disk_weighted_fraction')



In [17]:

    
#Join Table 2 and Table 3- will have z, mu, low/hi limits on p_features
lohi_meta = join(meta_data, lohi_data, keys='zooniverse_id',join_type='left')



In [18]:

    
meta_data[0]









    Out[18]:




<Row index=0>

zooniverse_id Z_BEST GZ_MU_I correction_type
str10 float64 float64 float64
AHZ2000001 0.885999977589 19.7407632791 0.0



In [19]:

    
votes_data[0]









    Out[19]:




<Row index=0 masked=True>

zooniverse_id survey_id Table RA DEC imaging total_count total_weight t01_smooth_or_features_a01_smooth_fraction t01_smooth_or_features_a01_smooth_weighted_fraction t01_smooth_or_features_a02_features_or_disk_fraction t01_smooth_or_features_a02_features_or_disk_weighted_fraction t01_smooth_or_features_a03_star_or_artifact_fraction t01_smooth_or_features_a03_star_or_artifact_weighted_fraction t01_smooth_or_features_total_count t01_smooth_or_features_total_weight t02_edgeon_a01_yes_fraction t02_edgeon_a01_yes_weighted_fraction t02_edgeon_a02_no_fraction t02_edgeon_a02_no_weighted_fraction t02_edgeon_total_count t02_edgeon_total_weight t03_bar_a01_bar_fraction t03_bar_a01_bar_weighted_fraction t03_bar_a02_no_bar_fraction t03_bar_a02_no_bar_weighted_fraction t03_bar_total_count t03_bar_total_weight t04_spiral_a01_spiral_fraction t04_spiral_a01_spiral_weighted_fraction t04_spiral_a02_no_spiral_fraction t04_spiral_a02_no_spiral_weighted_fraction t04_spiral_total_count t04_spiral_total_weight t05_bulge_prominence_a01_no_bulge_fraction t05_bulge_prominence_a01_no_bulge_weighted_fraction t05_bulge_prominence_a02_just_noticeable_fraction t05_bulge_prominence_a02_just_noticeable_weighted_fraction t05_bulge_prominence_a03_obvious_fraction t05_bulge_prominence_a03_obvious_weighted_fraction t05_bulge_prominence_a04_dominant_fraction t05_bulge_prominence_a04_dominant_weighted_fraction t05_bulge_prominence_total_count t05_bulge_prominence_total_weight t06_odd_a01_yes_fraction t06_odd_a01_yes_weighted_fraction t06_odd_a02_no_fraction t06_odd_a02_no_weighted_fraction t06_odd_total_count t06_odd_total_weight t07_rounded_a01_completely_round_fraction t07_rounded_a01_completely_round_weighted_fraction t07_rounded_a02_in_between_fraction t07_rounded_a02_in_between_weighted_fraction t07_rounded_a03_cigar_shaped_fraction t07_rounded_a03_cigar_shaped_weighted_fraction t07_rounded_total_count t07_rounded_total_weight t08_odd_feature_a01_ring_fraction t08_odd_feature_a01_ring_weighted_fraction t08_odd_feature_a02_lens_or_arc_fraction t08_odd_feature_a02_lens_or_arc_weighted_fraction t08_odd_feature_a03_disturbed_fraction t08_odd_feature_a03_disturbed_weighted_fraction t08_odd_feature_a04_irregular_fraction t08_odd_feature_a04_irregular_weighted_fraction t08_odd_feature_a05_other_fraction t08_odd_feature_a05_other_weighted_fraction t08_odd_feature_a06_merger_fraction t08_odd_feature_a06_merger_weighted_fraction t08_odd_feature_a07_dust_lane_fraction t08_odd_feature_a07_dust_lane_weighted_fraction t08_odd_feature_total_count t08_odd_feature_total_weight t09_bulge_shape_a01_rounded_fraction t09_bulge_shape_a01_rounded_weighted_fraction t09_bulge_shape_a02_boxy_fraction t09_bulge_shape_a02_boxy_weighted_fraction t09_bulge_shape_a03_no_bulge_fraction t09_bulge_shape_a03_no_bulge_weighted_fraction t09_bulge_shape_total_count t09_bulge_shape_total_weight t10_arms_winding_a01_tight_fraction t10_arms_winding_a01_tight_weighted_fraction t10_arms_winding_a02_medium_fraction t10_arms_winding_a02_medium_weighted_fraction t10_arms_winding_a03_loose_fraction t10_arms_winding_a03_loose_weighted_fraction t10_arms_winding_total_count t10_arms_winding_total_weight t11_arms_number_a01_1_fraction t11_arms_number_a01_1_weighted_fraction t11_arms_number_a02_2_fraction t11_arms_number_a02_2_weighted_fraction t11_arms_number_a03_3_fraction t11_arms_number_a03_3_weighted_fraction t11_arms_number_a04_4_fraction t11_arms_number_a04_4_weighted_fraction t11_arms_number_a05_more_than_4_fraction t11_arms_number_a05_more_than_4_weighted_fraction t11_arms_number_a06_cant_tell_fraction t11_arms_number_a06_cant_tell_weighted_fraction t11_arms_number_total_count t11_arms_number_total_weight t12_clumpy_a01_yes_fraction t12_clumpy_a01_yes_weighted_fraction t12_clumpy_a02_no_fraction t12_clumpy_a02_no_weighted_fraction t12_clumpy_total_count t12_clumpy_total_weight t13_bright_clump_a01_yes_fraction t13_bright_clump_a01_yes_weighted_fraction t13_bright_clump_a02_no_fraction t13_bright_clump_a02_no_weighted_fraction t13_bright_clump_total_count t13_bright_clump_total_weight t14_bright_clump_central_a01_yes_fraction t14_bright_clump_central_a01_yes_weighted_fraction t14_bright_clump_central_a02_no_fraction t14_bright_clump_central_a02_no_weighted_fraction t14_bright_clump_central_total_count t14_bright_clump_central_total_weight t15_clumps_arrangement_a01_line_fraction t15_clumps_arrangement_a01_line_weighted_fraction t15_clumps_arrangement_a02_chain_fraction t15_clumps_arrangement_a02_chain_weighted_fraction t15_clumps_arrangement_a03_cluster_fraction t15_clumps_arrangement_a03_cluster_weighted_fraction t15_clumps_arrangement_a04_spiral_fraction t15_clumps_arrangement_a04_spiral_weighted_fraction t15_clumps_arrangement_total_count t15_clumps_arrangement_total_weight t16_clumps_count_a02_2_fraction t16_clumps_count_a02_2_weighted_fraction t16_clumps_count_a03_3_fraction t16_clumps_count_a03_3_weighted_fraction t16_clumps_count_a04_4_fraction t16_clumps_count_a04_4_weighted_fraction t16_clumps_count_a05_more_than_4_fraction t16_clumps_count_a05_more_than_4_weighted_fraction t16_clumps_count_a06_cant_tell_fraction t16_clumps_count_a06_cant_tell_weighted_fraction t16_clumps_count_a01_1_fraction t16_clumps_count_a01_1_weighted_fraction t16_clumps_count_total_count t16_clumps_count_total_weight t17_clumps_symmetrical_a01_yes_fraction t17_clumps_symmetrical_a01_yes_weighted_fraction t17_clumps_symmetrical_a02_no_fraction t17_clumps_symmetrical_a02_no_weighted_fraction t17_clumps_symmetrical_total_count t17_clumps_symmetrical_total_weight t18_clumps_embedded_a01_yes_fraction t18_clumps_embedded_a01_yes_weighted_fraction t18_clumps_embedded_a02_no_fraction t18_clumps_embedded_a02_no_weighted_fraction t18_clumps_embedded_total_count t18_clumps_embedded_total_weight
str10 str19 str13 float64 float64 str17 int16 float32 float32 float32 float32 float32 float32 float32 int16 float32 float32 float32 float32 float32 int16 float32 float32 float32 float32 float32 int16 float32 float32 float32 float32 float32 int16 float32 float32 float32 float32 float32 float32 float32 float32 float32 int16 float32 float32 float32 float32 float32 int16 float32 float32 float32 float32 float32 float32 float32 int16 float32 float32 float32 float32 float32 float32 float32 float32 float32 float32 float32 float32 float32 float32 float32 int16 float32 float32 float32 float32 float32 float32 float32 int16 float32 float32 float32 float32 float32 float32 float32 int16 float32 float32 float32 float32 float32 float32 float32 float32 float32 float32 float32 float32 float32 int16 float32 float32 float32 float32 float32 int16 float32 float32 float32 float32 float32 int16 float32 float32 float32 float32 float32 int16 float32 float32 float32 float32 float32 float32 float32 float32 float32 int16 float32 float32 float32 float32 float32 float32 float32 float32 float32 float32 float32 float32 float32 int16 float32 float32 float32 float32 float32 int16 float32 float32 float32 float32 float32 int16 float32
AHZ3000001 50000000 goods_shallow 189.481351 62.2591072 GOODS-N 591 579.243 0.39 0.415 0.382 0.415 0.228 0.17 123 113.241 0.0 0.0 1.0 1.0 11 11.0 0.091 0.091 0.909 0.909 11 11.0 0.0 0.0 1.0 1.0 11 11.0 0.273 0.273 0.182 0.182 0.364 0.364 0.182 0.182 11 11.0 0.442 0.447 0.558 0.553 95 94.001 0.082 0.063 0.898 0.917 0.02 0.021 49 48.001 0.0 0.0 0.048 0.048 0.071 0.071 0.286 0.286 0.476 0.476 0.119 0.119 0.0 0.0 42 42.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0.0 0.766 0.766 0.234 0.234 47 47.0 0.853 0.853 0.147 0.147 34 34.0 0.862 0.862 0.138 0.138 29 29.0 0.136 0.136 0.636 0.636 0.227 0.227 0.0 0.0 22 22.0 0.389 0.389 0.472 0.472 0.0 0.0 0.0 0.0 0.083 0.083 0.056 0.056 36 36.0 0.286 0.286 0.714 0.714 35 35.0 0.571 0.571 0.429 0.429 35 35.0



In [20]:

    
#strip Table names of blank spaces:
for i in range(0,len(votes_data)):
    votes_data['Table'][i]=votes_data['Table'][i].strip()



In [21]:

    
#join Table 1 (votes + Imaging) with the other stuff:
input_table=join(votes_data,lohi_meta,keys='zooniverse_id',join_type='left')



In [22]:

    
len(input_table)









    Out[22]:





179564



In [23]:

    
#Set path for output
fname = '/home/mel/Dropbox/gzhubble/hubble_files/catalog_debiasing_files/new_sb_method/input_for_hubble_debiased_catalog.fits'

input_table.write(fname,overwrite=True)



In [ ]:

zooniverse_id	Z_BEST	GZ_MU_I	correction_type
str10	float64	float64	float64
AHZ2000001	0.885999977589	19.7407632791	0.0

zooniverse_id	survey_id	Table	RA	DEC	imaging	total_count	total_weight	t01_smooth_or_features_a01_smooth_fraction	t01_smooth_or_features_a01_smooth_weighted_fraction	t01_smooth_or_features_a02_features_or_disk_fraction	t01_smooth_or_features_a02_features_or_disk_weighted_fraction	t01_smooth_or_features_a03_star_or_artifact_fraction	t01_smooth_or_features_a03_star_or_artifact_weighted_fraction	t01_smooth_or_features_total_count	t01_smooth_or_features_total_weight	t02_edgeon_a01_yes_fraction	t02_edgeon_a01_yes_weighted_fraction	t02_edgeon_a02_no_fraction	t02_edgeon_a02_no_weighted_fraction	t02_edgeon_total_count	t02_edgeon_total_weight	t03_bar_a01_bar_fraction	t03_bar_a01_bar_weighted_fraction	t03_bar_a02_no_bar_fraction	t03_bar_a02_no_bar_weighted_fraction	t03_bar_total_count	t03_bar_total_weight	t04_spiral_a01_spiral_fraction	t04_spiral_a01_spiral_weighted_fraction	t04_spiral_a02_no_spiral_fraction	t04_spiral_a02_no_spiral_weighted_fraction	t04_spiral_total_count	t04_spiral_total_weight	t05_bulge_prominence_a01_no_bulge_fraction	t05_bulge_prominence_a01_no_bulge_weighted_fraction	t05_bulge_prominence_a02_just_noticeable_fraction	t05_bulge_prominence_a02_just_noticeable_weighted_fraction	t05_bulge_prominence_a03_obvious_fraction	t05_bulge_prominence_a03_obvious_weighted_fraction	t05_bulge_prominence_a04_dominant_fraction	t05_bulge_prominence_a04_dominant_weighted_fraction	t05_bulge_prominence_total_count	t05_bulge_prominence_total_weight	t06_odd_a01_yes_fraction	t06_odd_a01_yes_weighted_fraction	t06_odd_a02_no_fraction	t06_odd_a02_no_weighted_fraction	t06_odd_total_count	t06_odd_total_weight	t07_rounded_a01_completely_round_fraction	t07_rounded_a01_completely_round_weighted_fraction	t07_rounded_a02_in_between_fraction	t07_rounded_a02_in_between_weighted_fraction	t07_rounded_a03_cigar_shaped_fraction	t07_rounded_a03_cigar_shaped_weighted_fraction	t07_rounded_total_count	t07_rounded_total_weight	t08_odd_feature_a01_ring_fraction	t08_odd_feature_a01_ring_weighted_fraction	t08_odd_feature_a02_lens_or_arc_fraction	t08_odd_feature_a02_lens_or_arc_weighted_fraction	t08_odd_feature_a03_disturbed_fraction	t08_odd_feature_a03_disturbed_weighted_fraction	t08_odd_feature_a04_irregular_fraction	t08_odd_feature_a04_irregular_weighted_fraction	t08_odd_feature_a05_other_fraction	t08_odd_feature_a05_other_weighted_fraction	t08_odd_feature_a06_merger_fraction	t08_odd_feature_a06_merger_weighted_fraction	t08_odd_feature_a07_dust_lane_fraction	t08_odd_feature_a07_dust_lane_weighted_fraction	t08_odd_feature_total_count	t08_odd_feature_total_weight	t09_bulge_shape_a01_rounded_fraction	t09_bulge_shape_a01_rounded_weighted_fraction	t09_bulge_shape_a02_boxy_fraction	t09_bulge_shape_a02_boxy_weighted_fraction	t09_bulge_shape_a03_no_bulge_fraction	t09_bulge_shape_a03_no_bulge_weighted_fraction	t09_bulge_shape_total_count	t09_bulge_shape_total_weight	t10_arms_winding_a01_tight_fraction	t10_arms_winding_a01_tight_weighted_fraction	t10_arms_winding_a02_medium_fraction	t10_arms_winding_a02_medium_weighted_fraction	t10_arms_winding_a03_loose_fraction	t10_arms_winding_a03_loose_weighted_fraction	t10_arms_winding_total_count	t10_arms_winding_total_weight	t11_arms_number_a01_1_fraction	t11_arms_number_a01_1_weighted_fraction	t11_arms_number_a02_2_fraction	t11_arms_number_a02_2_weighted_fraction	t11_arms_number_a03_3_fraction	t11_arms_number_a03_3_weighted_fraction	t11_arms_number_a04_4_fraction	t11_arms_number_a04_4_weighted_fraction	t11_arms_number_a05_more_than_4_fraction	t11_arms_number_a05_more_than_4_weighted_fraction	t11_arms_number_a06_cant_tell_fraction	t11_arms_number_a06_cant_tell_weighted_fraction	t11_arms_number_total_count	t11_arms_number_total_weight	t12_clumpy_a01_yes_fraction	t12_clumpy_a01_yes_weighted_fraction	t12_clumpy_a02_no_fraction	t12_clumpy_a02_no_weighted_fraction	t12_clumpy_total_count	t12_clumpy_total_weight	t13_bright_clump_a01_yes_fraction	t13_bright_clump_a01_yes_weighted_fraction	t13_bright_clump_a02_no_fraction	t13_bright_clump_a02_no_weighted_fraction	t13_bright_clump_total_count	t13_bright_clump_total_weight	t14_bright_clump_central_a01_yes_fraction	t14_bright_clump_central_a01_yes_weighted_fraction	t14_bright_clump_central_a02_no_fraction	t14_bright_clump_central_a02_no_weighted_fraction	t14_bright_clump_central_total_count	t14_bright_clump_central_total_weight	t15_clumps_arrangement_a01_line_fraction	t15_clumps_arrangement_a01_line_weighted_fraction	t15_clumps_arrangement_a02_chain_fraction	t15_clumps_arrangement_a02_chain_weighted_fraction	t15_clumps_arrangement_a03_cluster_fraction	t15_clumps_arrangement_a03_cluster_weighted_fraction	t15_clumps_arrangement_a04_spiral_fraction	t15_clumps_arrangement_a04_spiral_weighted_fraction	t15_clumps_arrangement_total_count	t15_clumps_arrangement_total_weight	t16_clumps_count_a02_2_fraction	t16_clumps_count_a02_2_weighted_fraction	t16_clumps_count_a03_3_fraction	t16_clumps_count_a03_3_weighted_fraction	t16_clumps_count_a04_4_fraction	t16_clumps_count_a04_4_weighted_fraction	t16_clumps_count_a05_more_than_4_fraction	t16_clumps_count_a05_more_than_4_weighted_fraction	t16_clumps_count_a06_cant_tell_fraction	t16_clumps_count_a06_cant_tell_weighted_fraction	t16_clumps_count_a01_1_fraction	t16_clumps_count_a01_1_weighted_fraction	t16_clumps_count_total_count	t16_clumps_count_total_weight	t17_clumps_symmetrical_a01_yes_fraction	t17_clumps_symmetrical_a01_yes_weighted_fraction	t17_clumps_symmetrical_a02_no_fraction	t17_clumps_symmetrical_a02_no_weighted_fraction	t17_clumps_symmetrical_total_count	t17_clumps_symmetrical_total_weight	t18_clumps_embedded_a01_yes_fraction	t18_clumps_embedded_a01_yes_weighted_fraction	t18_clumps_embedded_a02_no_fraction	t18_clumps_embedded_a02_no_weighted_fraction	t18_clumps_embedded_total_count	t18_clumps_embedded_total_weight
str10	str19	str13	float64	float64	str17	int16	float32	float32	float32	float32	float32	float32	float32	int16	float32	float32	float32	float32	float32	int16	float32	float32	float32	float32	float32	int16	float32	float32	float32	float32	float32	int16	float32	float32	float32	float32	float32	float32	float32	float32	float32	int16	float32	float32	float32	float32	float32	int16	float32	float32	float32	float32	float32	float32	float32	int16	float32	float32	float32	float32	float32	float32	float32	float32	float32	float32	float32	float32	float32	float32	float32	int16	float32	float32	float32	float32	float32	float32	float32	int16	float32	float32	float32	float32	float32	float32	float32	int16	float32	float32	float32	float32	float32	float32	float32	float32	float32	float32	float32	float32	float32	int16	float32	float32	float32	float32	float32	int16	float32	float32	float32	float32	float32	int16	float32	float32	float32	float32	float32	int16	float32	float32	float32	float32	float32	float32	float32	float32	float32	int16	float32	float32	float32	float32	float32	float32	float32	float32	float32	float32	float32	float32	float32	int16	float32	float32	float32	float32	float32	int16	float32	float32	float32	float32	float32	int16	float32
AHZ3000001	50000000	goods_shallow	189.481351	62.2591072	GOODS-N	591	579.243	0.39	0.415	0.382	0.415	0.228	0.17	123	113.241	0.0	0.0	1.0	1.0	11	11.0	0.091	0.091	0.909	0.909	11	11.0	0.0	0.0	1.0	1.0	11	11.0	0.273	0.273	0.182	0.182	0.364	0.364	0.182	0.182	11	11.0	0.442	0.447	0.558	0.553	95	94.001	0.082	0.063	0.898	0.917	0.02	0.021	49	48.001	0.0	0.0	0.048	0.048	0.071	0.071	0.286	0.286	0.476	0.476	0.119	0.119	0.0	0.0	42	42.0	0.0	0.0	0.0	0.0	0.0	0.0	0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0	0.0	0.766	0.766	0.234	0.234	47	47.0	0.853	0.853	0.147	0.147	34	34.0	0.862	0.862	0.138	0.138	29	29.0	0.136	0.136	0.636	0.636	0.227	0.227	0.0	0.0	22	22.0	0.389	0.389	0.472	0.472	0.0	0.0	0.0	0.0	0.083	0.083	0.056	0.056	36	36.0	0.286	0.286	0.714	0.714	35	35.0	0.571	0.571	0.429	0.429	35	35.0