In [1]:

    

from planet4 import clustering, io, markings, stats, plotting
from pathlib import Path


    

In [39]:

    

obsid = 'ESP_012889_0985'


    

In [2]:

    

db = io.DBManager()


    

In [3]:

    

image_names = db.image_names


    

In [4]:

    

len(image_names)


    

    
Out[4]:





221

In [9]:

    

sorted(image_names)


    

    
Out[9]:





['ESP_011296_0975',
 'ESP_011341_0980',
 'ESP_011348_0950',
 'ESP_011350_0945',
 'ESP_011351_0945',
 'ESP_011370_0980',
 'ESP_011394_0935',
 'ESP_011403_0945',
 'ESP_011404_0945',
 'ESP_011406_0945',
 'ESP_011407_0945',
 'ESP_011408_0930',
 'ESP_011413_0970',
 'ESP_011420_0930',
 'ESP_011422_0930',
 'ESP_011431_0930',
 'ESP_011447_0950',
 'ESP_011448_0950',
 'ESP_011460_0980',
 'ESP_011486_0980',
 'ESP_011491_0985',
 'ESP_011515_0980',
 'ESP_011526_0980',
 'ESP_011544_0985',
 'ESP_011557_0985',
 'ESP_011565_0930',
 'ESP_011606_0930',
 'ESP_011623_0985',
 'ESP_011671_0935',
 'ESP_011680_1055',
 'ESP_011697_0980',
 'ESP_011702_0985',
 'ESP_011711_0930',
 'ESP_011723_0945',
 'ESP_011729_0985',
 'ESP_011737_0980',
 'ESP_011776_0930',
 'ESP_011777_0950',
 'ESP_011792_0980',
 'ESP_011842_0980',
 'ESP_011843_0950',
 'ESP_011856_0930',
 'ESP_011900_0985',
 'ESP_011921_0950',
 'ESP_011931_0945',
 'ESP_011934_0945',
 'ESP_011960_0925',
 'ESP_011961_0935',
 'ESP_012008_0975',
 'ESP_012028_0930',
 'ESP_012038_0930',
 'ESP_012053_0980',
 'ESP_012063_0945',
 'ESP_012076_0945',
 'ESP_012077_0930',
 'ESP_012079_0945',
 'ESP_012081_0930',
 'ESP_012211_0930',
 'ESP_012212_0950',
 'ESP_012225_0980',
 'ESP_012227_0980',
 'ESP_012248_0965',
 'ESP_012251_0935',
 'ESP_012254_1065',
 'ESP_012256_0985',
 'ESP_012264_0980',
 'ESP_012265_0950',
 'ESP_012277_0975',
 'ESP_012290_0945',
 'ESP_012291_0980',
 'ESP_012304_0980',
 'ESP_012314_0965',
 'ESP_012316_0925',
 'ESP_012322_0985',
 'ESP_012344_0950',
 'ESP_012348_0975',
 'ESP_012393_0965',
 'ESP_012436_0980',
 'ESP_012461_0925',
 'ESP_012463_0930',
 'ESP_012467_0985',
 'ESP_012488_0930',
 'ESP_012499_0930',
 'ESP_012501_0945',
 'ESP_012504_0980',
 'ESP_012515_0980',
 'ESP_012527_0925',
 'ESP_012594_0980',
 'ESP_012604_0965',
 'ESP_012607_0980',
 'ESP_012620_0975',
 'ESP_012633_0930',
 'ESP_012634_0930',
 'ESP_012643_0945',
 'ESP_012647_0980',
 'ESP_012648_0930',
 'ESP_012671_0930',
 'ESP_012691_0985',
 'ESP_012693_0950',
 'ESP_012704_0850',
 'ESP_012739_0935',
 'ESP_012744_0985',
 'ESP_012753_0950',
 'ESP_012779_0930',
 'ESP_012805_0980',
 'ESP_012816_0930',
 'ESP_012817_0925',
 'ESP_012819_0930',
 'ESP_012821_0865',
 'ESP_012836_0850',
 'ESP_012838_0950',
 'ESP_012844_0930',
 'ESP_012845_0950',
 'ESP_012854_0945',
 'ESP_012858_0855',
 'ESP_012860_0980',
 'ESP_012871_0980',
 'ESP_012884_0935',
 'ESP_012887_1065',
 'ESP_012889_0985',
 'ESP_012939_0980',
 'ESP_012990_0930',
 'ESP_013016_0935',
 'ESP_013030_0930',
 'ESP_013034_0985',
 'ESP_013049_0950',
 'ESP_013095_0935',
 'ESP_013113_0985',
 'ESP_013260_0950',
 'ESP_020049_0985',
 'ESP_020115_0985',
 'ESP_020128_0985',
 'ESP_020146_0950',
 'ESP_020150_0950',
 'ESP_020163_0980',
 'ESP_020176_0930',
 'ESP_020194_0985',
 'ESP_020202_0935',
 'ESP_020214_0935',
 'ESP_020229_0930',
 'ESP_020242_0945',
 'ESP_020255_0935',
 'ESP_020282_0930',
 'ESP_020294_0945',
 'ESP_020297_0980',
 'ESP_020321_0935',
 'ESP_020322_0930',
 'ESP_020339_0985',
 'ESP_020347_0930',
 'ESP_020357_0950',
 'ESP_020374_0980',
 'ESP_020376_0980',
 'ESP_020386_0925',
 'ESP_020401_0950',
 'ESP_020427_0930',
 'ESP_020476_0950',
 'ESP_020480_0950',
 'ESP_020508_0980',
 'ESP_020519_0935',
 'ESP_020532_0935',
 'ESP_020533_0930',
 'ESP_020558_0930',
 'ESP_020568_0950',
 'ESP_020571_0945',
 'ESP_020597_0925',
 'ESP_020598_0935',
 'ESP_020677_0980',
 'ESP_020678_0930',
 'ESP_020716_0945',
 'ESP_020719_0980',
 'ESP_020731_0930',
 'ESP_020742_0925',
 'ESP_020748_0985',
 'ESP_020756_0980',
 'ESP_020779_0950',
 'ESP_020780_0930',
 'ESP_020782_0945',
 'ESP_020783_0950',
 'ESP_020827_0985',
 'ESP_020875_0980',
 'ESP_020888_0935',
 'ESP_020902_0950',
 'ESP_020927_0945',
 'ESP_020930_0980',
 'ESP_020941_0980',
 'ESP_020953_0925',
 'ESP_020954_0935',
 'ESP_020955_0930',
 'ESP_020959_0985',
 'ESP_021454_0925',
 'ESP_021455_0935',
 'ESP_021460_0985',
 'ESP_021468_0935',
 'ESP_021482_0950',
 'ESP_021483_0930',
 'ESP_021491_0950',
 'ESP_021494_0945',
 'ESP_021495_0935',
 'ESP_021497_0980',
 'ESP_021508_0980',
 'ESP_021520_0925',
 'ESP_021521_0980',
 'ESP_021522_0930',
 'ESP_021526_0985',
 'ESP_021574_0980',
 'ESP_021587_0980',
 'ESP_021605_0985',
 'ESP_021642_0980',
 'ESP_021654_0930',
 'ESP_021671_0985',
 'ESP_021684_0985',
 'ESP_021829_0985',
 'ESP_021903_0930',
 'ESP_021969_0980',
 'ESP_022193_0950',
 'ESP_022260_0935',
 'ESP_022273_0950',
 'ESP_022339_0935',
 'ESP_022379_0930',
 'ESP_022510_0980',
 'ESP_022699_0985']

In [5]:

    

db.dbname


    

    
Out[5]:





'/Users/klay6683/Dropbox/data/planet4/2016-05-29_planet_four_classifications_queryable_cleaned_seasons2and3.h5'

In [6]:

    

all_image_ids = db.image_ids


    

In [7]:

    

len(all_image_idse_ids)


    

    
Out[7]:





42904

In [41]:

    

data = db.get_image_name_markings(obsid)


    

In [42]:

    

image_ids = data.image_id.unique()


    

In [43]:

    

len(image_ids)


    

    
Out[43]:





72

In [44]:

    

saveroot = Path("/Volumes/Data/planet4/new_pipeline")


    

In [45]:

    

saveroot = saveroot / obsid
saveroot.mkdir(exist_ok=True)


    

In [46]:

    

cm = clustering.ClusteringManager(db.dbname, scope='planet4', output_dir=saveroot,
                                  min_samples_factor=0.15, do_dynamic_min_samples=True,
                                  quiet=True)


    

In [47]:

    

for image_id in image_ids:
    print(image_id)
    cm.cluster_image_id(image_id)


    

    




APF000018s
n_classifications: 50
min_samples: 8
APF000017l
n_classifications: 44
min_samples: 7
APF000019j
n_classifications: 34
min_samples: 5
APF0000187
n_classifications: 59
min_samples: 9
APF000019a
n_classifications: 34
min_samples: 5
APF0000197
n_classifications: 61
min_samples: 9
APF0000198
n_classifications: 35
min_samples: 5
APF000017m
n_classifications: 40
min_samples: 6
APF000018h
n_classifications: 32
min_samples: 5
APF000018r
n_classifications: 37
min_samples: 6
APF0000184
n_classifications: 47
min_samples: 7
APF0000185
n_classifications: 38
min_samples: 6
APF000018x
n_classifications: 32
min_samples: 5
APF000018g
n_classifications: 48
min_samples: 7
APF000017v
n_classifications: 32
min_samples: 5
APF000017r
n_classifications: 30
min_samples: 4
APF000018p
n_classifications: 39
min_samples: 6
APF0000195
n_classifications: 62
min_samples: 9
APF000018q
n_classifications: 34
min_samples: 5
APF0000199
n_classifications: 42
min_samples: 6
APF000018f
n_classifications: 44
min_samples: 7
APF000017q
n_classifications: 58
min_samples: 9
APF000018n
n_classifications: 49
min_samples: 7
APF000019e
n_classifications: 41
min_samples: 6
APF0000182
n_classifications: 43
min_samples: 6
APF000017n
n_classifications: 35
min_samples: 5
APF0000189
n_classifications: 39
min_samples: 6
APF000018b
n_classifications: 60
min_samples: 9
APF000019i
n_classifications: 23
min_samples: 3
APF000019f
n_classifications: 39
min_samples: 6
APF0000180
n_classifications: 30
min_samples: 4
APF000018t
n_classifications: 53
min_samples: 8
APF000018l
n_classifications: 46
min_samples: 7
APF0000192
n_classifications: 23
min_samples: 3
APF000018u
n_classifications: 23
min_samples: 3
APF000018z
n_classifications: 15
min_samples: 3
APF000017t
n_classifications: 31
min_samples: 5
APF0000196
n_classifications: 46
min_samples: 7
APF000018m
n_classifications: 11
min_samples: 3
APF000018i
n_classifications: 17
min_samples: 3
APF000019c
n_classifications: 19
min_samples: 3
APF000017y
n_classifications: 38
min_samples: 6
APF000017w
n_classifications: 25
min_samples: 4
APF000018w
n_classifications: 13
min_samples: 3
APF000018v
n_classifications: 11
min_samples: 3
APF000019h
n_classifications: 19
min_samples: 3
APF000019g
n_classifications: 14
min_samples: 3
APF0000190
n_classifications: 20
min_samples: 3
APF000018a
n_classifications: 19
min_samples: 3
APF0000186
n_classifications: 18
min_samples: 3
APF0000183
n_classifications: 17
min_samples: 3
APF000019b
n_classifications: 18
min_samples: 3
APF000017u
n_classifications: 43
min_samples: 6
APF000017s
n_classifications: 25
min_samples: 4
APF000018y
n_classifications: 24
min_samples: 4
APF000017x
n_classifications: 29
min_samples: 4
APF000017p
n_classifications: 32
min_samples: 5
APF000019d
n_classifications: 24
min_samples: 4
APF000018c
n_classifications: 28
min_samples: 4
APF000018k
n_classifications: 13
min_samples: 3
APF0000194
n_classifications: 16
min_samples: 3
APF0000181
n_classifications: 28
min_samples: 4
APF000018d
n_classifications: 12
min_samples: 3
APF000018e
n_classifications: 17
min_samples: 3
APF000017o
n_classifications: 25
min_samples: 4
APF000018o
n_classifications: 24
min_samples: 4
APF000017k
n_classifications: 28
min_samples: 4
APF0000191
n_classifications: 18
min_samples: 3
APF0000193
n_classifications: 25
min_samples: 4
APF000017z
n_classifications: 29
min_samples: 4
APF0000188
n_classifications: 11
min_samples: 3
APF000018j
n_classifications: 24
min_samples: 4

In [54]:

    

blotchfiles = (saveroot / 'just_clustering').glob('*_blotches.csv')
fanfiles = (saveroot / 'just_clustering').glob('*_fans.csv')


    

In [55]:

    

blotches = []
for bf in blotchfiles:
    blotches.append(pd.read_csv(bf))
    
blotches = pd.concat(blotches)


    

In [56]:

    

blotches.head()


    

    
Out[56]:






  

    

      

      
x
      
y
      
image_x
      
image_y
      
angle
      
radius_1
      
radius_2
      
p1_x
      
p1_y
      
p2_x
      
p2_y
      
p3_x
      
p3_y
      
p4_x
      
p4_y
      
image_id
      
n_members
    
  
  

    

      
0
      
676.071493
      
71.847369
      
676.071493
      
1715.847369
      
69.810632
      
145.247168
      
120.403847
      
726.199782
      
164.927801
      
625.943204
      
-21.233064
      
563.065609
      
113.401630
      
789.077376
      
30.293107
      
APF000017k
      
20
    
    

      
1
      
825.322541
      
448.097167
      
825.322541
      
2092.097167
      
61.654026
      
110.754146
      
81.332115
      
877.908006
      
345.785141
      
772.737077
      
550.409193
      
753.742419
      
486.713211
      
896.902664
      
409.481122
      
APF000017k
      
13
    
    

      
2
      
281.383774
      
379.846493
      
281.383774
      
2023.846493
      
28.081339
      
156.262901
      
109.101008
      
419.251441
      
409.817510
      
143.516107
      
349.875475
      
230.027250
      
476.104154
      
332.740297
      
283.588832
      
APF000017k
      
12
    
    

      
3
      
14.600000
      
336.200000
      
14.600000
      
1980.200000
      
44.255261
      
71.404175
      
53.553131
      
65.742374
      
355.449647
      
-36.542374
      
316.950353
      
-22.772386
      
374.556780
      
51.972386
      
297.843220
      
APF000017k
      
5
    
    

      
4
      
111.889478
      
463.884213
      
111.889478
      
2107.884213
      
41.589543
      
74.993685
      
56.245264
      
167.978698
      
412.829370
      
55.800257
      
514.939057
      
74.554450
      
505.951129
      
149.224505
      
421.817298
      
APF000017k
      
5
    
  

In [59]:

    

%matplotlib nbagg


    

In [66]:

    

blotches.image_y.max()


    

    
Out[66]:





19819.555555555558

In [71]:

    

fig, ax = plt.subplots()
blotches.plot(ax=ax, kind='scatter', x='image_x', y='image_y', grid=False)
for x in range(0, 2000, 840):
    ax.axvline(x=x, lw=1)
    ax.axvline(x=x-100, lw=1, ls='--')
    ax.axvline(x=x+100, lw=1, ls='--')
for y in range(0, int(blotches.image_y.max()), 648):
    ax.axhline(y=y, lw=1)
    ax.axhline(y=y-100, lw=1, ls='--')
    ax.axhline(y=y+100, lw=1, ls='--')


    

In [61]:

    

blotches.columns


    

    
Out[61]:





Index(['x', 'y', 'image_x', 'image_y', 'angle', 'radius_1', 'radius_2', 'p1_x',
       'p1_y', 'p2_x', 'p2_y', 'p3_x', 'p3_y', 'p4_x', 'p4_y', 'image_id',
       'n_members'],
      dtype='object')

In [ ]: