In [1]:
from planet4 import clustering, io, markings, stats, plotting
from pathlib import Path

In [39]:
obsid = 'ESP_012889_0985'

In [2]:
db = io.DBManager()

In [3]:
image_names = db.image_names

In [4]:
len(image_names)


Out[4]:
221

In [9]:
sorted(image_names)


Out[9]:
['ESP_011296_0975',
 'ESP_011341_0980',
 'ESP_011348_0950',
 'ESP_011350_0945',
 'ESP_011351_0945',
 'ESP_011370_0980',
 'ESP_011394_0935',
 'ESP_011403_0945',
 'ESP_011404_0945',
 'ESP_011406_0945',
 'ESP_011407_0945',
 'ESP_011408_0930',
 'ESP_011413_0970',
 'ESP_011420_0930',
 'ESP_011422_0930',
 'ESP_011431_0930',
 'ESP_011447_0950',
 'ESP_011448_0950',
 'ESP_011460_0980',
 'ESP_011486_0980',
 'ESP_011491_0985',
 'ESP_011515_0980',
 'ESP_011526_0980',
 'ESP_011544_0985',
 'ESP_011557_0985',
 'ESP_011565_0930',
 'ESP_011606_0930',
 'ESP_011623_0985',
 'ESP_011671_0935',
 'ESP_011680_1055',
 'ESP_011697_0980',
 'ESP_011702_0985',
 'ESP_011711_0930',
 'ESP_011723_0945',
 'ESP_011729_0985',
 'ESP_011737_0980',
 'ESP_011776_0930',
 'ESP_011777_0950',
 'ESP_011792_0980',
 'ESP_011842_0980',
 'ESP_011843_0950',
 'ESP_011856_0930',
 'ESP_011900_0985',
 'ESP_011921_0950',
 'ESP_011931_0945',
 'ESP_011934_0945',
 'ESP_011960_0925',
 'ESP_011961_0935',
 'ESP_012008_0975',
 'ESP_012028_0930',
 'ESP_012038_0930',
 'ESP_012053_0980',
 'ESP_012063_0945',
 'ESP_012076_0945',
 'ESP_012077_0930',
 'ESP_012079_0945',
 'ESP_012081_0930',
 'ESP_012211_0930',
 'ESP_012212_0950',
 'ESP_012225_0980',
 'ESP_012227_0980',
 'ESP_012248_0965',
 'ESP_012251_0935',
 'ESP_012254_1065',
 'ESP_012256_0985',
 'ESP_012264_0980',
 'ESP_012265_0950',
 'ESP_012277_0975',
 'ESP_012290_0945',
 'ESP_012291_0980',
 'ESP_012304_0980',
 'ESP_012314_0965',
 'ESP_012316_0925',
 'ESP_012322_0985',
 'ESP_012344_0950',
 'ESP_012348_0975',
 'ESP_012393_0965',
 'ESP_012436_0980',
 'ESP_012461_0925',
 'ESP_012463_0930',
 'ESP_012467_0985',
 'ESP_012488_0930',
 'ESP_012499_0930',
 'ESP_012501_0945',
 'ESP_012504_0980',
 'ESP_012515_0980',
 'ESP_012527_0925',
 'ESP_012594_0980',
 'ESP_012604_0965',
 'ESP_012607_0980',
 'ESP_012620_0975',
 'ESP_012633_0930',
 'ESP_012634_0930',
 'ESP_012643_0945',
 'ESP_012647_0980',
 'ESP_012648_0930',
 'ESP_012671_0930',
 'ESP_012691_0985',
 'ESP_012693_0950',
 'ESP_012704_0850',
 'ESP_012739_0935',
 'ESP_012744_0985',
 'ESP_012753_0950',
 'ESP_012779_0930',
 'ESP_012805_0980',
 'ESP_012816_0930',
 'ESP_012817_0925',
 'ESP_012819_0930',
 'ESP_012821_0865',
 'ESP_012836_0850',
 'ESP_012838_0950',
 'ESP_012844_0930',
 'ESP_012845_0950',
 'ESP_012854_0945',
 'ESP_012858_0855',
 'ESP_012860_0980',
 'ESP_012871_0980',
 'ESP_012884_0935',
 'ESP_012887_1065',
 'ESP_012889_0985',
 'ESP_012939_0980',
 'ESP_012990_0930',
 'ESP_013016_0935',
 'ESP_013030_0930',
 'ESP_013034_0985',
 'ESP_013049_0950',
 'ESP_013095_0935',
 'ESP_013113_0985',
 'ESP_013260_0950',
 'ESP_020049_0985',
 'ESP_020115_0985',
 'ESP_020128_0985',
 'ESP_020146_0950',
 'ESP_020150_0950',
 'ESP_020163_0980',
 'ESP_020176_0930',
 'ESP_020194_0985',
 'ESP_020202_0935',
 'ESP_020214_0935',
 'ESP_020229_0930',
 'ESP_020242_0945',
 'ESP_020255_0935',
 'ESP_020282_0930',
 'ESP_020294_0945',
 'ESP_020297_0980',
 'ESP_020321_0935',
 'ESP_020322_0930',
 'ESP_020339_0985',
 'ESP_020347_0930',
 'ESP_020357_0950',
 'ESP_020374_0980',
 'ESP_020376_0980',
 'ESP_020386_0925',
 'ESP_020401_0950',
 'ESP_020427_0930',
 'ESP_020476_0950',
 'ESP_020480_0950',
 'ESP_020508_0980',
 'ESP_020519_0935',
 'ESP_020532_0935',
 'ESP_020533_0930',
 'ESP_020558_0930',
 'ESP_020568_0950',
 'ESP_020571_0945',
 'ESP_020597_0925',
 'ESP_020598_0935',
 'ESP_020677_0980',
 'ESP_020678_0930',
 'ESP_020716_0945',
 'ESP_020719_0980',
 'ESP_020731_0930',
 'ESP_020742_0925',
 'ESP_020748_0985',
 'ESP_020756_0980',
 'ESP_020779_0950',
 'ESP_020780_0930',
 'ESP_020782_0945',
 'ESP_020783_0950',
 'ESP_020827_0985',
 'ESP_020875_0980',
 'ESP_020888_0935',
 'ESP_020902_0950',
 'ESP_020927_0945',
 'ESP_020930_0980',
 'ESP_020941_0980',
 'ESP_020953_0925',
 'ESP_020954_0935',
 'ESP_020955_0930',
 'ESP_020959_0985',
 'ESP_021454_0925',
 'ESP_021455_0935',
 'ESP_021460_0985',
 'ESP_021468_0935',
 'ESP_021482_0950',
 'ESP_021483_0930',
 'ESP_021491_0950',
 'ESP_021494_0945',
 'ESP_021495_0935',
 'ESP_021497_0980',
 'ESP_021508_0980',
 'ESP_021520_0925',
 'ESP_021521_0980',
 'ESP_021522_0930',
 'ESP_021526_0985',
 'ESP_021574_0980',
 'ESP_021587_0980',
 'ESP_021605_0985',
 'ESP_021642_0980',
 'ESP_021654_0930',
 'ESP_021671_0985',
 'ESP_021684_0985',
 'ESP_021829_0985',
 'ESP_021903_0930',
 'ESP_021969_0980',
 'ESP_022193_0950',
 'ESP_022260_0935',
 'ESP_022273_0950',
 'ESP_022339_0935',
 'ESP_022379_0930',
 'ESP_022510_0980',
 'ESP_022699_0985']

In [5]:
db.dbname


Out[5]:
'/Users/klay6683/Dropbox/data/planet4/2016-05-29_planet_four_classifications_queryable_cleaned_seasons2and3.h5'

In [6]:
all_image_ids = db.image_ids

In [7]:
len(all_image_idse_ids)


Out[7]:
42904

In [41]:
data = db.get_image_name_markings(obsid)

In [42]:
image_ids = data.image_id.unique()

In [43]:
len(image_ids)


Out[43]:
72

In [44]:
saveroot = Path("/Volumes/Data/planet4/new_pipeline")

In [45]:
saveroot = saveroot / obsid
saveroot.mkdir(exist_ok=True)

In [46]:
cm = clustering.ClusteringManager(db.dbname, scope='planet4', output_dir=saveroot,
                                  min_samples_factor=0.15, do_dynamic_min_samples=True,
                                  quiet=True)

In [47]:
for image_id in image_ids:
    print(image_id)
    cm.cluster_image_id(image_id)


APF000018s
n_classifications: 50
min_samples: 8
APF000017l
n_classifications: 44
min_samples: 7
APF000019j
n_classifications: 34
min_samples: 5
APF0000187
n_classifications: 59
min_samples: 9
APF000019a
n_classifications: 34
min_samples: 5
APF0000197
n_classifications: 61
min_samples: 9
APF0000198
n_classifications: 35
min_samples: 5
APF000017m
n_classifications: 40
min_samples: 6
APF000018h
n_classifications: 32
min_samples: 5
APF000018r
n_classifications: 37
min_samples: 6
APF0000184
n_classifications: 47
min_samples: 7
APF0000185
n_classifications: 38
min_samples: 6
APF000018x
n_classifications: 32
min_samples: 5
APF000018g
n_classifications: 48
min_samples: 7
APF000017v
n_classifications: 32
min_samples: 5
APF000017r
n_classifications: 30
min_samples: 4
APF000018p
n_classifications: 39
min_samples: 6
APF0000195
n_classifications: 62
min_samples: 9
APF000018q
n_classifications: 34
min_samples: 5
APF0000199
n_classifications: 42
min_samples: 6
APF000018f
n_classifications: 44
min_samples: 7
APF000017q
n_classifications: 58
min_samples: 9
APF000018n
n_classifications: 49
min_samples: 7
APF000019e
n_classifications: 41
min_samples: 6
APF0000182
n_classifications: 43
min_samples: 6
APF000017n
n_classifications: 35
min_samples: 5
APF0000189
n_classifications: 39
min_samples: 6
APF000018b
n_classifications: 60
min_samples: 9
APF000019i
n_classifications: 23
min_samples: 3
APF000019f
n_classifications: 39
min_samples: 6
APF0000180
n_classifications: 30
min_samples: 4
APF000018t
n_classifications: 53
min_samples: 8
APF000018l
n_classifications: 46
min_samples: 7
APF0000192
n_classifications: 23
min_samples: 3
APF000018u
n_classifications: 23
min_samples: 3
APF000018z
n_classifications: 15
min_samples: 3
APF000017t
n_classifications: 31
min_samples: 5
APF0000196
n_classifications: 46
min_samples: 7
APF000018m
n_classifications: 11
min_samples: 3
APF000018i
n_classifications: 17
min_samples: 3
APF000019c
n_classifications: 19
min_samples: 3
APF000017y
n_classifications: 38
min_samples: 6
APF000017w
n_classifications: 25
min_samples: 4
APF000018w
n_classifications: 13
min_samples: 3
APF000018v
n_classifications: 11
min_samples: 3
APF000019h
n_classifications: 19
min_samples: 3
APF000019g
n_classifications: 14
min_samples: 3
APF0000190
n_classifications: 20
min_samples: 3
APF000018a
n_classifications: 19
min_samples: 3
APF0000186
n_classifications: 18
min_samples: 3
APF0000183
n_classifications: 17
min_samples: 3
APF000019b
n_classifications: 18
min_samples: 3
APF000017u
n_classifications: 43
min_samples: 6
APF000017s
n_classifications: 25
min_samples: 4
APF000018y
n_classifications: 24
min_samples: 4
APF000017x
n_classifications: 29
min_samples: 4
APF000017p
n_classifications: 32
min_samples: 5
APF000019d
n_classifications: 24
min_samples: 4
APF000018c
n_classifications: 28
min_samples: 4
APF000018k
n_classifications: 13
min_samples: 3
APF0000194
n_classifications: 16
min_samples: 3
APF0000181
n_classifications: 28
min_samples: 4
APF000018d
n_classifications: 12
min_samples: 3
APF000018e
n_classifications: 17
min_samples: 3
APF000017o
n_classifications: 25
min_samples: 4
APF000018o
n_classifications: 24
min_samples: 4
APF000017k
n_classifications: 28
min_samples: 4
APF0000191
n_classifications: 18
min_samples: 3
APF0000193
n_classifications: 25
min_samples: 4
APF000017z
n_classifications: 29
min_samples: 4
APF0000188
n_classifications: 11
min_samples: 3
APF000018j
n_classifications: 24
min_samples: 4

In [54]:
blotchfiles = (saveroot / 'just_clustering').glob('*_blotches.csv')
fanfiles = (saveroot / 'just_clustering').glob('*_fans.csv')

In [55]:
blotches = []
for bf in blotchfiles:
    blotches.append(pd.read_csv(bf))
    
blotches = pd.concat(blotches)

In [56]:
blotches.head()


Out[56]:
x y image_x image_y angle radius_1 radius_2 p1_x p1_y p2_x p2_y p3_x p3_y p4_x p4_y image_id n_members
0 676.071493 71.847369 676.071493 1715.847369 69.810632 145.247168 120.403847 726.199782 164.927801 625.943204 -21.233064 563.065609 113.401630 789.077376 30.293107 APF000017k 20
1 825.322541 448.097167 825.322541 2092.097167 61.654026 110.754146 81.332115 877.908006 345.785141 772.737077 550.409193 753.742419 486.713211 896.902664 409.481122 APF000017k 13
2 281.383774 379.846493 281.383774 2023.846493 28.081339 156.262901 109.101008 419.251441 409.817510 143.516107 349.875475 230.027250 476.104154 332.740297 283.588832 APF000017k 12
3 14.600000 336.200000 14.600000 1980.200000 44.255261 71.404175 53.553131 65.742374 355.449647 -36.542374 316.950353 -22.772386 374.556780 51.972386 297.843220 APF000017k 5
4 111.889478 463.884213 111.889478 2107.884213 41.589543 74.993685 56.245264 167.978698 412.829370 55.800257 514.939057 74.554450 505.951129 149.224505 421.817298 APF000017k 5

In [59]:
%matplotlib nbagg

In [66]:
blotches.image_y.max()


Out[66]:
19819.555555555558

In [71]:
fig, ax = plt.subplots()
blotches.plot(ax=ax, kind='scatter', x='image_x', y='image_y', grid=False)
for x in range(0, 2000, 840):
    ax.axvline(x=x, lw=1)
    ax.axvline(x=x-100, lw=1, ls='--')
    ax.axvline(x=x+100, lw=1, ls='--')
for y in range(0, int(blotches.image_y.max()), 648):
    ax.axhline(y=y, lw=1)
    ax.axhline(y=y-100, lw=1, ls='--')
    ax.axhline(y=y+100, lw=1, ls='--')



In [61]:
blotches.columns


Out[61]:
Index(['x', 'y', 'image_x', 'image_y', 'angle', 'radius_1', 'radius_2', 'p1_x',
       'p1_y', 'p2_x', 'p2_y', 'p3_x', 'p3_y', 'p4_x', 'p4_y', 'image_id',
       'n_members'],
      dtype='object')

In [ ]: