In [1]:
# At the beginning of the notebook
import logging
import sys
logger = logging.getLogger('planet4')
ch = logging.StreamHandler(sys.stderr)
# fmt = logging.Formatter('%(name)s - %(message)s')
fmt = logging.Formatter('%(message)s')
ch.setFormatter(fmt)
logger.addHandler(ch)
In [2]:
logger.setLevel(logging.DEBUG)
In [3]:
%matplotlib nbagg
In [4]:
from planet4.fnotching import get_clusters_in_path, data_to_centers, cdist, fnotch_image_ids
from planet4 import region_data, dbscan, io, markings
In [5]:
obsid = region_data.Ithaca.season2[-1]
obsid
Out[5]:
In [4]:
scanner = dbscan.DBScanner()
In [5]:
scanner.cluster_image_name(obsid)
In [14]:
pm = io.PathManager('10p')
In [12]:
paths = pm.get_obsid_paths('L1A')
In [13]:
paths[1]
Out[13]:
In [15]:
pm.blotchfile.parent
Out[15]:
In [16]:
fans, blotches = get_clusters_in_path(pm.blotchfile.parent)
In [17]:
blotches
Out[17]:
In [18]:
fans
Out[18]:
In [26]:
from planet4.fnotching import get_id_from_path
In [20]:
from scipy.spatial.distance import pdist
In [27]:
paths = pm.get_obsid_paths('L1A')
In [39]:
pm.id
Out[39]:
In [120]:
import random
In [136]:
random.randint(0,1)
Out[136]:
In [59]:
from IPython.display import display
In [ ]:
for path in paths:
pm.id = get_id_from_path(path)
try:
angle_diffs = pdist(pm.fandf.angle[:, None]) # add dimension to get (n, 1) array.
except FileNotFoundError:
continue
within_180 = np.where(np.abs(angle_diffs - 180) < 5, True, False)
if any(within_180):
print(pm.id)
display(pm.fandf)
In [62]:
pm.id = '139'
In [63]:
fans, blotches = get_clusters_in_path(pm.fanfile.parent)
In [64]:
fans
Out[64]:
In [106]:
fans.loc[[3, 6]]
Out[106]:
In [115]:
distances = pdist(data_to_centers(fans))
distances
Out[115]:
In [211]:
import math
def calc_indices_from_index(n, c):
n= math.ceil(math.sqrt(2* n))
ti= np.triu_indices(n, 1)
return ti[0][c][0], ti[1][c][0]
def fan_opposites(fans, eps=20):
"""Find fans that have opposite orientation and remove lower voted one.
First check if any fans are close enough to be fnotched (same criteria
as blotch-fan fnotching), then check if any of those have opposite orientation.
Delete the one with lower votes.
If number of votes is equal, take a random choice.
Parameters
----------
fans : pd.DataFrame
Fan marking data
eps : int
Returns
-------
pd.DataFrame
Data with opposing fans removed.
"""
distances = pdist(data_to_centers(fans))
close_indices = np.where(distances < eps)[0]
ind_to_remove = []
for index in close_indices:
fan_indices = calc_indices_from_index(len(distances), index)
# use squeeze() to force creation of pd.Series from 1D dataframe
f1 = fans.iloc[fan_indices[0]].squeeze()
f2 = fans.iloc[fan_indices[1]].squeeze()
angle_diff = f1.angle - f2.angle
# if they differ by between 175 and 185:
if abs(angle_diff - 180) < 5:
if f1.n_votes < f2.n_votes:
ind_to_remove.append(fan_indices[0])
elif f1.n_votes > f2.n_votes:
ind_to_remove.append(fan_indices[1])
else:
ind_to_remove.append(fan_indices[random.randint(0, 1)])
return ind_to_remove
In [195]:
pm.obsid
Out[195]:
In [189]:
to_drop = fan_opposites(fans)
In [191]:
to_drop.append(3)
In [192]:
to_drop
Out[192]:
In [193]:
fans.drop(to_drop)
Out[193]:
In [170]:
f1.squeeze().angle - f2.squeeze().angle
Out[170]:
In [141]:
pdist(data_to_centers(fans))
Out[141]:
In [29]:
data_to_centers(fans)
Out[29]:
In [24]:
data_to_centers(blotches)
Out[24]:
In [32]:
pdist(data_to_centers(fans))
Out[32]:
In [25]:
distances = cdist(data_to_centers(fans), data_to_centers(blotches))
distances
Out[25]:
In [32]:
X, Y = np.where(distances < 20)
# X are the indices along the fans input, Y for blotches respectively
# loop over fans and blotches that are within `eps` pixels:
fnotches = []
for fan_loc, blotch_loc in zip(X, Y):
fan = fans.iloc[[fan_loc]]
blotch = blotches.iloc[[blotch_loc]]
fnotches.append(markings.Fnotch(fan, blotch).data)
fnotches = pd.concat(fnotches)
In [33]:
fnotches
Out[33]:
In [10]:
pm = io.PathManager(obsid=obsid)
In [11]:
pm.fnotchfile
Out[11]:
In [212]:
fnotch_image_ids(pm.obsid)
In [ ]:
In [15]:
pm = io.PathManager(obsid=obsid)
In [214]:
from planet4.fnotching import apply_cut, get_slashed_for_path, write_l1c
In [ ]:
apply_cut(obsid)
In [219]:
fnotches = pm.fnotchdf
In [222]:
slashed = fnotches[fnotches.vote_ratio > pm.cut]
In [223]:
slashed
Out[223]:
In [70]:
write_l1c('fan', slashed, pm)
In [226]:
kind = 'blotch'
new_kinds = slashed.loc[[kind]]
new_kinds
Out[226]:
In [227]:
l1c = getattr(pm, f"final_{kind}file")
l1c
Out[227]:
In [228]:
old_kinds = getattr(pm, f"reduced_{kind}df")
old_kinds
Out[228]:
In [230]:
write_l1c('fan', slashed, pm)
In [104]:
from planet4.plotting import plot_finals
In [127]:
plot_finals('bz7', obsid=obsid)
In [120]:
pm = io.PathManager('c0a', obsid=obsid)
In [121]:
all([pm.final_fanfile.exists(),
pm.final_blotchfile.exists()])
Out[121]:
In [122]:
pm.final_blotchfile.parent.parent
Out[122]:
In [20]:
cut
Out[20]:
In [21]:
cut.loc['fan']
Out[21]:
In [22]:
new_fans = cut.loc['fan']
In [23]:
new_fans.to_csv?
In [17]:
paths = pm.get_obsid_paths('L1B')
In [18]:
paths[0]
Out[18]:
In [384]:
id_ = get_id_from_path(paths[0])
id_
Out[384]:
In [7]:
p4id = markings.ImageID('bz7')
In [7]:
%matplotlib inline
In [5]:
scanner = dbscan.DBScanner()
In [6]:
scanner.cluster_and_plot('bz7', 'blotch')
In [11]:
p4id.plot_blotches()
In [12]:
p4id.plot_fans()
In [367]:
pm = io.PathManager('b', obsid=obsid)
In [368]:
pm.blotchfile
Out[368]:
In [369]:
pm.reduced_blotchfile
Out[369]:
In [ ]:
In [68]:
arr1 =np.array([[0,0], [1,1]])
arr2 = np.array([[1,1], [2, 2],[3,3]])
arr2.shape
Out[68]:
In [69]:
cdist(arr1, arr2)
Out[69]:
In [ ]:
In [ ]:
In [1]:
imgid = markings.ImageID('10k')
In [11]:
imgid.plot_blotches(data=blotches)
In [12]:
fig, ax = plt.subplots()
imgid.plot_fans(data=fans, ax=ax)
imgid.plot_blotches(data=blotches,ax=ax)
In [ ]:
@property
def confusion_data(self):
return pd.DataFrame(self.confusion, columns=['image_name', 'kind',
'n_markings',
'n_cluster_members',
'n_rejected'])
def get_newfans_newblotches(self):
logging.debug("Executing get_newfans_newblotches")
df = self.pm.fnotchdf
# check if we got a fnotch dataframe. If not, we assume none were found.
if df is None:
logging.debug("No fnotches found on disk.")
self.newfans = []
self.newblotches = []
return
# apply Fnotch method `get_marking` with given cut.
fnotches = df.apply(markings.Fnotch.from_series, axis=1,
args=('hirise',))
final_clusters = fnotches.apply(lambda x: x.get_marking(self.cut))
def filter_for_fans(x):
if isinstance(x, markings.Fan):
return x
def filter_for_blotches(x):
if isinstance(x, markings.Blotch):
return x
# now need to filter for whatever object was returned by Fnotch.get_marking
self.newfans = final_clusters[
final_clusters.apply(filter_for_fans).notnull()]
self.newblotches = final_clusters[
final_clusters.apply(filter_for_blotches).notnull()]
def apply_fnotch_cut(self, cut=None):
logging.debug("Executing apply_fnotch_cut")
if cut is None:
cut = self.cut
# storage path for the final catalog after applying `cut`
# PathManager self.pm is doing that.
self.pm.get_cut_folder(cut)
self.get_newfans_newblotches()
if len(self.newfans) > 0:
newfans = self.newfans.apply(lambda x: x.store())
try:
completefans = pd.DataFrame(
self.pm.fandf).append(newfans, ignore_index=True)
except OSError:
completefans = newfans
logging.debug("No of fans now: %i" % len(completefans))
else:
logging.debug("Apply fnotch cut: No new fans found.")
completefans = self.pm.fandf
if len(self.newblotches) > 0:
newblotches = self.newblotches.apply(lambda x: x.store())
try:
completeblotches = pd.DataFrame(
self.pm.blotchdf).append(newblotches, ignore_index=True)
except OSError:
completeblotches = newblotches
logging.debug("No of blotches now: %i" % len(completeblotches))
else:
logging.debug('Apply fnotch cut: no blotches survived.')
completeblotches = self.pm.blotchdf
self.save(completefans, self.pm.final_fanfile)
self.save(completeblotches, self.final_blotchfile)
logging.debug("Finished apply_fnotch_cut.")
def save(self, obj, path):
try:
if self.output_format in ['hdf', 'both']:
obj.to_hdf(str(path.with_suffix('.hdf')), 'df')
if self.output_format in ['csv', 'both']:
obj.to_csv(str(path.with_suffix('.csv')), index=False)
# obj could be NoneType if no blotches or fans were found. Catching it here.
except AttributeError:
pass
def save_confusion_data(self, fname):
self.confusion_data.to_csv(fname)
######
# Functions
#####
def calc_fnotch(nfans, nblotches):
"""Calculate the fnotch value (or fan-ness)."""
return (nfans) / (nfans + nblotches)
def gold_star_plotter(gold_id, axis, kind='blotches'):
"""Plot gold data."""
for goldstar, color in zip(markings.gold_members,
markings.gold_plot_colors):
if kind == 'blotches':
gold_id.plot_blotches(user_name=goldstar, ax=axis,
user_color=color)
if kind == 'fans':
gold_id.plot_fans(user_name=goldstar, ax=axis, user_color=color)
markings.gold_legend(axis)
def is_catalog_production_good():
"""A simple quality check for the catalog production."""
from pandas.core.index import InvalidIndexError
db = io.DBManager(io.get_current_database_fname())
not_there = []
invalid_index = []
value_error = []
for image_name in db.image_names:
try:
io.PathManager(image_name)
except InvalidIndexError:
invalid_index.append(image_name)
except ValueError:
value_error.append(image_name)
if len(value_error) == 0 and len(not_there) == 0 and\
len(invalid_index) == 0:
return True
else:
return False
In [ ]: