In [ ]:
from planet4 import markings, dbscan, io, fnotching

In [ ]:
from nbtools.logging import setup_live_logging
import logging

In [ ]:
setup_live_logging('planet4', logging.DEBUG)

Data to cluster


In [ ]:
from planet4 import region_data, io

In [ ]:
db = io.DBManager()

In [ ]:
obsids = db.image_names

In [ ]:
db.dbname

In [ ]:
len(obsids)

Support functions


In [ ]:
from planet4.catalog_production import ReleaseManager
from planet4 import fnotching

In [ ]:
rm = ReleaseManager("v1.1")

In [ ]:
rm.launch_catalog_production()

In [ ]:
rm.merge_all()

In [ ]:
pd.read_csv(rm.blotch_merged).shape

In [ ]:
pd.read_csv(rm.fan_merged).shape

In [ ]:
fans = pd.read_csv(self.fan_file)
blotches = pd.read_csv(self.blotch_file)
meta = pd.read_csv(self.metadata_path, dtype='str')
tile_coords = pd.read_csv(self.tile_coords_path, dtype='str')

In [ ]:
meta.shape

In [ ]:
tile_coords.shape

In [ ]:
cols_to_merge = ['OBSERVATION_ID',
                         'SOLAR_LONGITUDE', 'north_azimuth', 'map_scale']
fans = fans.merge(meta[cols_to_merge],
                  left_on='obsid', right_on='OBSERVATION_ID')
blotches = blotches.merge(
    meta[cols_to_merge], left_on='obsid', right_on='OBSERVATION_ID')

In [ ]:
fans.shape

In [ ]:
blotches.shape

In [ ]:
fans.drop(self.DROP_FOR_FANS, axis=1, inplace=True)
blotches.drop(self.DROP_FOR_BLOTCHES, axis=1, inplace=True)

In [ ]:
blotches.shape

In [ ]:
fans, blotches = self.merge_fnotch_results(fans, blotches)

In [ ]:
fans.shape

In [ ]:
blotches.shape

In [ ]:
from planet4 import catalog_production as cp

In [ ]:
rm.EDRINDEX_meta_path

In [ ]:
rm.merge_all()

In [ ]:
rm.calc_metadata()

In [ ]:
meta = pd.read_csv(rm.metadata_path).set_index('OBSERVATION_ID')

In [ ]:
rm.EDRINDEX_meta_path

In [ ]:
meta.loc['ESP_012251_0935']

In [ ]:
edrindex = pd.read_hdf("/Volumes/Data/hirise/EDRCUMINDEX.hdf")
p4_edr = edrindex[edrindex.OBSERVATION_ID.isin(rm.obsids)].query(
    'CCD_NAME=="RED4"').drop_duplicates(subset='OBSERVATION_ID')

In [ ]:
p4_edr.set_index('OBSERVATION_ID', inplace=True)

In [ ]:
p4_edr.loc['ESP_012251_0935']

In [ ]:


In [ ]:
rm.calc_metadata()

In [ ]:
rm.

In [ ]:
df.set_index('OBSERVATION_ID').to_csv(rm.EDRINDEX_meta_path, index=True)

In [ ]:
cols = ['OBSERVATION_ID', 'IMAGE_CENTER_LATITUDE', 'IMAGE_CENTER_LONGITUDE', 'SOLAR_LONGITUDE', 'START_TIME',
        'north_azimuth', '# of tiles']

In [ ]:
df[cols].head()

In [ ]:
fname = "P4_catalog_v1.0_EDRINDEX_metadata.csv"
df.to_csv(rm.savefolder / fname, index=False)

In [ ]:
rm.savefolder

In [ ]:
import itertools
import string 

# Adapt this prefix as a unique identifier of the dataset

def fan_id_generator():
    for newid in itertools.product(string.digits+'abcdef', repeat=6):
        yield 'F' + ''.join(newid)

def blotch_id_generator():
    for newid in itertools.product(string.digits+'abcdef', repeat=6):
        yield 'B' + ''.join(newid)

In [ ]:
def add_marking_ids(path, fan_id, blotch_id):
    """Add marking_ids for catalog to cluster results.
    
    Parameters
    ----------
    path : str, pathlib.Path
        Path to L1A image_id clustering result directory
    fan_id, blotch_id : generator
        Generator for marking_id
    """
    image_id = path.parent.name
    for kind, id_ in zip(['fans', 'blotches'], [fan_id, blotch_id]):
        fname = str(path / f"{image_id}_L1A_{kind}.csv")
        try:
            df = pd.read_csv(fname)
        except FileNotFoundError:
            continue
        else:
            marking_ids = []
            for i in range(df.shape[0]):
                marking_ids.append(next(id_))
            df['marking_id'] = marking_ids
            df.to_csv(fname, index=False)

In [ ]:
from tqdm import tqdm

In [ ]:
obsids = rm.obsids

In [ ]:
rm.launch_catalog_production()

In [ ]:
rm.calc_metadata()

In [ ]:
rm.calc_marking_coordinates()

In [ ]:
rm.merge_all()

In [ ]:
fan_id = fan_id_generator()
blotch_id = blotch_id_generator()

In [ ]:
from planet4.fnotching import fnotch_image_ids, apply_cut

In [ ]:
for obsid in tqdm(obsids):
    pm = io.PathManager(obsid=obsid, datapath=rm.savefolder)
    paths = pm.get_obsid_paths('L1A')
    for path in paths:
        add_marking_ids(path, fan_id, blotch_id)

In [ ]:
rm.catalog

In [ ]:
for obsid in tqdm(obsids):
    fnotch_image_ids(obsid, savedir=rm.catalog)
    apply_cut(obsid, savedir=rm.catalog)

In [ ]:
from planet4.catalog_production import create_roi_file

In [ ]:
create_roi_file(obsids, rm.catalog, rm.catalog)

In [ ]:
rm.get_metadata()

In [ ]:
rm.merge_all()

In [ ]:
fans = rm.read_fan_file()

In [ ]:
fans.columns

In [ ]:
blotches = rm.read_blotch_file()

In [ ]:
blotches.columns

In [ ]:
pd.read_csv(rm.metadata_path).columns

In [ ]:
fans.vote_ratio.isna().value_counts()

In [ ]:
fans.head()

In [ ]:
rm.launch_catalog_production()

Debugging


In [ ]:
from planet4.fnotching import get_clusters_in_path

def find_bad_obsid(obsid, savedir):
    pm = io.PathManager(obsid=obsid, datapath=savedir)
    paths = pm.get_obsid_paths('L1A')
    if len(paths) == 0:
        logger.warning("No paths to fnotch found for %s", obsid)
    fans = []
    blotches = []
    for path in paths:
        f, b = get_clusters_in_path(path)
        fans.append(f)
        blotches.append(b)
    fans = pd.concat(fans, ignore_index=True)

In [ ]:
folder

In [ ]:
for obsid in obsids:
    print(obsid)
    find_bad_obsid(obsid, folder)

In [ ]:
import holoviews as hv
hv.notebook_extension()

In [ ]:
from planet4.io import get_subframe

In [ ]:
obsid = 'ESP_022273_0950'

In [ ]:
data = db.get_image_name_markings(obsid)[['image_id', 'image_url']]

In [ ]:
data.set_index('image_id', drop=True, inplace=True)

In [ ]:
data.drop_duplicates(inplace=True)

In [ ]:
def show_subframe(image_id):
    img = get_subframe(data.loc[data.index[0]].values[0])
    return hv.Raster(img)

In [ ]:
%output size=150

In [ ]:
%%opts Raster [xaxis=None, yaxis=None]
show_subframe(data.index[0])+show_subframe(data.index[1])

In [ ]:
dmap = hv.DynamicMap(show_subframe, kdims=['image_id'])
dmap

In [ ]:
dmap.redim.values(image_id=data.index.values)

In [ ]:
xvals = np.linspace(-4,0,202)
yvals = np.linspace(4,0,202)
xs,ys = np.meshgrid(xvals, yvals)

def waves_image(alpha, beta):
    return hv.Image(np.sin(((ys/alpha)**alpha+beta)*xs))

waves_image(0,0) + waves_image(0,4)

In [ ]:
dmap = hv.DynamicMap(waves_image, kdims=['alpha', 'beta'])
dmap

In [ ]:
dmap[0,1] + dmap.select(alpha=1, beta=2)

In [ ]:
dmap.redim.range(alpha=(0,5.0), beta=(1,5.0))

In [ ]:
hv.__version__

In [ ]:
for imgid in image_ids:
    do_cluster_ids(imgid, "without_large_fan_fnotch_on_p4_coords" )

In [ ]:
obsid = 'ESP_020930_0980'
savedir = 'fnotching_per_obsid'

In [ ]:
dbscanner = dbscan.DBScanner(savedir=savedir)

In [ ]:
dbscanner.cluster_image_name(obsid)

In [ ]:
pm.final_blotchfile

In [ ]:
fnotching.fnotch_obsid(obsid, savedir=savedir)

In [ ]:
fnotching.apply_cut_obsid(obsid, savedir=savedir)

In [ ]:


In [ ]:


In [ ]:
data = region_data.Inca.season2 + region_data.Inca.season3

In [ ]:
do_clustering(ithaca[0])

In [ ]:
ithaca[0]

In [ ]:
from nbtools import execute_in_parallel

In [ ]:
result = execute_in_parallel(do_clustering, data)

Creating obsid files


In [ ]:
from planet4.fnotching import get_clusters_in_path

In [ ]:
pm = io.PathManager(obsid=roi.seasons[0], datapath=folder)

In [ ]:
pm.obsid_final_blotches_path

In [ ]:
from planet4.catalog_production import create_roi_file

In [ ]:
savedir

In [ ]:
create_roi_file(obsids, 'catalog', savedir)

In [ ]:
pm = io.PathManager(obsid=obsids[0], datapath=savedir)

In [ ]:
folders = pm.get_obsid_paths('L1C')

In [ ]:
from planet4.catalog_production import read_csvfiles_into_lists_of_frames, concat_bucket_lists

In [ ]:
bucket.items()

In [ ]:


In [ ]:
roi.available_seasons

In [ ]:
folder

In [ ]:
def create_roi_file_with_seasons(roi, datapath, copy_to_analysis=True):
    Fans = []
    Blotches = []
    for season in roi.available_seasons:
        print(season)
        # keep lower case marking objects to add season column later
        fans = []
        blotches = []
        for obsid in getattr(roi, season):
            print(obsid)
            pm = io.PathManager(obsid=obsid, datapath=datapath)
            fans.append(pm.final_fandf)
            blotches.append(pm.final_blotchdf)

        fans = pd.concat(fans, ignore_index=True)
        fans['season'] = season
        Fans.append(fans)
    
        blotches = pd.concat(blotches, ignore_index=True)
        blotches['season'] = season
        Blotches.append(blotches)
        
    Fans = pd.concat(Fans, ignore_index=True)
    Blotches = pd.concat(Blotches, ignore_index=True)
    savedir = pm.path_so_far.parent
    for marks in ['Fans', 'Blotches']:
        savename = f"{roi.name}_{pm.L1C_folder}_{marks}.csv"
        savepath = savedir / savename
        obj = eval(marks)  # get handle to the fan or blotch object
        obj = obj.dropna(how='all', axis=1)
        obj.to_csv(savepath, index=False)
        if copy_to_analysis is True:
            savefolder = io.analysis_folder() / folder / f"{roi.name.lower()}"
            savefolder.mkdir(exist_ok=True)
            savepath = savefolder / savename
            obj.to_csv(savepath, index=False)

In [ ]:
create_roi_file_with_seasons(roi, folder)

In [ ]:
def create_obsid_result_files(obsid, datapath=None):
#     logger.debug("Working on %s", obsid)
    pm = io.PathManager(obsid=obsid, datapath=datapath)
    # the pm.get_obsid_paths method searches for existing image_id folders inside each
    # obsid folder
    folders = list(pm.get_obsid_paths(pm.L1C_folder))
    blotches = []
    fans = []
    for folder in folders:
        pm.id = fnotching.get_id_from_path(folder)
        if pm.final_blotchfile.exists() and pm.final_blotchfile.lstat().st_size > 1:
            blotches.append(pm.final_blotchdf)
        if pm.final_fanfile.exists() and pm.final_fanfile.lstat().st_size > 1:
            fans.append(pm.final_fandf)

    if len(blotches) > 0:
        blotches = pd.concat(blotches, ignore_index=True)
        blotches.dropna(how='all', axis=1, inplace=True)
        blotches.to_csv(pm.obsid_final_blotches_path, index=False)
        print("Created", pm.obsid_final_blotches_path)

    if len(fans) > 0:
        fans = pd.concat(fans, ignore_index=True)
        fans.dropna(how='all', axis=1, inplace=True)
        fans.to_csv(pm.obsid_final_fans_path, index=False)
        print("Created", pm.obsid_final_fans_path)

In [ ]:
for obsid in data:
    create_obsid_result_files(obsid, datapath='for_chase')

In [ ]:
region = getattr(region_data, name)
for season_no in [2, 3]:
    season = str(season_no)
    bucket = []
    for obsid in getattr(region, f"season{season}"):
        pm = io.PathManager(obsid=obsid)
        df = pd.read_csv(pm.obsid_final_blotches_path)
        df['obsid'] = obsid
        bucket.append(df)

    df = pd.concat(bucket, ignore_index=True)
    df.to_csv(io.analysis_folder() / f"p4_catalog/{name}_season{season}_blotches.csv",
              index=False)

In [ ]:
store_folder = io.analysis_folder() / 'for_chase'
store_folder.mkdir(exist_ok=True)

In [ ]:
name = 'inca'

In [ ]:
p = Path('/Users/klay6683/Dropbox/data/planet4/for_chase/' + name)
for kind in ['blotches', 'fans']:
    bucket = []
    filepaths = p.glob(f'*_{kind}.csv')
    for filepath in filepaths:
        obsid = filepath.name[:15]
        df = pd.read_csv(filepath)
        df['obsid'] = obsid
        bucket.append(df)
    df = pd.concat(bucket, ignore_index=True)
    savedir = store_folder / f'{name}'
    savedir.mkdir(exist_ok=True)
    savepath = savedir / f"{name}_{kind}.csv"
    print(savepath)
    df.to_csv(savepath, index=False)

In [ ]: