In [ ]:

    

cd ~/Dropbox/data/planet4/season23_catalog/


    

In [ ]:

    

blotch_fnames = !ls *_blotches.csv


    

In [ ]:

    

blotches = pd.read_csv(blotch_fnames[0], index_col=0)


    

In [ ]:

    

from planet4 import markings


    

In [ ]:

    

blotch = markings.Blotch(blotches.iloc[0])


    

In [ ]:

    

blotch


    

In [ ]:

    

fan_fnames = !ls *_fans.csv


    

In [ ]:

    

fans = pd.read_csv(fan_fnames[0], index_col=0)
fans.head()


    

In [ ]:

    

fan = markings.Fan(fans.iloc[0])


    

In [ ]:

    

fan.coords


    

In [ ]:

    

fan


    

In [ ]:

    

fan.data


    

In [ ]:

    

fan.store()


    

In [ ]:

    

fnotch_fnames = !ls *_fnotches.csv


    

In [ ]:

    

fnotches = pd.read_csv(fnotch_fnames[0], index_col=0)


    

In [ ]:

    

fnotch = markings.Fnotch.from_dataframe(fnotches.iloc[0])


    

In [ ]:

    

fnotch


    

In [ ]:

    

s = fnotches.iloc[0]


    

In [ ]:

    

fnotchfan = s.filter(regex='fan_')


    

In [ ]:

    

fnotchfan.rename(lambda x: x[4:])


    

In [ ]:

    

fnotch.data


    

In [ ]:

    

fnotch.store()


    

In [ ]:

    

fnotch.store().filter(regex='_[x,y]')


    

In [ ]:

    

from pathlib import Path
old_root = Path('/Users/klay6683/Dropbox/data/planet4/season23_catalog')


    

In [ ]:

    

old_root


    

In [ ]:

    

blotch_fnames = list(old_root.glob('*_fnotches.csv'))


    

In [ ]:

    

newfolder = old_root.parent / 'season2_3_expanded'


    

In [ ]:

    

def get_store(data, cls):
    marking = cls(data)
    return marking.store()


    

In [ ]:

    

def process_blotches(blotches_fname):
    from planet4 import markings
    from pathlib import Path
    p = Path(blotches_fname)
    newp = newfolder / p.with_suffix('.hdf').name
    blotches = pd.read_csv(p.absolute(), index_col=0)
    newdata = blotches.apply(axis='columns', func=get_store,
                             args=(markings.Fnotch.from_dataframe,))
    newdata.to_hdf(str(newp), 'df')
    return newp.name


    

In [ ]:

    

df = process_blotches(list(blotch_fnames)[0])
df


    

In [ ]:

    

from ipyparallel import Client


    

In [ ]:

    

c = Client()


    

In [ ]:

    

lbview = c.load_balanced_view()


    

In [ ]:

    

results = lbview.map_async(process_blotches, blotch_fnames)


    

In [ ]:

    

from iuvs.multitools import nb_progress_display


    

In [ ]:

    

nb_progress_display(results, blotch_fnames)


    

In [ ]:

    

pd.DataFrame(results.result).info()


    

In [ ]: