In [ ]:
from planet4 import region_data, io, catalog_production
import pvl
from hirise_tools import products, downloads, labels
from osgeo import gdal
In [ ]:
regions = ['Inca', 'Ithaca', 'Giza', 'Manhattan2', 'Starburst', 'Oswego_Edge',
'Maccelsfield', 'BuenosAires', 'Potsdam']
In [ ]:
seasons = ['season2', 'season3']
In [ ]:
def get_fraction_of_black_pixels(savepath):
ds = gdal.Open(str(savepath))
data = ds.ReadAsArray()
fractions = []
for band in data:
nonzeros = band.nonzero()[0]
fractions.append((band.size - nonzeros.size)/band.size)
return np.array(fractions).mean()
In [ ]:
def get_labelpath(obsid):
prodid = products.PRODUCT_ID(obsid)
prodid.kind = 'COLOR'
labelpath = downloads.labels_root() / prodid.label_fname
return labelpath
def get_p4_hirise_label(obsid):
labelpath = get_labelpath(obsid)
if not labelpath.exists():
downloads.get_rdr_color_label(obsid)
return labels.HiRISE_Label(labelpath)
def calc_real_area(savepath):
black_fraction = get_fraction_of_black_pixels(savepath)
all_area = label.line_samples*label.lines * label.map_scale**2
real_area = (1-black_fraction)*all_area
return real_area
def read_metadata(obsid):
label = get_p4_hirise_label(obsid)
d = dict(obsid=obsid, binning=label.binning,
l_s=label.l_s, line_samples=label.line_samples,
lines=label.lines, map_scale=label.map_scale)
# invalids=black_fraction, real_area=real_area)
# self calculated north azimuths
# folder = io.get_ground_projection_root()
# path = folder / obsid / f"{obsid}_campt_out.csv"
# df = pd.read_csv(path)
# d['north_azimuth'] = df.NorthAzimuth.median()
return d
In [ ]:
obsid = 'ESP_011422_0930'
In [ ]:
lbl = get_p4_hirise_label(obsid)
In [ ]:
from planetpy.pdstools import indices
In [ ]:
lbl = indices.IndexLabel("/Volumes/Data/hirise/EDRCUMINDEX.LBL")
In [ ]:
edrindex = pd.read_hdf("/Volumes/Data/hirise/EDRCUMINDEX.hdf")
In [ ]:
pd.set_option('display.max_columns', 100)
pd.set_option('display.max_rows', 100)
In [ ]:
rm = catalog_production.ReleaseManager('v1.0')
In [ ]:
obsids = rm.obsids
In [ ]:
p4_edr = edrindex[edrindex.OBSERVATION_ID.isin(obsids)].query('CCD_NAME=="RED4"').drop_duplicates(subset='OBSERVATION_ID')
In [ ]:
p4_edr.LINE_SAMPLES.value_counts()
In [ ]:
p4_edr.BINNING.value_counts()
In [ ]:
[i for i in p4_edr.columns if 'SCALE' in i]
In [ ]:
p4_edr.SCALED_PIXEL_WIDTH.round(decimals=1).value_counts()
In [ ]:
%matplotlib ipympl
In [ ]:
plt.figure()
p4_edr.query('BINNING==2').SCALED_PIXEL_WIDTH.hist(bins=50)
In [ ]:
from planet4 import metadata
In [ ]:
NAs = metadata.get_north_azimuths_from_SPICE(obsids)
In [ ]:
NAs.head()
In [ ]:
p4_edr = p4_edr.set_index('OBSERVATION_ID').join(NAs.set_index('OBSERVATION_ID'))
In [ ]:
p4_edr.head()
In [ ]:
db = io.DBManager()
In [ ]:
all_data = db.get_all()
In [ ]:
no_of_tiles_per_obsid = all_data.groupby('image_name').image_id.nunique()
In [ ]:
p4_edr = p4_edr.join(no_of_tiles_per_obsid)
In [ ]:
p4_edr.head()
In [ ]:
p4_edr.rename(dict(image_id="# of tiles"), axis=1,inplace=True)
In [ ]:
p4_edr.head()
In [ ]:
p4_edr.to_csv(rm.EDRINDEX_meta_path)
In [ ]:
done_obsids = pd.read_csv("/Users/klay6683/local_data/planet4/done_obsids_n_tiles.csv",
index_col=0)
In [ ]:
done_obsids.head()
In [ ]:
obsids = done_obsids.image_name.unique()
In [ ]:
%matplotlib ipympl
In [ ]:
done_obsids.groupby('image_name').size().hist(bins=50)
In [ ]:
mr = MetadataReader(obsids[0])
In [ ]:
mr.data_dic
In [ ]:
read_metadata(obsids[0])
In [ ]:
from pathlib import Path
In [ ]:
catalog = 'catalog_1.0b3'
In [ ]:
from tqdm import tqdm
metadata = []
for img in tqdm(obsids):
metadata.append(read_metadata(img))
df = pd.DataFrame(metadata)
In [ ]:
df.head()
In [ ]:
savefolder = io.data_root / catalog
In [ ]:
savepath = savefolder / "all_metadata.csv"
In [ ]:
savepath
In [ ]:
list(savefolder.glob("*.csv"))
In [ ]:
df.to_csv(savepath, index=False)
In [ ]:
name = "{}_metadata.csv".format(region.lower())
folder = io.analysis_folder() / catalog / region.lower()
folder.mkdir(exist_ok=True, parents=True)
fname = folder / name
pd.concat(bucket, ignore_index=True).to_csv(str(fname), index=False)
print("Created", fname)
In [ ]: