In [2]:
from planetarypy import io
In [3]:
io.set_database_path("/Users/klay6683/Dropbox/data/planetarypy/")
In [4]:
from planetarypy.pdstools import indices
In [5]:
indices.list_available_index_files()
In [3]:
from planetarypy.pdstools import indices
from planetarypy.utils import download
from tqdm import tqdm
class ProgressBar(tqdm):
"""Provides `update_to(n)` which uses `tqdm.update(delta_n)`."""
def update_to(self, b=1, bsize=1, tsize=None):
"""
b : int, optional
Number of blocks transferred so far [default: 1].
bsize : int, optional
Size of each block (in tqdm units) [default: 1].
tsize : int, optional
Total size (in tqdm units). If [default: None] remains unchanged.
"""
if tsize is not None:
self.total = tsize
self.update(b * bsize - self.n) # will also set self.n = b * bsize
META_URL = 'http://pds-rings.seti.org/metadata'
class META:
INDICES = {'index': 'Cumulative product index of volume series',
'inventory': 'Cumulative list of observed bodies by product',
'moon_summary': 'Cumulative list of observed geometry on moons',
'ring_summary': 'Cumulative list of observed geometry on rings',
'saturn_summary': 'Cumulative list of observed geometry on Saturn'
}
def __init__(self, name=''):
if name == '':
print("Call me with one of the following index names:")
for k,v in self.INDICES.items():
print(k,": ", v)
raise ValueError("Provide index name.")
else:
self._name = name
@property
def name(self):
return self._name
@property
def folder_url(self):
return META_URL + f'/CO{self.id}xxx/CO{self.id}999/'
@property
def meta_filename(self):
return f'CO{self.id}999_{self.name}'
@property
def label_url(self):
return self.folder_url + self.meta_filename + '.lbl'
@property
def table_url(self):
return self.folder_url + self.meta_filename + '.tab'
def download_table(self, local_folder='.'):
baseurl = self.folder_url + self.meta_filename
for ext in ['.lbl', '.tab']:
filename = self.meta_filename + ext
url = self.folder_url + filename
local_path = f"{local_folder}/{filename}"
print("Downloading", local_path)
with ProgressBar(unit='B', unit_scale=True, miniters=1, desc=url) as t:
download(url, local_path, reporthook=t.update_to, data=None)
@property
def label(self):
return indices.IndexLabel(self.meta_filename + '.lbl')
def read_table(self, **kwargs):
return self.label.read_index_data(**kwargs)
class UVIS_META(META):
id = 'UVIS_0'
INDICES = {'index': 'Cumulative product index of volume series',
'supplemental_index': 'Cumulative product index of volume series',
'moon_summary': 'Cumulative list of observed geometry on moons',
'ring_summary': 'Cumulative list of observed geometry on rings',
'saturn_summary': 'Cumulative list of observed geometry on Saturn'
}
class ISS_META(META):
id = 'ISS_2'
class VIMS_META(META):
id = 'VIMS_0'
In [ ]:
meta = UVIS_META('index')
In [ ]:
meta.label_url
In [ ]:
meta.label_url
In [ ]:
lbl = meta.label
In [ ]:
lbl.index_path
In [ ]:
meta = ISS_META('ring_summary')
In [ ]:
meta.download_table()
In [ ]:
meta.read_table??
In [ ]:
index = meta.read_table(convert_times=True)
In [ ]:
index.to_hdf("coiss_index.hdf", 'df')
In [81]:
from planetarypy.pdstools import indices
In [82]:
indices.list_available_index_files()
In [83]:
from pyciss import index
In [84]:
index.download_ring_summary_index()
In [59]:
from nbtools.logging import setup_live_logging
In [60]:
setup_live_logging('planetarypy', 'DEBUG')
Out[60]:
In [66]:
from planetarypy.utils import download
In [67]:
url
Out[67]:
In [70]:
p = Path('./downloads')
p.mkdir()
In [71]:
ret = download(url, p)
In [ ]:
indices.download('cassini:iss:index')
In [4]:
url = 'https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_index.lbl'
# downloading with urllib
# imported the urllib library
import urllib
# Copy a network object to a local file
urllib.request.urlretrieve(url, "python.png")
Out[4]:
In [13]:
label = indices.IndexLabel("/Users/klay6683/Dropbox/data/planetarypy/EDRCUMINDEX.LBL")
In [14]:
df = label.read_index_data()
In [15]:
df.to_hdf("/Users/klay6683/local_data/EDRCUMINDEX.hdf", 'df')
In [ ]:
lab = indices.IndexLabel("/Volumes/USB128II/uvis/COUVIS_0999_index.lbl")
In [ ]:
df = lab.read_index_data(convert_times=False)
In [ ]:
time_cols = [col for col in df.columns if 'TIME' in col]
time_cols
In [ ]:
df = pd.read_hdf("/Volumes/USB128II/uvis/COUVIS_0999_index.hdf")
In [ ]:
df.columns
In [ ]:
df.to_hdf("/Volumes/USB128II/uvis/COUVIS_0999_supplemental_index.hdf", 'df')
In [ ]:
df.START_TIME.head()
In [ ]:
from planetarypy.utils import nasa_datetime_to_iso
In [ ]:
nasa_datetime_to_iso('1999-01-07T16:53:07.953')
In [ ]:
df.to_hdf("/Volumes/USB128II/uvis/COUVIS_0999_index.hdf", 'df')
In [ ]:
df.columns
In [ ]:
df.H_LEVEL.value_counts
In [ ]:
df.TARGET_NAME.value_counts()
In [ ]:
df.OBSERVATION_TYPE.value_counts(dropna=False)
In [ ]:
In [ ]:
df = df.assign(fname=df.FILE_SPECIFICATION_NAME.map(lambda x: x.split('/')[-1][:-4]))
In [ ]:
df.fname.is_unique
In [ ]:
index = pd.read_hdf("/Volumes/Research/pyciss/coiss_index.hdf")
index= index.assign(fname=index.FILE_SPECIFICATION_NAME.map(lambda x: x.split('/')[-1][:-4]))
In [ ]:
rings = index[index.TARGET_DESC.str.contains('Saturn-Rings')]
In [ ]:
dataset = df.merge(rings, on='fname')
In [ ]:
dataset.shape
In [ ]:
dataset[dataset.COARSEST_RADIAL_RESOLUTION<0] = np.nan
In [ ]:
dataset[dataset.FINEST_RADIAL_RESOLUTION < 0] = np.nan
In [ ]:
dataset.COARSEST_RADIAL_RESOLUTION.hist(range=(0,1))
In [ ]:
dataset.FINEST_RADIAL_RESOLUTION.hist(range=(0,1))
In [ ]:
dataset.query("COARSEST_RADIAL_RESOLUTION<0.4").IMAGE_TIME.str[:4].value_counts()
In [ ]:
dataset.query("'N1467345090' in fname")
In [ ]:
index[index.fname.str.contains('N1467345090')].TARGET_NAME
In [ ]:
dataset.dropna(how='all', inplace=True)
In [ ]:
dataset[dataset.fname.str.contains('N1467345090')]
In [ ]:
In [ ]:
img_list = dataset.query("COARSEST_RADIAL_RESOLUTION<0.01")
In [ ]:
img_list.shape
In [ ]:
img_list.TARGET_DESC.value_counts()
In [ ]:
ringdata = index[index.TARGET_DESC.str.contains('ring', case=False)]
In [ ]:
ringdata.TARGET_DESC.value_counts()
In [ ]:
In [ ]:
from pyciss import downloader
In [ ]:
downloader.download_file_id(img_list.fname.iloc[0])
In [ ]:
from tqdm import tqdm
In [ ]:
index['fname'] = index.FILE_SPECIFICATION_NAME.map(lambda x: x.split('/')[-1][:-4])
In [ ]:
index['fname'].head()
In [ ]:
dataset = img_list.merge(index, on='fname')
In [ ]:
In [ ]:
dataset = pd.read_hdf('dataset_metadata.hdf', 'df')
In [ ]:
dataset.filter(regex='TIME').columns
In [ ]:
from pyciss.pipeline import Calibrator
from pyciss import io
In [ ]:
from pysis.exceptions import ProcessError
In [ ]:
def calib_image(fname):
pm = io.PathManager(fname)
if pm.cubepath.exists():
return
calib = Calibrator(fname, final_resolution=100)
try:
calib.standard_calib()
except ProcessError as e:
print(e.stderr)
In [ ]:
dataset.fname.map(calib_image)
In [ ]:
from planetarypy.utils import nasa_datetime_to_iso
In [ ]:
dataset['time'] = pd.to_datetime(dataset.IMAGE_TIME.map(nasa_datetime_to_iso))
In [ ]:
dataset.time
In [ ]:
newer = dataset[pd.DatetimeIndex(dataset.time).year >2014].fname
In [ ]:
from pyciss.ringcube import RingCube
In [ ]:
dataset.TARGET_NAME.value_counts()
In [ ]:
imp
In [ ]:
calib_image(newer.iloc[6])
In [ ]:
cube = RingCube(newer.iloc[0])
In [ ]:
cube.imshow()
In [116]:
from string import Template
from pathlib import Path
class CTXIndex:
volumes_url = "https://pds-imaging.jpl.nasa.gov/volumes/mro.html"
release_url_template = \
Template("https://pds-imaging.jpl.nasa.gov/volumes/mro/release${release}.html")
volume_url_template = \
Template("https://pds-imaging.jpl.nasa.gov/data/mro/mars_reconnaissance_orbiter/ctx/mrox_${volume}/")
@property
def web_tables_list(self):
print("Scraping volumes page ...")
return pd.read_html(self.volumes_url)
@property
def release_number(self):
l = self.web_tables_list
# The last item of last table looks like "Release XX"
return l[-1].iloc[-1, 0].split()[-1]
@property
def release_url(self):
return self.release_url_template.substitute(release=self.release_number)
@property
def latest_volume_url(self):
print("Scraping latest release page ...")
l = pd.read_html(self.release_url)
# get last row of 4th table
row = l[3].iloc[-1]
number = None
# first number that is NAN breaks the loop over last row of table
for elem in row.values:
try:
number = int(elem.split()[-1])
except AttributeError:
break
return self.volume_url_template.substitute(volume=number)
@property
def latest_index_label_url(self):
return f"{self.latest_volume_url}index/cumindex.lbl"
In [117]:
index = CTXIndex()
In [118]:
index.latest_index_label_url
Out[118]:
In [119]:
from planetarypy.pdstools import indices
In [121]:
indices.download(label_url=index.latest_index_label_url,
local_dir=Path("~/Dropbox/data/ctx").expanduser())
In [ ]: