Set database path


In [2]:
from planetarypy import io

In [3]:
io.set_database_path("/Users/klay6683/Dropbox/data/planetarypy/")


Saved database path [data_archive]
path = "/Users/klay6683/Dropbox/data/planetarypy/"
 into /Users/klay6683/.planetarypy.toml.

In [4]:
from planetarypy.pdstools import indices

In [5]:
indices.list_available_index_files()


cassini:
  iss:
    index: https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_index.lbl
    inventory: https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_inventory.lbl
    moon_summary: https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_moon_summary.lbl
    ring_summary: https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_ring_summary.lbl
    saturn_summary: https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_saturn_summary.lbl
  uvis:
    index: https://pds-rings.seti.org/holdings/metadata/COUVIS_0xxx/COUVIS_0999/COUVIS_0999_index.lbl
    moon_summary: https://pds-rings.seti.org/holdings/metadata/COUVIS_0xxx/COUVIS_0999/COUVIS_0999_moon_summary.lbl
    ring_summary: https://pds-rings.seti.org/holdings/metadata/COUVIS_0xxx/COUVIS_0999/COUVIS_0999_ring_summary.lbl
    saturn_summary: https://pds-rings.seti.org/holdings/metadata/COUVIS_0xxx/COUVIS_0999/COUVIS_0999_saturn_summary.lbl
    supplemental_index: https://pds-rings.seti.org/holdings/metadata/COUVIS_0xxx/COUVIS_0999/COUVIS_0999_supplemental_index.lbl
mro:
  hirise:
    dtm: https://hirise-pds.lpl.arizona.edu/PDS/INDEX/DTMCUMINDEX.LBL
    edr: https://hirise-pds.lpl.arizona.edu/PDS/INDEX/EDRCUMINDEX.LBL
    rdr: https://hirise-pds.lpl.arizona.edu/PDS/INDEX/RDRCUMINDEX.LBL

Use indices.download('mission:instrument:index') to download in index file.
For example: indices.download('cassini:uvis:moon_summary'

UVIS metadata


In [3]:
from planetarypy.pdstools import indices
from planetarypy.utils import download
from tqdm import tqdm

class ProgressBar(tqdm):
    """Provides `update_to(n)` which uses `tqdm.update(delta_n)`."""
    def update_to(self, b=1, bsize=1, tsize=None):
        """
        b  : int, optional
            Number of blocks transferred so far [default: 1].
        bsize  : int, optional
            Size of each block (in tqdm units) [default: 1].
        tsize  : int, optional
            Total size (in tqdm units). If [default: None] remains unchanged.
        """
        if tsize is not None:
            self.total = tsize
        self.update(b * bsize - self.n)  # will also set self.n = b * bsize

META_URL = 'http://pds-rings.seti.org/metadata'


class META:
    INDICES = {'index': 'Cumulative product index of volume series',
               'inventory': 'Cumulative list of observed bodies by product',
               'moon_summary': 'Cumulative list of observed geometry on moons',
               'ring_summary': 'Cumulative list of observed geometry on rings',
               'saturn_summary': 'Cumulative list of observed geometry on Saturn'
               }

    def __init__(self, name=''):
        if name == '':
            print("Call me with one of the following index names:")
            for k,v in self.INDICES.items():
                print(k,": ", v)
            raise ValueError("Provide index name.")
        else:
            self._name = name
        
    @property
    def name(self):
        return self._name
    
    @property
    def folder_url(self):
        return META_URL + f'/CO{self.id}xxx/CO{self.id}999/'
        
    @property
    def meta_filename(self):
        return f'CO{self.id}999_{self.name}'
            
    @property
    def label_url(self):
        return self.folder_url + self.meta_filename + '.lbl'
    
    @property
    def table_url(self):
        return self.folder_url + self.meta_filename + '.tab'
    
    def download_table(self, local_folder='.'):
        baseurl = self.folder_url + self.meta_filename
        for ext in ['.lbl', '.tab']:
            filename = self.meta_filename + ext
            url = self.folder_url + filename
            local_path = f"{local_folder}/{filename}"
            print("Downloading", local_path)
            with ProgressBar(unit='B', unit_scale=True, miniters=1, desc=url) as t:
                download(url, local_path, reporthook=t.update_to, data=None)
    
    @property
    def label(self):
        return indices.IndexLabel(self.meta_filename + '.lbl')

    def read_table(self, **kwargs):
        return self.label.read_index_data(**kwargs)

    
class UVIS_META(META):
    id = 'UVIS_0'
    INDICES = {'index': 'Cumulative product index of volume series',
           'supplemental_index': 'Cumulative product index of volume series',
           'moon_summary': 'Cumulative list of observed geometry on moons',
           'ring_summary': 'Cumulative list of observed geometry on rings',
           'saturn_summary': 'Cumulative list of observed geometry on Saturn'
           }
    
class ISS_META(META):
    id = 'ISS_2'

class VIMS_META(META):
    id = 'VIMS_0'

In [ ]:
meta = UVIS_META('index')

In [ ]:
meta.label_url

In [ ]:
meta.label_url

In [ ]:
lbl = meta.label

In [ ]:
lbl.index_path

In [ ]:
meta = ISS_META('ring_summary')

In [ ]:
meta.download_table()

In [ ]:
meta.read_table??

In [ ]:
index = meta.read_table(convert_times=True)

In [ ]:
index.to_hdf("coiss_index.hdf", 'df')

In [81]:
from planetarypy.pdstools import indices

In [82]:
indices.list_available_index_files()


cassini:
  iss:
    index: https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_index.lbl
    inventory: https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_inventory.lbl
    moon_summary: https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_moon_summary.lbl
    ring_summary: https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_ring_summary.lbl
    saturn_summary: https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_saturn_summary.lbl
  uvis:
    index: https://pds-rings.seti.org/holdings/metadata/COUVIS_0xxx/COUVIS_0999/COUVIS_0999_index.lbl
    moon_summary: https://pds-rings.seti.org/holdings/metadata/COUVIS_0xxx/COUVIS_0999/COUVIS_0999_moon_summary.lbl
mro:
  hirise:
    dtm: https://hirise-pds.lpl.arizona.edu/PDS/INDEX/DTMCUMINDEX.LBL
    edr: https://hirise-pds.lpl.arizona.edu/PDS/INDEX/EDRCUMINDEX.LBL
    rdr: https://hirise-pds.lpl.arizona.edu/PDS/INDEX/RDRCUMINDEX.LBL

Use indices.download('mission:instrument:index') to download in index file.
For example: indices.download('cassini:uvis:moon_summary'

In [83]:
from pyciss import index

In [84]:
index.download_ring_summary_index()


2018-12-10 01:14:31,587 - planetarypy.pdstools.indices - Downloading https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_ring_summary.lbl.
2018-12-10 01:14:31,588 - planetarypy.utils - Downloading https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_ring_summary.lbl into /Users/klay6683/Dropbox/data/ciss/db
2018-12-10 01:14:32,355 - planetarypy.pdstools.indices - Downloading https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_ring_summary.tab.
2018-12-10 01:14:32,356 - planetarypy.utils - Downloading https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_ring_summary.tab into /Users/klay6683/Dropbox/data/ciss/db

Done.
Downloaded and converted to pandas HDF: /Users/klay6683/Dropbox/data/ciss/db/COISS_2999_ring_summary.hdf

In [59]:
from nbtools.logging import setup_live_logging

In [60]:
setup_live_logging('planetarypy', 'DEBUG')


Out[60]:
<Logger planetarypy (DEBUG)>

In [66]:
from planetarypy.utils import download

In [67]:
url


Out[67]:
'https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_index.lbl'

In [70]:
p = Path('./downloads')
p.mkdir()

In [71]:
ret = download(url, p)


2018-12-09 23:36:00,857 - planetarypy.utils - Downloading https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_index.lbl into downloads


In [ ]:
indices.download('cassini:iss:index')

In [4]:
url = 'https://pds-rings.seti.org/holdings/metadata/COISS_2xxx/COISS_2999/COISS_2999_index.lbl'
 
# downloading with urllib
 
# imported the urllib library
import urllib
 
# Copy a network object to a local file
urllib.request.urlretrieve(url, "python.png")


Out[4]:
('python.png', <http.client.HTTPMessage at 0x11180a0b8>)

In [13]:
label = indices.IndexLabel("/Users/klay6683/Dropbox/data/planetarypy/EDRCUMINDEX.LBL")

In [14]:
df = label.read_index_data()


Converting times for column OBSERVATION_START_TIME.
Converting times for column START_TIME.
Converting times for column STOP_TIME.
Done.

In [15]:
df.to_hdf("/Users/klay6683/local_data/EDRCUMINDEX.hdf", 'df')


/Users/klay6683/miniconda3/envs/py37/lib/python3.7/site-packages/pandas/core/generic.py:1996: PerformanceWarning: 
your performance may suffer as PyTables will pickle object types that it cannot
map directly to c-types [inferred_type->mixed,key->block3_values] [items->['VOLUME_ID', 'FILE_NAME_SPECIFICATION', 'INSTRUMENT_HOST_ID', 'INSTRUMENT_ID', 'OBSERVATION_ID', 'PRODUCT_ID', 'HICAL_VERSION', 'TARGET_NAME', 'MISSION_PHASE_NAME', 'RATIONALE_DESC', 'OBSERVATION_START_COUNT', 'SPACECRAFT_CLOCK_START_COUNT', 'SPACECRAFT_CLOCK_STOP_COUNT', 'CCD_NAME', 'FILTER_NAME', 'FELICS_COMPRESSION_FLAG', 'STIMULATION_LAMP_FLAG_RED', 'STIMULATION_LAMP_FLAG_BLUEGREEN', 'STIMULATION_LAMP_FLAG_NEARINFRARED', 'LOOKUP_TABLE_TYPE', 'STEREO_FLAG']]

  return pytables.to_hdf(path_or_buf, key, self, **kwargs)

In [ ]:
lab = indices.IndexLabel("/Volumes/USB128II/uvis/COUVIS_0999_index.lbl")

In [ ]:
df = lab.read_index_data(convert_times=False)

In [ ]:
time_cols = [col for col in df.columns if 'TIME' in col]
time_cols

In [ ]:
df = pd.read_hdf("/Volumes/USB128II/uvis/COUVIS_0999_index.hdf")

In [ ]:
df.columns

In [ ]:
df.to_hdf("/Volumes/USB128II/uvis/COUVIS_0999_supplemental_index.hdf", 'df')

In [ ]:
df.START_TIME.head()

In [ ]:
from planetarypy.utils import nasa_datetime_to_iso

In [ ]:
nasa_datetime_to_iso('1999-01-07T16:53:07.953')

In [ ]:
df.to_hdf("/Volumes/USB128II/uvis/COUVIS_0999_index.hdf", 'df')

In [ ]:
df.columns

In [ ]:
df.H_LEVEL.value_counts

In [ ]:
df.TARGET_NAME.value_counts()

In [ ]:
df.OBSERVATION_TYPE.value_counts(dropna=False)

In [ ]:


In [ ]:
df = df.assign(fname=df.FILE_SPECIFICATION_NAME.map(lambda x: x.split('/')[-1][:-4]))

In [ ]:
df.fname.is_unique

In [ ]:
index = pd.read_hdf("/Volumes/Research/pyciss/coiss_index.hdf")
index= index.assign(fname=index.FILE_SPECIFICATION_NAME.map(lambda x: x.split('/')[-1][:-4]))

In [ ]:
rings = index[index.TARGET_DESC.str.contains('Saturn-Rings')]

In [ ]:
dataset = df.merge(rings, on='fname')

In [ ]:
dataset.shape

In [ ]:
dataset[dataset.COARSEST_RADIAL_RESOLUTION<0] = np.nan

In [ ]:
dataset[dataset.FINEST_RADIAL_RESOLUTION < 0] = np.nan

In [ ]:
dataset.COARSEST_RADIAL_RESOLUTION.hist(range=(0,1))

In [ ]:
dataset.FINEST_RADIAL_RESOLUTION.hist(range=(0,1))

In [ ]:
dataset.query("COARSEST_RADIAL_RESOLUTION<0.4").IMAGE_TIME.str[:4].value_counts()

In [ ]:
dataset.query("'N1467345090' in fname")

In [ ]:
index[index.fname.str.contains('N1467345090')].TARGET_NAME

In [ ]:
dataset.dropna(how='all', inplace=True)

In [ ]:
dataset[dataset.fname.str.contains('N1467345090')]

In [ ]:


In [ ]:
img_list = dataset.query("COARSEST_RADIAL_RESOLUTION<0.01")

In [ ]:
img_list.shape

In [ ]:
img_list.TARGET_DESC.value_counts()

In [ ]:
ringdata = index[index.TARGET_DESC.str.contains('ring', case=False)]

In [ ]:
ringdata.TARGET_DESC.value_counts()

In [ ]:


In [ ]:
from pyciss import downloader

In [ ]:
downloader.download_file_id(img_list.fname.iloc[0])

In [ ]:
from tqdm import tqdm

In [ ]:
index['fname'] = index.FILE_SPECIFICATION_NAME.map(lambda x: x.split('/')[-1][:-4])

In [ ]:
index['fname'].head()

In [ ]:
dataset = img_list.merge(index, on='fname')

In [ ]:


In [ ]:
dataset = pd.read_hdf('dataset_metadata.hdf', 'df')

In [ ]:
dataset.filter(regex='TIME').columns

In [ ]:
from pyciss.pipeline import Calibrator
from pyciss import io

In [ ]:
from pysis.exceptions import ProcessError

In [ ]:
def calib_image(fname):
    pm = io.PathManager(fname)
    if pm.cubepath.exists():
        return
    calib = Calibrator(fname, final_resolution=100)
    try:
        calib.standard_calib()
    except ProcessError as e:
        print(e.stderr)

In [ ]:
dataset.fname.map(calib_image)

In [ ]:
from planetarypy.utils import nasa_datetime_to_iso

In [ ]:
dataset['time'] = pd.to_datetime(dataset.IMAGE_TIME.map(nasa_datetime_to_iso))

In [ ]:
dataset.time

In [ ]:
newer = dataset[pd.DatetimeIndex(dataset.time).year >2014].fname

In [ ]:
from pyciss.ringcube import RingCube

In [ ]:
dataset.TARGET_NAME.value_counts()

In [ ]:
imp

In [ ]:
calib_image(newer.iloc[6])

In [ ]:
cube = RingCube(newer.iloc[0])

In [ ]:
cube.imshow()

CTX


In [116]:
from string import Template
from pathlib import Path

class CTXIndex:
    volumes_url = "https://pds-imaging.jpl.nasa.gov/volumes/mro.html"
    release_url_template = \
        Template("https://pds-imaging.jpl.nasa.gov/volumes/mro/release${release}.html")
    volume_url_template = \
        Template("https://pds-imaging.jpl.nasa.gov/data/mro/mars_reconnaissance_orbiter/ctx/mrox_${volume}/")
    @property
    def web_tables_list(self):
        print("Scraping volumes page ...")
        return pd.read_html(self.volumes_url)
    
    @property
    def release_number(self):
        l = self.web_tables_list
        # The last item of last table looks like "Release XX"
        return l[-1].iloc[-1, 0].split()[-1]
        
    @property
    def release_url(self):
        return self.release_url_template.substitute(release=self.release_number)
    
    @property
    def latest_volume_url(self):
        print("Scraping latest release page ...")
        l = pd.read_html(self.release_url)
        # get last row of 4th table
        row = l[3].iloc[-1]
        number = None
        # first number that is NAN breaks the loop over last row of table
        for elem in row.values:
            try:
                number = int(elem.split()[-1])
            except AttributeError:
                break
        return self.volume_url_template.substitute(volume=number)
    
    @property
    def latest_index_label_url(self):
        return f"{self.latest_volume_url}index/cumindex.lbl"

In [117]:
index = CTXIndex()

In [118]:
index.latest_index_label_url


Scraping latest release page ...
Scraping volumes page ...
Out[118]:
'https://pds-imaging.jpl.nasa.gov/data/mro/mars_reconnaissance_orbiter/ctx/mrox_3267/index/cumindex.lbl'

In [119]:
from planetarypy.pdstools import indices

In [121]:
indices.download(label_url=index.latest_index_label_url, 
                 local_dir=Path("~/Dropbox/data/ctx").expanduser())


Scraping latest release page ...
Scraping volumes page ...

Converting times for column IMAGE_TIME.
Done.
Downloaded and converted to pandas HDF: /Users/klay6683/Dropbox/data/ctx/cumindex.hdf

In [ ]: