How to update the Chandra CALDB at the HEASARC


In [1]:
from astropy.io import fits as pyfits
from astropy.time import Time
from astropy.table import Table
import ftputil
import time
import tarfile
import os

In [2]:
ChandraCaldbDir = "/pub/arcftp/caldb"
ChandraCaldbHost = "cda.harvard.edu"
CaldbWorkDir = "/Volumes/USRA16/CaldbWorkDir"

In [3]:
host = ftputil.FTPHost(ChandraCaldbHost, "anonymous", "mcorcoran@usra.edu")

In [4]:
#host.download("/pub/arcftp/caldb/caldb_4.7.2_main.tar.gz","/Volumes/USRA16/CaldbWorkDir/caldb_4.7.2_main.tar.gz",mode='b')

In [5]:
#host.download("/pub/arcftp/caldb/hrc_bkgrnd_4.7.0.tar.gz","/Volumes/USRA16/ChandraCaldbWorkDir/hrc_bkgrnd_4.7.0.tar.gz",mode='b')

In [4]:
ChandraCaldbFiles = host.listdir(ChandraCaldbDir)

host.close()

ChandraCaldbTarFiles = [f for f in ChandraCaldbFiles if '.tar' in f ]
ChandraCaldbTarFiles


Out[4]:
['acis_bkgrnd_4.6.9.tar.gz',
 'caldb_4.7.2_main.tar.gz',
 'hrc_bkgrnd_4.7.0.tar.gz']

In [6]:
print "Changing directory to {0}\n".format(CaldbWorkDir)
os.chdir(CaldbWorkDir)

print ("File List:")
print (ChandraCaldbTarFiles)
for f in ChandraCaldbTarFiles:
    remotefile = ChandraCaldbDir+"/"+f
    localfile = CaldbWorkDir+"/"+f
    if os.path.exists(f):
        print ("File {0} already exists; deleting prior to download".format(f))
        os.remove(f)
    print "\nGetting file "+f
    getfile = True
    inum = 0 # download counter 
    itrymax = 3 # maximum number of download attempts
    while (getfile and inum<itrymax):
        try:
            getfile=False
            host.download(remotefile,localfile,mode='b')
        except IOError:
            print "IOError on download of file {0}".format(remotefile)
            getfile = True
            inum =+ 1
        #
        # once file downloaded, then untar it and delete the tar file
        #
        print "Untarring {0}".format(f)
        tarf = tarfile.open(f)
        try:
            tarf.extractall()
        except:
            print "ERROR UNTARRING {0}".format(localfile)
        inum = 0
        print "Deleting {0}".format(f)
        os.remove(f)
    if inum >=itrymax:
        print "\nCould not retrieve {0} after {1} tries; Returning\n".format({remotefile, inum})


Changing directory to /Volumes/USRA16/CaldbWorkDir

File List:
['acis_bkgrnd_4.6.9.tar.gz', 'caldb_4.7.2_main.tar.gz', 'hrc_bkgrnd_4.7.0.tar.gz']
Getting file acis_bkgrnd_4.6.9.tar.gz
Untarring acis_bkgrnd_4.6.9.tar.gz
Getting file caldb_4.7.2_main.tar.gz
Untarring caldb_4.7.2_main.tar.gz
Getting file hrc_bkgrnd_4.7.0.tar.gz
---------------------------------------------------------------------------
EOFError                                  Traceback (most recent call last)
<ipython-input-6-8d2b06d3df0e> in <module>()
     17         try:
     18             getfile=False
---> 19             host.download(remotefile,localfile,mode='b')
     20         except IOError:
     21             print "IOError on download of file {0}".format(remotefile)

/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/ftputil/ftputil.pyc in download(self, source, target, mode, callback)
    518         source_file, target_file = self._download_files(source, target, mode)
    519         file_transfer.copy_file(source_file, target_file,
--> 520                                 conditional=False, callback=callback)
    521 
    522     def download_if_newer(self, source, target, mode='', callback=None):

/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/ftputil/file_transfer.pyc in copy_file(source_file, target_file, conditional, callback)
    151             # We didn't transfer.
    152             return False
--> 153     source_fobj = source_file.fobj()
    154     try:
    155         target_fobj = target_file.fobj()

/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/ftputil/file_transfer.pyc in fobj(self)
     83     def fobj(self):
     84         """Return a file object for the name/path in the constructor."""
---> 85         return self._host.file(self.name, self.mode)
     86 
     87 

/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/ftputil/ftputil.pyc in file(self, path, mode)
    205         # Fail early if we get a unicode path which can't be encoded.
    206         path = str(path)
--> 207         host = self._available_child()
    208         if host is None:
    209             host = self._copy()

/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/ftputil/ftputil.pyc in _available_child(self)
    185             if host._file.closed:
    186                 try:
--> 187                     host._session.pwd()
    188                 # Timed-out sessions raise `error_temp`.
    189                 except ftplib.error_temp:

/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/ftplib.pyc in pwd(self)
    584     def pwd(self):
    585         '''Return current working directory.'''
--> 586         resp = self.sendcmd('PWD')
    587         return parse257(resp)
    588 

/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/ftplib.pyc in sendcmd(self, cmd)
    247         '''Send a command and return the response.'''
    248         self.putcmd(cmd)
--> 249         return self.getresp()
    250 
    251     def voidcmd(self, cmd):

/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/ftplib.pyc in getresp(self)
    213     # Raise various errors if the response indicates an error
    214     def getresp(self):
--> 215         resp = self.getmultiline()
    216         if self.debugging: print '*resp*', self.sanitize(resp)
    217         self.lastresp = resp[:3]

/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/ftplib.pyc in getmultiline(self)
    199     # these are separated by '\n' characters in the string
    200     def getmultiline(self):
--> 201         line = self.getline()
    202         if line[3:4] == '-':
    203             code = line[:3]

/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/ftplib.pyc in getline(self)
    189         if self.debugging > 1:
    190             print '*get*', self.sanitize(line)
--> 191         if not line: raise EOFError
    192         if line[-2:] == CRLF: line = line[:-2]
    193         elif line[-1:] in CRLF: line = line[:-1]

EOFError: 

In [26]:
#
# don't forget the manifest and readme files
#
manfile = [f for f in ChandraCaldbFiles if 'MANIFEST' in f]
manfile = manfile[0]
readme = [f for f in ChandraCaldbFiles if 'README' in f]
readme = readme[0]
print manfile
print readme


MANIFEST_4.7.2_main.txt
README_caldb4.7.2.txt

In [31]:
maincaldb=[f for f in ChandraCaldbTarFiles if 'main' in f]
maincaldb = maincaldb[0]
tarf = tarfile.open(maincaldb)

In [32]:
tarf.extractall()

In [5]:
os.chdir(CaldbWorkDir)

In [9]:
acisbkg = [f for f in ChandraCaldbFiles if 'acis_bkgrnd' in f]
acisbkg = acisbkg[0]
tarf = tarfile.open(acisbkg)
tarf.extractall()


---------------------------------------------------------------------------
IOError                                   Traceback (most recent call last)
<ipython-input-9-eda44d64eb25> in <module>()
      3 acisbkg = acisbkg[0]
      4 tarf = tarfile.open(acisbkg)
----> 5 tarf.extractall()

/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/tarfile.pyc in extractall(self, path, members)
   2077                 tarinfo = copy.copy(tarinfo)
   2078                 tarinfo.mode = 0700
-> 2079             self.extract(tarinfo, path)
   2080 
   2081         # Reverse sort directories.

/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/tarfile.pyc in extract(self, member, path)
   2114 
   2115         try:
-> 2116             self._extract_member(tarinfo, os.path.join(path, tarinfo.name))
   2117         except EnvironmentError, e:
   2118             if self.errorlevel > 0:

/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/tarfile.pyc in _extract_member(self, tarinfo, targetpath)
   2190 
   2191         if tarinfo.isreg():
-> 2192             self.makefile(tarinfo, targetpath)
   2193         elif tarinfo.isdir():
   2194             self.makedir(tarinfo, targetpath)

/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/tarfile.pyc in makefile(self, tarinfo, targetpath)
   2231         try:
   2232             with bltn_open(targetpath, "wb") as target:
-> 2233                 copyfileobj(source, target)
   2234         finally:
   2235             source.close()

/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/tarfile.pyc in copyfileobj(src, dst, length)
    264         return
    265     if length is None:
--> 266         shutil.copyfileobj(src, dst)
    267         return
    268 

/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/shutil.pyc in copyfileobj(fsrc, fdst, length)
     50         if not buf:
     51             break
---> 52         fdst.write(buf)
     53 
     54 def _samefile(src, dst):

IOError: [Errno 28] No space left on device

In [ ]:
hribkg = [f for f in ChandraCaldbFiles if 'hri_bkgrnd' in f]
hribkg = hribkg[0]
tarf = tarfile.open(hribkg)
tarf.extractall()

In [10]:
os.chdir('/Users/corcoran')

In [11]:
!pwd


/Users/corcoran

In [ ]:


In [ ]: