In [2]:
from py2neo import authenticate, Node, Relationship, Graph
authenticate("localhost:7474", 'neo4j', 'prolog16')
graph = Graph("http://localhost:7474/db/data/")

from pyesgf.search import SearchConnection
PCMDI_SERVICE = 'http://pcmdi9.llnl.gov/esg-search/search'
conn = SearchConnection(PCMDI_SERVICE,distrib=True)
project = u'CMIP5'
ctx = conn.new_context(project=project,replica=False,latest=True)
root_node = Node("Collection",name='cmip5',level=10, identifier = 'project')

products = conn.new_context(project=project,replica=False,latest=True).facet_counts['product'].keys()
for product in products:
    product_node = Node("Collection",name=product, level=9)
    rel = Relationship(product_node,"belongs_to",root_node)
    graph.create(rel)
    #for institute in facet_counts['institute']:
    
    institutes = conn.new_context(project=project,product=product,replica=False,latest=True).facet_counts['institute'].keys()
    print "Institutes:",institutes
    for institute in [u'MPI-M']:
         institute_node = Node("Collection",name=institute, level=8, identifier = 'institute')
         rel = Relationship(institute_node,"belongs_to",product_node)
         graph.create(rel)
         models = conn.new_context(project=project,product=product,institute=institute,replica=False,latest=True).facet_counts['model'].keys()
         print "Models:",models 
         for model in models: 
                institute_node = Node("Collection",name=institute, level=7, identifier = 'institute')
                rel = Relationship(institute_node,"belongs_to",product_node)
                graph.create(rel) 
                experiments = conn.new_context(project=project,product=product,institute=institute,model=model,replica=False,latest=True).facet_counts['experiment'].keys()
                print "Experiments:", experiments
                for experiment in experiments:
                    institute_node = Node("Collection",name=institute, level=6, identifier = 'institute')
                    rel = Relationship(institute_node,"belongs_to",product_node)
                    graph.create(rel) 
                    time_frequencies = conn.new_context(project=project,product=product,institute=institute,model=model,experiment=experiment,replica=False,latest=True).facet_counts['time_frequency'].keys()
                    print "time_frequencies",time_frequencies
                    for time_frequency in time_frequencies:
                        frequency_node = Node("Collection",name=time_frequency, level=5, identifier = 'time_frequency')
                        rel = Relationship(frequency_node,"belongs_to",institute_node)
                        graph.create(rel) 
                        realms = conn.new_context(project=project,product=product,institute=institute,model=model,experiment=experiment,time_frequency=time_frequency,replica=False,latest=True).facet_counts['realm'].keys()
                        print "Realms:",realms
                        for realm in realms:
                            realm_node = Node("Collection",name=institute, level=4, identifier = 'realm')
                            rel = Relationship(realm_node,"belongs_to",frequency_node)
                            graph.create(rel)
                            mips = conn.new_context(project=project,product=product,institute=institute,model=model,experiment=experiment,time_frequency=time_frequency,realm=realm,replica=False,latest=True).facet_counts['cmor_table'].keys()
                            

                            results = ctx.search()
                            print "Results for facets:",model,experiment,time_frequency,realm

                            if len(results) > 1:
                               print " vor file access Such-Ergebinisse:",len(results)
                               #print len(results)
                               #print results
                               #print 'Hits:', ctx.hit_count
                               print "============files=========================="
                               for i in range(0,len(results)):
                                            file_ctx = results[i].file_context()
                                            files = file_ctx.search()
                                            print "Anzahl files:",len(files)
                                            size = files.batch_size
                                            file_nodes = []
                                            for file in files:
                                                print file.filename
                                               
                               print "======================================"


Institutes: [u'INM', u'UNSW']
Models: []
Institutes: [u'LASG-IAP', u'CCCMA', u'CMCC', u'LASG-CESS', u'INM', u'MIROC', u'NIMR-KMA', u'MPI-M']
Models: [u'MPI-ESM-P']
Experiments: [u'sstClim', u'midHolocene', u'lgm', u'sstClim4xCO2', u'past1000', u'1pctCO2', u'abrupt4xCO2', u'historical', u'piControl']
time_frequencies [u'fx', u'mon', u'day']
Realms: [u'atmos', u'land']
Results for facets: MPI-ESM-P sstClim fx atmos
 vor file access Such-Ergebinisse: 4610
============files==========================
Anzahl files: 70
hfss_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
hfls_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
hfss_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
hfls_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
clt_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
clt_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
huss_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
hurs_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
huss_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
hurs_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
pr_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
hus_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
hur_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
hus_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
hur_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
prc_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
prc_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
psl_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
prsn_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
psl_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
prsn_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
ps_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
ps_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
pr_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rldscs_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rldscs_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rlutcs_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rlutcs_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rsds_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rlds_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rlds_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rlut_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rlut_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rsdscs_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rsdscs_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rsutcs_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rsutcs_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rsds_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rsdt_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rsut_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rsdt_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
rsut_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
tasmax_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
tasmin_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
tasmax_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
sfcWind_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
sfcWind_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
tas_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
tas_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
ta_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
ta_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
tauv_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
tauu_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
tauv_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
tauu_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
tasmin_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
ts_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
ts_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
ua_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
ua_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
wap_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
wap_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
vas_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
uas_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
vas_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
uas_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
zg_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
va_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
va_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
zg_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
Anzahl files: 418
clt_Amon_HadGEM2-AO_piControl_r1i1p1_060101-070012.nc
clt_Amon_HadGEM2-AO_piControl_r1i1p1_040101-050012.nc
clt_Amon_HadGEM2-AO_piControl_r1i1p1_050101-060012.nc
clt_Amon_HadGEM2-AO_piControl_r1i1p1_060101-070012.nc
clt_Amon_HadGEM2-AO_piControl_r1i1p1_040101-050012.nc
clt_Amon_HadGEM2-AO_piControl_r1i1p1_050101-060012.nc
evspsbl_Amon_HadGEM2-AO_piControl_r1i1p1_060101-070012.nc
clt_Amon_HadGEM2-AO_piControl_r1i1p1_000101-010012.nc
clt_Amon_HadGEM2-AO_piControl_r1i1p1_030101-040012.nc
clt_Amon_HadGEM2-AO_piControl_r1i1p1_020101-030012.nc
clt_Amon_HadGEM2-AO_piControl_r1i1p1_010101-020012.nc
clt_Amon_HadGEM2-AO_piControl_r1i1p1_000101-010012.nc
clt_Amon_HadGEM2-AO_piControl_r1i1p1_030101-040012.nc
clt_Amon_HadGEM2-AO_piControl_r1i1p1_020101-030012.nc
clt_Amon_HadGEM2-AO_piControl_r1i1p1_010101-020012.nc
evspsbl_Amon_HadGEM2-AO_piControl_r1i1p1_020101-030012.nc
evspsbl_Amon_HadGEM2-AO_piControl_r1i1p1_040101-050012.nc
evspsbl_Amon_HadGEM2-AO_piControl_r1i1p1_050101-060012.nc
evspsbl_Amon_HadGEM2-AO_piControl_r1i1p1_030101-040012.nc
evspsbl_Amon_HadGEM2-AO_piControl_r1i1p1_020101-030012.nc
evspsbl_Amon_HadGEM2-AO_piControl_r1i1p1_040101-050012.nc
evspsbl_Amon_HadGEM2-AO_piControl_r1i1p1_060101-070012.nc
evspsbl_Amon_HadGEM2-AO_piControl_r1i1p1_050101-060012.nc
evspsbl_Amon_HadGEM2-AO_piControl_r1i1p1_030101-040012.nc
hfls_Amon_HadGEM2-AO_piControl_r1i1p1_060101-070012.nc
hfls_Amon_HadGEM2-AO_piControl_r1i1p1_050101-060012.nc
hfls_Amon_HadGEM2-AO_piControl_r1i1p1_040101-050012.nc
hfls_Amon_HadGEM2-AO_piControl_r1i1p1_060101-070012.nc
hfls_Amon_HadGEM2-AO_piControl_r1i1p1_050101-060012.nc
evspsbl_Amon_HadGEM2-AO_piControl_r1i1p1_000101-010012.nc
evspsbl_Amon_HadGEM2-AO_piControl_r1i1p1_010101-020012.nc
evspsbl_Amon_HadGEM2-AO_piControl_r1i1p1_000101-010012.nc
evspsbl_Amon_HadGEM2-AO_piControl_r1i1p1_010101-020012.nc
hfls_Amon_HadGEM2-AO_piControl_r1i1p1_020101-030012.nc
hfls_Amon_HadGEM2-AO_piControl_r1i1p1_000101-010012.nc
hfls_Amon_HadGEM2-AO_piControl_r1i1p1_010101-020012.nc
hfls_Amon_HadGEM2-AO_piControl_r1i1p1_030101-040012.nc
hfls_Amon_HadGEM2-AO_piControl_r1i1p1_020101-030012.nc
hfls_Amon_HadGEM2-AO_piControl_r1i1p1_000101-010012.nc
hfls_Amon_HadGEM2-AO_piControl_r1i1p1_040101-050012.nc
hfls_Amon_HadGEM2-AO_piControl_r1i1p1_010101-020012.nc
hfls_Amon_HadGEM2-AO_piControl_r1i1p1_030101-040012.nc
hurs_Amon_HadGEM2-AO_piControl_r1i1p1_020101-030012.nc
hurs_Amon_HadGEM2-AO_piControl_r1i1p1_030101-040012.nc
hurs_Amon_HadGEM2-AO_piControl_r1i1p1_060101-070012.nc
hurs_Amon_HadGEM2-AO_piControl_r1i1p1_040101-050012.nc
hurs_Amon_HadGEM2-AO_piControl_r1i1p1_050101-060012.nc
hurs_Amon_HadGEM2-AO_piControl_r1i1p1_030101-040012.nc
hurs_Amon_HadGEM2-AO_piControl_r1i1p1_060101-070012.nc
hurs_Amon_HadGEM2-AO_piControl_r1i1p1_040101-050012.nc
hurs_Amon_HadGEM2-AO_piControl_r1i1p1_050101-060012.nc
hurs_Amon_HadGEM2-AO_piControl_r1i1p1_000101-010012.nc
hurs_Amon_HadGEM2-AO_piControl_r1i1p1_010101-020012.nc
hurs_Amon_HadGEM2-AO_piControl_r1i1p1_020101-030012.nc
hurs_Amon_HadGEM2-AO_piControl_r1i1p1_000101-010012.nc
hurs_Amon_HadGEM2-AO_piControl_r1i1p1_010101-020012.nc
hus_Amon_HadGEM2-AO_piControl_r1i1p1_060101-070012.nc
hus_Amon_HadGEM2-AO_piControl_r1i1p1_050101-060012.nc
hus_Amon_HadGEM2-AO_piControl_r1i1p1_060101-070012.nc
hus_Amon_HadGEM2-AO_piControl_r1i1p1_050101-060012.nc
hus_Amon_HadGEM2-AO_piControl_r1i1p1_000101-010012.nc
hus_Amon_HadGEM2-AO_piControl_r1i1p1_020101-030012.nc
hus_Amon_HadGEM2-AO_piControl_r1i1p1_010101-020012.nc
hus_Amon_HadGEM2-AO_piControl_r1i1p1_030101-040012.nc
hus_Amon_HadGEM2-AO_piControl_r1i1p1_040101-050012.nc
hus_Amon_HadGEM2-AO_piControl_r1i1p1_020101-030012.nc
hus_Amon_HadGEM2-AO_piControl_r1i1p1_010101-020012.nc
hus_Amon_HadGEM2-AO_piControl_r1i1p1_030101-040012.nc
hus_Amon_HadGEM2-AO_piControl_r1i1p1_040101-050012.nc
huss_Amon_HadGEM2-AO_piControl_r1i1p1_030101-040012.nc
huss_Amon_HadGEM2-AO_piControl_r1i1p1_060101-070012.nc
huss_Amon_HadGEM2-AO_piControl_r1i1p1_050101-060012.nc
huss_Amon_HadGEM2-AO_piControl_r1i1p1_040101-050012.nc
huss_Amon_HadGEM2-AO_piControl_r1i1p1_030101-040012.nc
huss_Amon_HadGEM2-AO_piControl_r1i1p1_060101-070012.nc
huss_Amon_HadGEM2-AO_piControl_r1i1p1_050101-060012.nc
huss_Amon_HadGEM2-AO_piControl_r1i1p1_040101-050012.nc
hus_Amon_HadGEM2-AO_piControl_r1i1p1_000101-010012.nc
huss_Amon_HadGEM2-AO_piControl_r1i1p1_000101-010012.nc
huss_Amon_HadGEM2-AO_piControl_r1i1p1_020101-030012.nc
huss_Amon_HadGEM2-AO_piControl_r1i1p1_010101-020012.nc
huss_Amon_HadGEM2-AO_piControl_r1i1p1_000101-010012.nc
huss_Amon_HadGEM2-AO_piControl_r1i1p1_020101-030012.nc
huss_Amon_HadGEM2-AO_piControl_r1i1p1_010101-020012.nc
pr_Amon_HadGEM2-AO_piControl_r1i1p1_060101-070012.nc
pr_Amon_HadGEM2-AO_piControl_r1i1p1_050101-060012.nc
pr_Amon_HadGEM2-AO_piControl_r1i1p1_060101-070012.nc
pr_Amon_HadGEM2-AO_piControl_r1i1p1_010101-020012.nc
pr_Amon_HadGEM2-AO_piControl_r1i1p1_030101-040012.nc
pr_Amon_HadGEM2-AO_piControl_r1i1p1_020101-030012.nc
pr_Amon_HadGEM2-AO_piControl_r1i1p1_040101-050012.nc
pr_Amon_HadGEM2-AO_piControl_r1i1p1_010101-020012.nc
pr_Amon_HadGEM2-AO_piControl_r1i1p1_030101-040012.nc
pr_Amon_HadGEM2-AO_piControl_r1i1p1_020101-030012.nc
pr_Amon_HadGEM2-AO_piControl_r1i1p1_050101-060012.nc
pr_Amon_HadGEM2-AO_piControl_r1i1p1_040101-050012.nc
prsn_Amon_HadGEM2-AO_piControl_r1i1p1_060101-070012.nc
prsn_Amon_HadGEM2-AO_piControl_r1i1p1_050101-060012.nc
prsn_Amon_HadGEM2-AO_piControl_r1i1p1_040101-050012.nc
prsn_Amon_HadGEM2-AO_piControl_r1i1p1_030101-040012.nc
---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-2-4da3903f51bc> in <module>()
     65                                             size = files.batch_size
     66                                             file_nodes = []
---> 67                                             for file in files:
     68                                                 print file.filename
     69 

/home/stephan/anaconda/envs/scientific1/lib/python2.7/_abcoll.pyc in __iter__(self)
    600         try:
    601             while True:
--> 602                 v = self[i]
    603                 yield v
    604                 i += 1

/home/stephan/anaconda/envs/scientific1/lib/python2.7/site-packages/pyesgf/search/results.pyc in __getitem__(self, index)
     44         offset = index % self.batch_size
     45         if self.__batch_cache[batch_i] is None:
---> 46             self.__batch_cache[batch_i] = self.__get_batch(batch_i)
     47         batch = self.__batch_cache[batch_i]
     48 

/home/stephan/anaconda/envs/scientific1/lib/python2.7/site-packages/pyesgf/search/results.pyc in __get_batch(self, batch_i)
     78         response = self.context.connection.send_search(query_dict, limit=limit, 
     79                                                        offset=offset,
---> 80                                                        shards=self.context.shards)
     81 
     82         #!TODO: strip out results

/home/stephan/anaconda/envs/scientific1/lib/python2.7/site-packages/pyesgf/search/connection.pyc in send_search(self, query_dict, limit, offset, shards)
    103         """
    104         full_query = self._build_query(query_dict, limit, offset, shards)
--> 105         response = self._send_query('search', full_query)
    106         ret = json.load(response)
    107 

/home/stephan/anaconda/envs/scientific1/lib/python2.7/site-packages/pyesgf/search/connection.pyc in _send_query(self, endpoint, full_query)
    142 
    143         try:
--> 144             response = urllib2.urlopen(query_url)
    145         except urllib2.HTTPError, err:
    146             log.warn("HTTP request received error code: %s" % err.code)

/home/stephan/anaconda/envs/scientific1/lib/python2.7/urllib2.pyc in urlopen(url, data, timeout, cafile, capath, cadefault, context)
    152     else:
    153         opener = _opener
--> 154     return opener.open(url, data, timeout)
    155 
    156 def install_opener(opener):

/home/stephan/anaconda/envs/scientific1/lib/python2.7/urllib2.pyc in open(self, fullurl, data, timeout)
    429             req = meth(req)
    430 
--> 431         response = self._open(req, data)
    432 
    433         # post-process response

/home/stephan/anaconda/envs/scientific1/lib/python2.7/urllib2.pyc in _open(self, req, data)
    447         protocol = req.get_type()
    448         result = self._call_chain(self.handle_open, protocol, protocol +
--> 449                                   '_open', req)
    450         if result:
    451             return result

/home/stephan/anaconda/envs/scientific1/lib/python2.7/urllib2.pyc in _call_chain(self, chain, kind, meth_name, *args)
    407             func = getattr(handler, meth_name)
    408 
--> 409             result = func(*args)
    410             if result is not None:
    411                 return result

/home/stephan/anaconda/envs/scientific1/lib/python2.7/urllib2.pyc in http_open(self, req)
   1225 
   1226     def http_open(self, req):
-> 1227         return self.do_open(httplib.HTTPConnection, req)
   1228 
   1229     http_request = AbstractHTTPHandler.do_request_

/home/stephan/anaconda/envs/scientific1/lib/python2.7/urllib2.pyc in do_open(self, http_class, req, **http_conn_args)
   1198         else:
   1199             try:
-> 1200                 r = h.getresponse(buffering=True)
   1201             except TypeError: # buffering kw not supported
   1202                 r = h.getresponse()

/home/stephan/anaconda/envs/scientific1/lib/python2.7/httplib.pyc in getresponse(self, buffering)
   1130 
   1131         try:
-> 1132             response.begin()
   1133             assert response.will_close != _UNKNOWN
   1134             self.__state = _CS_IDLE

/home/stephan/anaconda/envs/scientific1/lib/python2.7/httplib.pyc in begin(self)
    451         # read until we get a non-100 response
    452         while True:
--> 453             version, status, reason = self._read_status()
    454             if status != CONTINUE:
    455                 break

/home/stephan/anaconda/envs/scientific1/lib/python2.7/httplib.pyc in _read_status(self)
    407     def _read_status(self):
    408         # Initialize with Simple-Response defaults
--> 409         line = self.fp.readline(_MAXLINE + 1)
    410         if len(line) > _MAXLINE:
    411             raise LineTooLong("header line")

/home/stephan/anaconda/envs/scientific1/lib/python2.7/socket.pyc in readline(self, size)
    478             while True:
    479                 try:
--> 480                     data = self._sock.recv(self._rbufsize)
    481                 except error, e:
    482                     if e.args[0] == EINTR:

KeyboardInterrupt: 

In [ ]:
from py2neo import authenticate, Node, Relationship, Graph
authenticate("localhost:7474", 'neo4j', 'prolog16')
graph = Graph("http://localhost:7474/db/data/")

from pyesgf.search import SearchConnection
PCMDI_SERVICE = 'http://pcmdi9.llnl.gov/esg-search/search'
conn = SearchConnection(PCMDI_SERVICE,distrib=True)
project = u'CMIP5'
ctx = conn.new_context(project=project,replica=False,latest=True)
root_node = Node("Collection",name='cmip5',level=10, identifier = 'project')

products = conn.new_context(project=project,replica=False,latest=True).facet_counts['product'].keys()
for product in products:
    product_node = Node("Collection",name=product, level=9)
    rel = Relationship(product_node,"belongs_to",root_node)
    graph.create(rel)
    #for institute in facet_counts['institute']:
    
    institutes = conn.new_context(project=project,product=product,replica=False,latest=True).facet_counts['institute'].keys()
    print "Institutes:",institutes
    for institute in [u'MPI-M']:
         institute_node = Node("Collection",name=institute, level=8, identifier = 'institute')
         rel = Relationship(institute_node,"belongs_to",product_node)
         graph.create(rel)
         models = conn.new_context(project=project,product=product,institute=institute,replica=False,latest=True).facet_counts['model'].keys()
         print "Models:",models 
         for model in models: 
                institute_node = Node("Collection",name=institute, level=7, identifier = 'institute')
                rel = Relationship(institute_node,"belongs_to",product_node)
                graph.create(rel) 
                experiments = conn.new_context(project=project,product=product,institute=institute,model=model,replica=False,latest=True).facet_counts['experiment'].keys()
                print "Experiments:", experiments
                for experiment in experiments:
                    institute_node = Node("Collection",name=institute, level=6, identifier = 'institute')
                    rel = Relationship(institute_node,"belongs_to",product_node)
                    graph.create(rel) 
                    time_frequencies = conn.new_context(project=project,product=product,institute=institute,model=model,experiment=experiment,replica=False,latest=True).facet_counts['time_frequency'].keys()
                    print "time_frequencies",time_frequencies
                    for time_frequency in time_frequencies:
                        frequency_node = Node("Collection",name=time_frequency, level=5, identifier = 'time_frequency')
                        rel = Relationship(frequency_node,"belongs_to",institute_node)
                        graph.create(rel) 
                        realms = conn.new_context(project=project,product=product,institute=institute,model=model,experiment=experiment,time_frequency=time_frequency,replica=False,latest=True).facet_counts['realm'].keys()
                    
                            

                        results = ctx.search()
                        print "Results for facets:",model,experiment,time_frequency

                        if len(results) > 1:
                            print " vor file access Such-Ergebinisse:",len(results)
                            #print len(results)
                            #print results
                            #print 'Hits:', ctx.hit_count
                            print "============files=========================="
                            for i in range(0,len(results)):
                                            file_ctx = results[i].file_context()
                                            files = file_ctx.search()
                                            print "Anzahl files:",len(files)
                                            size = files.batch_size
                                            file_nodes = []
                                            for file in files:
                                                print file.filename
                                               
                            print "======================================"

In [1]:
from py2neo import authenticate, Node, Relationship, Graph
authenticate("localhost:7474", 'neo4j', 'prolog16')
graph = Graph("http://localhost:7474/db/data/")

from pyesgf.search import SearchConnection
PCMDI_SERVICE = 'http://pcmdi9.llnl.gov/esg-search/search'
conn = SearchConnection(PCMDI_SERVICE,distrib=True)
project = u'CMIP5'
ctx = conn.new_context(project=project,replica=False,latest=True)
root_node = Node("Collection",name='cmip5',level=10, identifier = 'project')

products = conn.new_context(project=project,replica=False,latest=True).facet_counts['product'].keys()
for product in products:
    product_node = Node("Collection",name=product, level=9)
    rel = Relationship(product_node,"belongs_to",root_node)
    graph.create(rel)
    #for institute in facet_counts['institute']:
    
    institutes = conn.new_context(project=project,product=product,replica=False,latest=True).facet_counts['institute'].keys()
    print "Institutes:",institutes
    for institute in [u'MPI-M']:
         institute_node = Node("Collection",name=institute, level=8, identifier = 'institute')
         rel = Relationship(institute_node,"belongs_to",product_node)
         graph.create(rel)
         models = conn.new_context(project=project,product=product,institute=institute,replica=False,latest=True).facet_counts['model'].keys()
         print "Models:",models 
         for model in models: 
                institute_node = Node("Collection",name=institute, level=7, identifier = 'institute')
                rel = Relationship(institute_node,"belongs_to",product_node)
                graph.create(rel) 
                experiments = conn.new_context(project=project,product=product,institute=institute,model=model,replica=False,latest=True).facet_counts['experiment'].keys()
                print "Experiments:", experiments
                for experiment in experiments:
                    institute_node = Node("Collection",name=institute, level=6, identifier = 'institute')
                    rel = Relationship(institute_node,"belongs_to",product_node)
                    graph.create(rel) 
                    time_frequencies = conn.new_context(project=project,product=product,institute=institute,model=model,experiment=experiment,replica=False,latest=True).facet_counts['time_frequency'].keys()
                    print "time_frequencies",time_frequencies
                    for time_frequency in time_frequencies:
                        frequency_node = Node("Collection",name=time_frequency, level=5, identifier = 'time_frequency')
                        rel = Relationship(frequency_node,"belongs_to",institute_node)
                        graph.create(rel) 
                        realms = conn.new_context(project=project,product=product,institute=institute,model=model,experiment=experiment,time_frequency=time_frequency,replica=False,latest=True).facet_counts['realm'].keys()
                        print "Realms:",realms
                        for realm in realms:
                            realm_node = Node("Collection",name=institute, level=4, identifier = 'realm')
                            rel = Relationship(realm_node,"belongs_to",frequency_node)
                            graph.create(rel)
                            mips = conn.new_context(project=project,product=product,institute=institute,model=model,experiment=experiment,time_frequency=time_frequency,realm=realm,replica=False,latest=True).facet_counts['cmor_table'].keys()
                            print "cmor_table:",mips
                            for mip in mips:
                                mip_node = Node("Collection",name=mip, level=3, identifier = 'cmor_table')
                                rel = Relationship(mip_node,"belongs_to",realm_node)
                                graph.create(rel) 
                                ensembles = conn.new_context(project=project,product=product,institute=institute,model=model,experiment=experiment,time_frequency=time_frequency,realm=realm,cmor_table=mip,replica=False,latest=True).facet_counts['ensemble'].keys()
                                print "Ensembles:",ensembles
                                for ensemble in ensembles:
                                    ensemble_node = Node("Collection",name=ensemble, level=2, identifier = 'ensemble')
                                    rel = Relationship(ensemble_node,"belongs_to",mip_node)
                                    graph.create(rel)
                                    variables = conn.new_context(project=project,product=product,institute=institute,model=model,experiment=experiment,time_frequency=time_frequency,realm=realm,cmor_table=mip,ensemble=ensemble,replica=False,latest=True).facet_counts['variable'].keys()
                                    print "Variables: ", variables
                                    ctx = conn.new_context(project=project,product=product,institute=institute,model=model,experiment=experiment,time_frequency=time_frequency,realm=realm,cmor_table=mip,ensemble=ensemble,replica=False,latest=True)
                                    results = ctx.search()
                                    
                                    for variable in variables:
                                        variable_node = Node("Collection",name=variable, level=1, identifier = 'variable')
                                        rel = Relationship(variable_node,"belongs_to",ensemble_node)
                                        graph.create(rel)    
                                        print "Results for facets:",model,experiment,time_frequency,realm,mip,ensemble,variable
                                        
                                        if len(results) > 1:
                                           print " vor file access Such-Ergebinisse:",len(results)
                                        #print len(results)
                                        #print results
                                        #print 'Hits:', ctx.hit_count
                                        print "============files=========================="
                                        for i in range(0,len(results)):
                                            file_ctx = results[i].file_context()
                                            files = file_ctx.search()
                                            print "Anzahl files:",len(files)
                                            size = files.batch_size
                                            file_nodes = []
                                            for file in files:
                                                if file.filename.startswith(variable+'_'):
                                                    #print file
                                                    file_node = Node("File",name=file.file_id, level = 0, checksum=file.checksum, identifier = 'file')
                                                    rel = Relationship(file_node,"belongs_to",variable_node)
                                                    graph.create(rel)
                                                    print file.filename
                                                    # other: size,checksum_type,file_id
                                                    #print size
                                        print "======================================"


Institutes: [u'INM', u'UNSW']
Models: []
Institutes: [u'LASG-IAP', u'CCCMA', u'CMCC', u'LASG-CESS', u'INM', u'MIROC', u'NIMR-KMA', u'MPI-M']
Models: [u'MPI-ESM-P']
Experiments: [u'sstClim', u'midHolocene', u'lgm', u'sstClim4xCO2', u'past1000', u'1pctCO2', u'abrupt4xCO2', u'historical', u'piControl']
time_frequencies [u'fx', u'mon', u'day']
Realms: [u'atmos', u'land']
cmor_table: [u'fx']
Ensembles: [u'r0i0p0']
Variables:  [u'orog', u'sftlf']
Results for facets: MPI-ESM-P sstClim fx atmos fx r0i0p0 orog
============files==========================
Anzahl files: 2
orog_fx_MPI-ESM-P_sstClim_r0i0p0.nc
======================================
Results for facets: MPI-ESM-P sstClim fx atmos fx r0i0p0 sftlf
============files==========================
Anzahl files: 2
sftlf_fx_MPI-ESM-P_sstClim_r0i0p0.nc
======================================
cmor_table: [u'fx']
Ensembles: [u'r0i0p0']
Variables:  [u'mrsofc', u'sftgif']
Results for facets: MPI-ESM-P sstClim fx land fx r0i0p0 mrsofc
============files==========================
Anzahl files: 2
mrsofc_fx_MPI-ESM-P_sstClim_r0i0p0.nc
======================================
Results for facets: MPI-ESM-P sstClim fx land fx r0i0p0 sftgif
============files==========================
Anzahl files: 2
sftgif_fx_MPI-ESM-P_sstClim_r0i0p0.nc
======================================
Realms: [u'atmos']
cmor_table: [u'Amon']
Ensembles: [u'r1i1p1']
Variables:  [u'va', u'hur', u'prc', u'tas', u'rsuscs', u'uas', u'zg', u'hfss', u'rtmt', u'rsutcs', u'prsn', u'rldscs', u'pr', u'rlut', u'cli', u'rlus', u'rlds', u'cl', u'ts', u'clt', u'vas', u'clw', u'rsdscs', u'ta', u'clwvi', u'psl', u'ps', u'rsdt', u'rsds', u'hus', u'tro3', u'clivi', u'rlutcs', u'cct', u'prw', u'sfcWind', u'evspsbl', u'tauv', u'tasmin', u'tauu', u'tasmax', u'rsus', u'wap', u'ua', u'rsut', u'hfls']
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 va
============files==========================
Anzahl files: 52
va_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 hur
============files==========================
Anzahl files: 52
hur_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 prc
============files==========================
Anzahl files: 52
prc_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 tas
============files==========================
Anzahl files: 52
tas_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 rsuscs
============files==========================
Anzahl files: 52
rsuscs_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 uas
============files==========================
Anzahl files: 52
uas_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 zg
============files==========================
Anzahl files: 52
zg_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 hfss
============files==========================
Anzahl files: 52
hfss_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 rtmt
============files==========================
Anzahl files: 52
rtmt_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 rsutcs
============files==========================
Anzahl files: 52
rsutcs_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 prsn
============files==========================
Anzahl files: 52
prsn_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 rldscs
============files==========================
Anzahl files: 52
rldscs_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 pr
============files==========================
Anzahl files: 52
pr_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 rlut
============files==========================
Anzahl files: 52
rlut_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 cli
============files==========================
Anzahl files: 52
cli_Amon_MPI-ESM-P_sstClim_r1i1p1_186001-186912.nc
cli_Amon_MPI-ESM-P_sstClim_r1i1p1_187001-187912.nc
cli_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-185912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 rlus
============files==========================
Anzahl files: 52
rlus_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 rlds
============files==========================
Anzahl files: 52
rlds_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 cl
============files==========================
Anzahl files: 52
cl_Amon_MPI-ESM-P_sstClim_r1i1p1_186001-186912.nc
cl_Amon_MPI-ESM-P_sstClim_r1i1p1_187001-187912.nc
cl_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-185912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 ts
============files==========================
Anzahl files: 52
ts_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 clt
============files==========================
Anzahl files: 52
clt_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 vas
============files==========================
Anzahl files: 52
vas_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 clw
============files==========================
Anzahl files: 52
clw_Amon_MPI-ESM-P_sstClim_r1i1p1_187001-187912.nc
clw_Amon_MPI-ESM-P_sstClim_r1i1p1_186001-186912.nc
clw_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-185912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 rsdscs
============files==========================
Anzahl files: 52
rsdscs_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 ta
============files==========================
Anzahl files: 52
ta_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 clwvi
============files==========================
Anzahl files: 52
clwvi_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 psl
============files==========================
Anzahl files: 52
psl_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 ps
============files==========================
Anzahl files: 52
ps_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 rsdt
============files==========================
Anzahl files: 52
rsdt_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 rsds
============files==========================
Anzahl files: 52
rsds_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 hus
============files==========================
Anzahl files: 52
hus_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 tro3
============files==========================
Anzahl files: 52
tro3_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 clivi
============files==========================
Anzahl files: 52
clivi_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 rlutcs
============files==========================
Anzahl files: 52
rlutcs_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 cct
============files==========================
Anzahl files: 52
cct_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 prw
============files==========================
Anzahl files: 52
prw_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 sfcWind
============files==========================
Anzahl files: 52
sfcWind_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 evspsbl
============files==========================
Anzahl files: 52
evspsbl_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 tauv
============files==========================
Anzahl files: 52
tauv_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 tasmin
============files==========================
Anzahl files: 52
tasmin_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 tauu
============files==========================
Anzahl files: 52
tauu_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 tasmax
============files==========================
Anzahl files: 52
tasmax_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 rsus
============files==========================
Anzahl files: 52
rsus_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 wap
============files==========================
Anzahl files: 52
wap_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 ua
============files==========================
Anzahl files: 52
ua_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 rsut
============files==========================
Anzahl files: 52
rsut_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Results for facets: MPI-ESM-P sstClim mon atmos Amon r1i1p1 hfls
============files==========================
Anzahl files: 52
hfls_Amon_MPI-ESM-P_sstClim_r1i1p1_185001-187912.nc
======================================
Realms: [u'atmos']
cmor_table: [u'day']
Ensembles: [u'r1i1p1']
Variables:  [u'pr', u'psl', u'tas', u'sfcWind', u'tasmin', u'tasmax']
Results for facets: MPI-ESM-P sstClim day atmos day r1i1p1 pr
============files==========================
Anzahl files: 6
pr_day_MPI-ESM-P_sstClim_r1i1p1_18500101-18791231.nc
======================================
Results for facets: MPI-ESM-P sstClim day atmos day r1i1p1 psl
============files==========================
Anzahl files: 6
psl_day_MPI-ESM-P_sstClim_r1i1p1_18500101-18791231.nc
======================================
Results for facets: MPI-ESM-P sstClim day atmos day r1i1p1 tas
============files==========================
Anzahl files: 6
tas_day_MPI-ESM-P_sstClim_r1i1p1_18500101-18791231.nc
======================================
Results for facets: MPI-ESM-P sstClim day atmos day r1i1p1 sfcWind
============files==========================
Anzahl files: 6
sfcWind_day_MPI-ESM-P_sstClim_r1i1p1_18500101-18791231.nc
======================================
Results for facets: MPI-ESM-P sstClim day atmos day r1i1p1 tasmin
============files==========================
/home/stephan/anaconda/envs/scientific1/lib/python2.7/site-packages/pyesgf/search/connection.py:91: UserWarning: Old-style SearchContext URL specified.  In future please specify the URL excluding the "/search" endpoint.
  warnings.warn('Old-style SearchContext URL specified.  '
---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-1-d3f7fc4657d7> in <module>()
     78                                         for i in range(0,len(results)):
     79                                             file_ctx = results[i].file_context()
---> 80                                             files = file_ctx.search()
     81                                             print "Anzahl files:",len(files)
     82                                             size = files.batch_size

/home/stephan/anaconda/envs/scientific1/lib/python2.7/site-packages/pyesgf/search/context.pyc in search(self, **constraints)
    121             sc = self
    122 
--> 123         self.__update_counts()
    124 
    125         return ResultSet(sc)

/home/stephan/anaconda/envs/scientific1/lib/python2.7/site-packages/pyesgf/search/context.pyc in __update_counts(self)
    197         query_dict['facets'] = '*'
    198 
--> 199         response = self.connection.send_search(query_dict, limit=0)
    200         for facet, counts in (
    201             response['facet_counts']['facet_fields'].items()):

/home/stephan/anaconda/envs/scientific1/lib/python2.7/site-packages/pyesgf/search/connection.pyc in send_search(self, query_dict, limit, offset, shards)
    103         """
    104         full_query = self._build_query(query_dict, limit, offset, shards)
--> 105         response = self._send_query('search', full_query)
    106         ret = json.load(response)
    107 

/home/stephan/anaconda/envs/scientific1/lib/python2.7/site-packages/pyesgf/search/connection.pyc in _send_query(self, endpoint, full_query)
    142 
    143         try:
--> 144             response = urllib2.urlopen(query_url)
    145         except urllib2.HTTPError, err:
    146             log.warn("HTTP request received error code: %s" % err.code)

/home/stephan/anaconda/envs/scientific1/lib/python2.7/urllib2.pyc in urlopen(url, data, timeout, cafile, capath, cadefault, context)
    152     else:
    153         opener = _opener
--> 154     return opener.open(url, data, timeout)
    155 
    156 def install_opener(opener):

/home/stephan/anaconda/envs/scientific1/lib/python2.7/urllib2.pyc in open(self, fullurl, data, timeout)
    429             req = meth(req)
    430 
--> 431         response = self._open(req, data)
    432 
    433         # post-process response

/home/stephan/anaconda/envs/scientific1/lib/python2.7/urllib2.pyc in _open(self, req, data)
    447         protocol = req.get_type()
    448         result = self._call_chain(self.handle_open, protocol, protocol +
--> 449                                   '_open', req)
    450         if result:
    451             return result

/home/stephan/anaconda/envs/scientific1/lib/python2.7/urllib2.pyc in _call_chain(self, chain, kind, meth_name, *args)
    407             func = getattr(handler, meth_name)
    408 
--> 409             result = func(*args)
    410             if result is not None:
    411                 return result

/home/stephan/anaconda/envs/scientific1/lib/python2.7/urllib2.pyc in http_open(self, req)
   1225 
   1226     def http_open(self, req):
-> 1227         return self.do_open(httplib.HTTPConnection, req)
   1228 
   1229     http_request = AbstractHTTPHandler.do_request_

/home/stephan/anaconda/envs/scientific1/lib/python2.7/urllib2.pyc in do_open(self, http_class, req, **http_conn_args)
   1192 
   1193         try:
-> 1194             h.request(req.get_method(), req.get_selector(), req.data, headers)
   1195         except socket.error, err: # XXX what error?
   1196             h.close()

/home/stephan/anaconda/envs/scientific1/lib/python2.7/httplib.pyc in request(self, method, url, body, headers)
   1051     def request(self, method, url, body=None, headers={}):
   1052         """Send a complete request to the server."""
-> 1053         self._send_request(method, url, body, headers)
   1054 
   1055     def _set_content_length(self, body, method):

/home/stephan/anaconda/envs/scientific1/lib/python2.7/httplib.pyc in _send_request(self, method, url, body, headers)
   1091         for hdr, value in headers.iteritems():
   1092             self.putheader(hdr, value)
-> 1093         self.endheaders(body)
   1094 
   1095     def getresponse(self, buffering=False):

/home/stephan/anaconda/envs/scientific1/lib/python2.7/httplib.pyc in endheaders(self, message_body)
   1047         else:
   1048             raise CannotSendHeader()
-> 1049         self._send_output(message_body)
   1050 
   1051     def request(self, method, url, body=None, headers={}):

/home/stephan/anaconda/envs/scientific1/lib/python2.7/httplib.pyc in _send_output(self, message_body)
    891             msg += message_body
    892             message_body = None
--> 893         self.send(msg)
    894         if message_body is not None:
    895             #message_body was not a string (i.e. it is a file) and

/home/stephan/anaconda/envs/scientific1/lib/python2.7/httplib.pyc in send(self, data)
    853         if self.sock is None:
    854             if self.auto_open:
--> 855                 self.connect()
    856             else:
    857                 raise NotConnected()

/home/stephan/anaconda/envs/scientific1/lib/python2.7/httplib.pyc in connect(self)
    830         """Connect to the host and port specified in __init__."""
    831         self.sock = self._create_connection((self.host,self.port),
--> 832                                            self.timeout, self.source_address)
    833 
    834         if self._tunnel_host:

/home/stephan/anaconda/envs/scientific1/lib/python2.7/socket.pyc in create_connection(address, timeout, source_address)
    555     host, port = address
    556     err = None
--> 557     for res in getaddrinfo(host, port, 0, SOCK_STREAM):
    558         af, socktype, proto, canonname, sa = res
    559         sock = None

KeyboardInterrupt: 

In [ ]: