In [1]:

    

import pandas as pd
from pandas import Series, DataFrame
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.cbook as cbook
import csv
import re
import cStringIO
import urllib2
import parser
import pdb
import random
import datetime as dt
from datetime import datetime
from pylab import *
from owslib.csw import CatalogueServiceWeb
from owslib.wms import WebMapService
from owslib.sos import SensorObservationService
from owslib.etree import etree
from owslib import fes
import netCDF4

# from: https://github.com/ioos/catalog/blob/master/ioos_catalog/tasks/reindex_services.py#L43-L58
from datetime import datetime
from urlparse import urlparse

import requests
import xml.etree.ElementTree as ET

from owslib import fes, csw
from owslib.util import nspath_eval
from owslib.namespaces import Namespaces
#import ioos_catalog
#from ioos_catalog import app,db


    

In [2]:

    

#This is a collection of lists that we will need to examine Catalogs 

web_service_strings = ['urn:x-esri:specification:ServiceType:OPeNDAP',
                       'urn:x-esri:specification:ServiceType:odp:url',
                       'urn:x-esri:specification:ServiceType:WMS',
                       'urn:x-esri:specification:ServiceType:wms:url',
                       'urn:x-esri:specification:ServiceType:sos:url',
                       'urn:x-esri:specification:ServiceType:wcs:url']


services =      {'SOS'              : 'urn:x-esri:specification:ServiceType:sos:url',
                 'WMS'              : 'urn:x-esri:specification:ServiceType:wms:url',
                 'WCS'              : 'urn:x-esri:specification:ServiceType:wcs:url',
                 'DAP'              : 'urn:x-esri:specification:ServiceType:odp:url' }

# This looks like a good notebook to work from
# https://www.wakari.io/sharing/bundle/rsignell/Model_search


    

In [3]:

    

#This cell lists catalog endpoints.  As CSW's are discovered within the larger
#    IOOS Umbrealla, this list is updated by the IOOS Program Office here:
#    https://github.com/ioos/system-test/wiki/Service-Registries-and-Data-Catalogs

#endpoint = 'http://data.nodc.noaa.gov/geoportal/csw'  # NODC Geoportal: collection level
#endpoint = 'http://geodiscover.cgdi.ca/wes/serviceManagerCSW/csw'  # NRCAN 
#endpoint = 'http://geoport.whoi.edu/gi-cat/services/cswiso' # USGS Woods Hole GI_CAT
#endpoint = 'http://cida.usgs.gov/gdp/geonetwork/srv/en/csw' # USGS CIDA Geonetwork
#endpoint = 'http://www.nodc.noaa.gov/geoportal/csw'   # NODC Geoportal: granule level
#endpoint = 'http://cmgds.marine.usgs.gov/geonetwork/srv/en/csw'  # USGS Coastal & Marine Program Geonetwork
#endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw' # NGDC Geoportal
#endpoint = 'http://www.ncdc.noaa.gov/cdo-web/api/v2/' #NCDC CDO Web Services
#endpoint = 'http://geo.gov.ckan.org/csw' #CKAN Testing Site for new Data.gov
#endpoint = 'https://edg.epa.gov/metadata/csw' #EPA
#endpoint = 'http://geoport.whoi.edu/geoportal/csw' #WHOI Geoportal
#endpoint = 'http://cwic.csiss.gmu.edu/cwicv1/discovery' #CWIC
#endpoint = 'http://portal.westcoastoceans.org/connect/' #West Coast Governors Alliance (Based on ESRI Geoportal back end
#print out version
#endpoint = 'http://gcmdsrv.gsfc.nasa.gov/csw' #NASA's Global Change Master Directory (GCMD) CSW Service (Requires Authorization)
#endpoint = 'http://gcmdsrv3.gsfc.nasa.gov/csw' #NASA's Global Change Master Directory (GCMD) CSW Service (Requires Authorization)
#endpoint = 'https://data.noaa.gov/csw' #data.noaa.gov csw

endpoints = ['http://www.nodc.noaa.gov/geoportal/csw',
             'http://www.ngdc.noaa.gov/geoportal/csw',
             'http://catalog.data.gov/csw-all',
             'http://cwic.csiss.gmu.edu/cwicv1/discovery',
             'http://geoport.whoi.edu/geoportal/csw',
             'https://edg.epa.gov/metadata/csw',
             'http://cmgds.marine.usgs.gov/geonetwork/srv/en/csw',
             'http://cida.usgs.gov/gdp/geonetwork/srv/en/csw',
             'http://geodiscover.cgdi.ca/wes/serviceManagerCSW/csw', 
             'http://geoport.whoi.edu/gi-cat/services/cswiso']


    

In [4]:

    

def service_urls(records,service_string='urn:x-esri:specification:ServiceType:wms:url'):
    urls=[]
    for key,rec in records.iteritems():
        #create a generator object, and iterate through it until the match is found
        #if not found, gets the default value (here "none")
        url = next((d['url'] for d in rec.references if d['scheme'] == service_string), None)
        if url is not None:
            urls.append(url)
    return urls
records1 = []
titles1 = []
lenrecords1 = []
lentitles1 = []
list1 = []
list2 = []
list3 = []
dict1 = {}
dict2 = {}
dict3 = {}
list4 = []
lenurls = []

for endpoint in endpoints:
    try:
        csw = CatalogueServiceWeb(endpoint,timeout=100)
        csw.getrecords2(maxrecords = 100)
        for web_service_string in web_service_strings:
            urls1 = service_urls(csw.records,service_string=web_service_string)
            list3.append(urls1)
            list1.append(web_service_string)
            list2.append(endpoint)
            list4.append(len(urls1))
            dict2['Service_URL']= list1
            dict2['endpoint'] = list2
            dict2['urls'] = list3
            dict2['number_urls'] = list4
    except Exception, ex1:
        print 'Error'
        
        
    
    #dict2['lenrecords'] = lenrecords1
    
#print dict2        
print pd.DataFrame(dict2)
#print pd.DataFrame(dict2.keys())


    

    




Error
                                     Service_URL  \
0   urn:x-esri:specification:ServiceType:OPeNDAP   
1   urn:x-esri:specification:ServiceType:odp:url   
2       urn:x-esri:specification:ServiceType:WMS   
3   urn:x-esri:specification:ServiceType:wms:url   
4   urn:x-esri:specification:ServiceType:sos:url   
5   urn:x-esri:specification:ServiceType:wcs:url   
6   urn:x-esri:specification:ServiceType:OPeNDAP   
7   urn:x-esri:specification:ServiceType:odp:url   
8       urn:x-esri:specification:ServiceType:WMS   
9   urn:x-esri:specification:ServiceType:wms:url   
10  urn:x-esri:specification:ServiceType:sos:url   
11  urn:x-esri:specification:ServiceType:wcs:url   
12  urn:x-esri:specification:ServiceType:OPeNDAP   
13  urn:x-esri:specification:ServiceType:odp:url   
14      urn:x-esri:specification:ServiceType:WMS   
15  urn:x-esri:specification:ServiceType:wms:url   
16  urn:x-esri:specification:ServiceType:sos:url   
17  urn:x-esri:specification:ServiceType:wcs:url   
18  urn:x-esri:specification:ServiceType:OPeNDAP   
19  urn:x-esri:specification:ServiceType:odp:url   
20      urn:x-esri:specification:ServiceType:WMS   
21  urn:x-esri:specification:ServiceType:wms:url   
22  urn:x-esri:specification:ServiceType:sos:url   
23  urn:x-esri:specification:ServiceType:wcs:url   
24  urn:x-esri:specification:ServiceType:OPeNDAP   
25  urn:x-esri:specification:ServiceType:odp:url   
26      urn:x-esri:specification:ServiceType:WMS   
27  urn:x-esri:specification:ServiceType:wms:url   
28  urn:x-esri:specification:ServiceType:sos:url   
29  urn:x-esri:specification:ServiceType:wcs:url   
30  urn:x-esri:specification:ServiceType:OPeNDAP   
31  urn:x-esri:specification:ServiceType:odp:url   
32      urn:x-esri:specification:ServiceType:WMS   
33  urn:x-esri:specification:ServiceType:wms:url   
34  urn:x-esri:specification:ServiceType:sos:url   
35  urn:x-esri:specification:ServiceType:wcs:url   
36  urn:x-esri:specification:ServiceType:OPeNDAP   
37  urn:x-esri:specification:ServiceType:odp:url   
38      urn:x-esri:specification:ServiceType:WMS   
39  urn:x-esri:specification:ServiceType:wms:url   
40  urn:x-esri:specification:ServiceType:sos:url   
41  urn:x-esri:specification:ServiceType:wcs:url   
42  urn:x-esri:specification:ServiceType:OPeNDAP   
43  urn:x-esri:specification:ServiceType:odp:url   
44      urn:x-esri:specification:ServiceType:WMS   
45  urn:x-esri:specification:ServiceType:wms:url   
46  urn:x-esri:specification:ServiceType:sos:url   
47  urn:x-esri:specification:ServiceType:wcs:url   
48  urn:x-esri:specification:ServiceType:OPeNDAP   
49  urn:x-esri:specification:ServiceType:odp:url   
50      urn:x-esri:specification:ServiceType:WMS   
51  urn:x-esri:specification:ServiceType:wms:url   
52  urn:x-esri:specification:ServiceType:sos:url   
53  urn:x-esri:specification:ServiceType:wcs:url   

                                             endpoint  number_urls  \
0              http://www.nodc.noaa.gov/geoportal/csw          100   
1              http://www.nodc.noaa.gov/geoportal/csw            0   
2              http://www.nodc.noaa.gov/geoportal/csw          100   
3              http://www.nodc.noaa.gov/geoportal/csw            0   
4              http://www.nodc.noaa.gov/geoportal/csw            0   
5              http://www.nodc.noaa.gov/geoportal/csw            0   
6              http://www.ngdc.noaa.gov/geoportal/csw            0   
7              http://www.ngdc.noaa.gov/geoportal/csw            0   
8              http://www.ngdc.noaa.gov/geoportal/csw            0   
9              http://www.ngdc.noaa.gov/geoportal/csw            0   
10             http://www.ngdc.noaa.gov/geoportal/csw            0   
11             http://www.ngdc.noaa.gov/geoportal/csw            0   
12                    http://catalog.data.gov/csw-all            0   
13                    http://catalog.data.gov/csw-all            0   
14                    http://catalog.data.gov/csw-all            0   
15                    http://catalog.data.gov/csw-all            0   
16                    http://catalog.data.gov/csw-all            0   
17                    http://catalog.data.gov/csw-all            0   
18              http://geoport.whoi.edu/geoportal/csw            0   
19              http://geoport.whoi.edu/geoportal/csw            0   
20              http://geoport.whoi.edu/geoportal/csw            0   
21              http://geoport.whoi.edu/geoportal/csw            0   
22              http://geoport.whoi.edu/geoportal/csw            0   
23              http://geoport.whoi.edu/geoportal/csw            0   
24                   https://edg.epa.gov/metadata/csw            0   
25                   https://edg.epa.gov/metadata/csw            0   
26                   https://edg.epa.gov/metadata/csw            0   
27                   https://edg.epa.gov/metadata/csw            0   
28                   https://edg.epa.gov/metadata/csw            0   
29                   https://edg.epa.gov/metadata/csw            0   
30  http://cmgds.marine.usgs.gov/geonetwork/srv/en...            0   
31  http://cmgds.marine.usgs.gov/geonetwork/srv/en...            0   
32  http://cmgds.marine.usgs.gov/geonetwork/srv/en...            0   
33  http://cmgds.marine.usgs.gov/geonetwork/srv/en...            0   
34  http://cmgds.marine.usgs.gov/geonetwork/srv/en...            0   
35  http://cmgds.marine.usgs.gov/geonetwork/srv/en...            0   
36     http://cida.usgs.gov/gdp/geonetwork/srv/en/csw            0   
37     http://cida.usgs.gov/gdp/geonetwork/srv/en/csw            0   
38     http://cida.usgs.gov/gdp/geonetwork/srv/en/csw            0   
39     http://cida.usgs.gov/gdp/geonetwork/srv/en/csw            0   
40     http://cida.usgs.gov/gdp/geonetwork/srv/en/csw            0   
41     http://cida.usgs.gov/gdp/geonetwork/srv/en/csw            0   
42  http://geodiscover.cgdi.ca/wes/serviceManagerC...            0   
43  http://geodiscover.cgdi.ca/wes/serviceManagerC...            0   
44  http://geodiscover.cgdi.ca/wes/serviceManagerC...            0   
45  http://geodiscover.cgdi.ca/wes/serviceManagerC...            0   
46  http://geodiscover.cgdi.ca/wes/serviceManagerC...            0   
47  http://geodiscover.cgdi.ca/wes/serviceManagerC...            0   
48     http://geoport.whoi.edu/gi-cat/services/cswiso            0   
49     http://geoport.whoi.edu/gi-cat/services/cswiso            0   
50     http://geoport.whoi.edu/gi-cat/services/cswiso            0   
51     http://geoport.whoi.edu/gi-cat/services/cswiso            0   
52     http://geoport.whoi.edu/gi-cat/services/cswiso            0   
53     http://geoport.whoi.edu/gi-cat/services/cswiso            0   

                                                 urls  
0   [http://data.nodc.noaa.gov/thredds/dodsC/woa/g...  
1                                                  []  
2   [http://data.nodc.noaa.gov/thredds/wms/woa/gin...  
3                                                  []  
4                                                  []  
5                                                  []  
6                                                  []  
7                                                  []  
8                                                  []  
9                                                  []  
10                                                 []  
11                                                 []  
12                                                 []  
13                                                 []  
14                                                 []  
15                                                 []  
16                                                 []  
17                                                 []  
18                                                 []  
19                                                 []  
20                                                 []  
21                                                 []  
22                                                 []  
23                                                 []  
24                                                 []  
25                                                 []  
26                                                 []  
27                                                 []  
28                                                 []  
29                                                 []  
30                                                 []  
31                                                 []  
32                                                 []  
33                                                 []  
34                                                 []  
35                                                 []  
36                                                 []  
37                                                 []  
38                                                 []  
39                                                 []  
40                                                 []  
41                                                 []  
42                                                 []  
43                                                 []  
44                                                 []  
45                                                 []  
46                                                 []  
47                                                 []  
48                                                 []  
49                                                 []  
50                                                 []  
51                                                 []  
52                                                 []  
53                                                 []  

[54 rows x 4 columns]