notebook.community

Edit and run

Creating Geopackage Layers

  • Landung Setiawan 5/27/2016
  • Updated 6/29/2016
  • Note: In order for fiona to be able to read and write geopackage, numpy 1.10.0 and gdal 1.11.0 or greater is required, however, gdal cannot be 2.0.0 or greater!
  • Creating the environment
    conda create -n gpkg -c conda-forge numpy=1.10.0 fiona=1.6.4 gdal=1.11.4 geopandas matplotlib
source activate gpkg
conda install ipython notebook anaconda-client
conda install -c auto gsconfig=0.6.7





In [1]:



    


%matplotlib inline
# Import the necessary libraries
import csv, os
from shapely.geometry import Point, mapping
import fiona, shapely
from fiona import Collection
import numpy as np



    


















    






/home/lsetiawan/anaconda2/envs/OOI/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.
  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')

















In [2]:



    


print "fiona version: {}".format(fiona.__version__)
print "shapely version: {}".format(shapely.__version__)
print "gdal version: {}".format(fiona.__gdal_version__)
print "numpy version: {}".format(np.__version__)



    


















    






fiona version: 1.6.4
shapely version: 1.5.15
gdal version: 1.11.4
numpy version: 1.10.0

















In [3]:



    


# Assign file_path
pth = "/mnt/hgfs/shared_ubuntu/APL/OOI/OOI_ipynb/"
fname = 'Nanoos.gpkg'
fcsv = "OOI_Assets.csv"

Reading csv and printing as dictionary





In [4]:



    


with open(os.path.join(pth,fcsv),'rb') as f:
    reader = csv.DictReader(f)
    for row in reader:
        print row # Notice that numbers are strings in this case



    


















    






{'Description': 'OR Inshore Surface Mooring', 'URL': 'http://oceanobservatories.org/site/ce01issm/', 'Provider URL': 'http://oceanobservatories.org', 'Region': 'Newport, OOI Oregon Line', 'Lon': '-124.0957', 'State / Province': 'Oregon', 'Provider': 'OOI', 'Lat': '44.6585', 'Provider Type': 'Academic', 'Type': 'Buoy', 'ID': 'OOI_CE01ISSM', 'Name': 'CE01ISSM'}
{'Description': 'OR Shelf Cabled Benthic Experiment Package', 'URL': 'http://oceanobservatories.org/site/ce02shbp/', 'Provider URL': 'http://oceanobservatories.org', 'Region': 'Newport, OOI Oregon Line', 'Lon': '-124.31', 'State / Province': 'Oregon', 'Provider': 'OOI', 'Lat': '44.64', 'Provider Type': 'Academic', 'Type': 'Seabed Cabled Platform', 'ID': 'OOI_CE02SHBP', 'Name': 'CE02SHBP'}
{'Description': 'OR Shelf Surface Mooring', 'URL': 'http://oceanobservatories.org/site/ce02shsm/', 'Provider URL': 'http://oceanobservatories.org', 'Region': 'Newport, OOI Oregon Line', 'Lon': '-124.31', 'State / Province': 'Oregon', 'Provider': 'OOI', 'Lat': '44.64', 'Provider Type': 'Academic', 'Type': 'Buoy', 'ID': 'OOI_CE02SHSM', 'Name': 'CE02SHSM'}
{'Description': 'OR Offshore Cabled Benthic Experiment Package', 'URL': 'http://oceanobservatories.org/site/ce04osbp/', 'Provider URL': 'http://oceanobservatories.org', 'Region': 'Newport, OOI Oregon Line', 'Lon': '-124.95', 'State / Province': 'Oregon', 'Provider': 'OOI', 'Lat': '44.37', 'Provider Type': 'Academic', 'Type': 'Seabed Cabled Platform', 'ID': 'OOI_CE04OSBP', 'Name': 'CE04OSBP'}
{'Description': 'OR Offshore Surface Mooring', 'URL': 'http://oceanobservatories.org/site/ce04ossm/', 'Provider URL': 'http://oceanobservatories.org', 'Region': 'Newport, OOI Oregon Line', 'Lon': '-124.95', 'State / Province': 'Oregon', 'Provider': 'OOI', 'Lat': '44.37', 'Provider Type': 'Academic', 'Type': 'Buoy', 'ID': 'OOI_CE04OSSM', 'Name': 'CE04OSSM'}
{'Description': 'WA Inshore Surface Mooring', 'URL': 'http://oceanobservatories.org/site/ce06issm/', 'Provider URL': 'http://oceanobservatories.org', 'Region': 'Grays Harbor, OOI Washington Line', 'Lon': '-124.27', 'State / Province': 'Washington', 'Provider': 'OOI', 'Lat': '47.13', 'Provider Type': 'Academic', 'Type': 'Buoy', 'ID': 'OOI_CE06ISSM', 'Name': 'CE06ISSM'}
{'Description': 'WA Shelf Surface Mooring', 'URL': 'http://oceanobservatories.org/site/ce07shsm/', 'Provider URL': 'http://oceanobservatories.org', 'Region': 'Grays Harbor, OOI Washington Line', 'Lon': '-124.57', 'State / Province': 'Washington', 'Provider': 'OOI', 'Lat': '46.99', 'Provider Type': 'Academic', 'Type': 'Buoy', 'ID': 'OOI_CE07SHSM', 'Name': 'CE07SHSM'}
{'Description': 'WA Offshore Profiler Mooring', 'URL': 'http://oceanobservatories.org/site/ce09ospm/', 'Provider URL': 'http://oceanobservatories.org', 'Region': 'Grays Harbor, OOI Washington Line', 'Lon': '-124.9715', 'State / Province': 'Washington', 'Provider': 'OOI', 'Lat': '46.8517', 'Provider Type': 'Academic', 'Type': 'Buoy', 'ID': 'OOI_CE09OSPM', 'Name': 'CE09OSPM'}
{'Description': 'WA Offshore Surface Mooring', 'URL': 'http://oceanobservatories.org/site/ce09ossm/', 'Provider URL': 'http://oceanobservatories.org', 'Region': 'Grays Harbor, OOI Washington Line', 'Lon': '-124.97', 'State / Province': 'Washington', 'Provider': 'OOI', 'Lat': '46.85', 'Provider Type': 'Academic', 'Type': 'Buoy', 'ID': 'OOI_CE09OSSM', 'Name': 'CE09OSSM'}

Use shapely to make points

Since csv module doesn't distinguish between types, shapely is used to make points





In [5]:



    


with open(os.path.join(pth,fcsv), 'rb') as f:
    reader = csv.DictReader(f)
    for row in reader:
        point = Point(float(row['Lon']),float(row['Lat']))
        print point



    


















    






POINT (-124.0957 44.6585)
POINT (-124.31 44.64)
POINT (-124.31 44.64)
POINT (-124.95 44.37)
POINT (-124.95 44.37)
POINT (-124.27 47.13)
POINT (-124.57 46.99)
POINT (-124.9715 46.8517)
POINT (-124.97 46.85)

Geopandas reading a geopackage





In [6]:



    


import pandas as pd
from geopandas import GeoDataFrame
from shapely.geometry import Point
import matplotlib.pyplot as plt
import geopandas as gpd
import pyproj



    











In [7]:



    


print "geopandas version: {}".format(gpd.__version__)



    


















    






geopandas version: 0.1.1

















In [8]:



    


# Test reading geopackage
geopackage = gpd.read_file(os.path.join(pth,fname))



    











In [9]:



    


geopackage.head(2)



    


















    
Out[9]:










  
    

      
      Description
      ID
      Lat
      Lon
      Name
      Region
      Type
      URL
      geometry
    

  
  
    

      0
      UW/NANOOS NEMO Subsurface profiler with NOAA E...
      APL_Nemo
      47.9740
      -124.9550
      APL-UW NEMO-ESP Profiler
      La Push
      Buoy
      None
      POINT (-124.955 47.974)
    

    

      1
      Ćháʔba· UW/NANOOS Moored Buoy near La Push
      APL_Chaba
      47.9672
      -124.9502
      APL-UW Ćháʔba·
      La Push
      Buoy
      http://wavechasers.apl.washington.edu/projects...
      POINT (-124.9502 47.9672)

Write geopandas dataframe to geopackage





In [10]:



    


df = pd.read_csv(os.path.join(pth,fcsv))



    











In [11]:



    


# Assign CRS, retrieved from epsg.io, the example below is EPSG:4326
crs = 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]'
geometry = [Point(xy) for xy in zip(df.Lon, df.Lat)]
geo_df = GeoDataFrame(df, crs=crs, geometry=geometry)
print "Original Column Header: {}\n".format(geo_df.columns.values)


# Renamed the problematic keys
renamed = geo_df.rename(columns={'Provider URL':'Provider_URL',
                                 'Provider':'Provider',
                                 'Provider Type':'Provider_Type',
                                 'State / Province':'State_or_Province'})
print "Renamed Column Header: {}".format(renamed.columns.values)

# Removing the problematic keys
# Problematic keys can either be renamed or removed.
# package = geo_df.drop(geo_df.columns[[8,9,10,11]],axis=1)
# print package.columns.values



    


















    






Original Column Header: ['ID' 'Name' 'Description' 'Type' 'Lat' 'Lon' 'URL' 'Region'
 'State / Province' 'Provider' 'Provider Type' 'Provider URL' 'geometry']

Renamed Column Header: ['ID' 'Name' 'Description' 'Type' 'Lat' 'Lon' 'URL' 'Region'
 'State_or_Province' 'Provider' 'Provider_Type' 'Provider_URL' 'geometry']

















In [12]:



    


# Write the renamed geodataframe to a geopackage
renamed.to_file('OOI_Assets.gpkg',driver='GPKG')



    











In [13]:



    


# Check if the geopackage was written correctly
test = gpd.read_file('OOI_Assets.gpkg')
test



    


















    
Out[13]:










  
    

      
      Description
      ID
      Lat
      Lon
      Name
      Provider
      Provider_Type
      Provider_URL
      Region
      State_or_Province
      Type
      URL
      geometry
    

  
  
    

      0
      OR Inshore Surface Mooring
      OOI_CE01ISSM
      44.6585
      -124.0957
      CE01ISSM
      OOI
      Academic
      http://oceanobservatories.org
      Newport, OOI Oregon Line
      Oregon
      Buoy
      http://oceanobservatories.org/site/ce01issm/
      POINT (-124.0957 44.6585)
    

    

      1
      OR Shelf Cabled Benthic Experiment Package
      OOI_CE02SHBP
      44.6400
      -124.3100
      CE02SHBP
      OOI
      Academic
      http://oceanobservatories.org
      Newport, OOI Oregon Line
      Oregon
      Seabed Cabled Platform
      http://oceanobservatories.org/site/ce02shbp/
      POINT (-124.31 44.64)
    

    

      2
      OR Shelf Surface Mooring
      OOI_CE02SHSM
      44.6400
      -124.3100
      CE02SHSM
      OOI
      Academic
      http://oceanobservatories.org
      Newport, OOI Oregon Line
      Oregon
      Buoy
      http://oceanobservatories.org/site/ce02shsm/
      POINT (-124.31 44.64)
    

    

      3
      OR Offshore Cabled Benthic Experiment Package
      OOI_CE04OSBP
      44.3700
      -124.9500
      CE04OSBP
      OOI
      Academic
      http://oceanobservatories.org
      Newport, OOI Oregon Line
      Oregon
      Seabed Cabled Platform
      http://oceanobservatories.org/site/ce04osbp/
      POINT (-124.95 44.37)
    

    

      4
      OR Offshore Surface Mooring
      OOI_CE04OSSM
      44.3700
      -124.9500
      CE04OSSM
      OOI
      Academic
      http://oceanobservatories.org
      Newport, OOI Oregon Line
      Oregon
      Buoy
      http://oceanobservatories.org/site/ce04ossm/
      POINT (-124.95 44.37)
    

    

      5
      WA Inshore Surface Mooring
      OOI_CE06ISSM
      47.1300
      -124.2700
      CE06ISSM
      OOI
      Academic
      http://oceanobservatories.org
      Grays Harbor, OOI Washington Line
      Washington
      Buoy
      http://oceanobservatories.org/site/ce06issm/
      POINT (-124.27 47.13)
    

    

      6
      WA Shelf Surface Mooring
      OOI_CE07SHSM
      46.9900
      -124.5700
      CE07SHSM
      OOI
      Academic
      http://oceanobservatories.org
      Grays Harbor, OOI Washington Line
      Washington
      Buoy
      http://oceanobservatories.org/site/ce07shsm/
      POINT (-124.57 46.99)
    

    

      7
      WA Offshore Profiler Mooring
      OOI_CE09OSPM
      46.8517
      -124.9715
      CE09OSPM
      OOI
      Academic
      http://oceanobservatories.org
      Grays Harbor, OOI Washington Line
      Washington
      Buoy
      http://oceanobservatories.org/site/ce09ospm/
      POINT (-124.9715 46.8517)
    

    

      8
      WA Offshore Surface Mooring
      OOI_CE09OSSM
      46.8500
      -124.9700
      CE09OSSM
      OOI
      Academic
      http://oceanobservatories.org
      Grays Harbor, OOI Washington Line
      Washington
      Buoy
      http://oceanobservatories.org/site/ce09ossm/
      POINT (-124.97 46.85)

Uploading Geopackage to GeoServer





In [14]:



    


# Import the Catalog module
from geoserver.catalog import Catalog
# Import subprocess to use cURL REST API since gsconfig, doesn't seem to have this capability anymore
import subprocess



    











In [15]:



    


# Retrieve catalog from Geoserver Instance via REST (REpresentational State Transfer)
cat = Catalog("http://data.nanoos.org/geoserver2_8/rest", username='####', password='####')



    











In [16]:



    


# Get list of workspaces
print cat.get_workspaces()



    


















    






[nanoos_dev @ http://data.nanoos.org/geoserver2_8/rest/workspaces/nanoos_dev.xml, czo @ http://data.nanoos.org/geoserver2_8/rest/workspaces/czo.xml, otnnep @ http://data.nanoos.org/geoserver2_8/rest/workspaces/otnnep.xml, nvs_assets @ http://data.nanoos.org/geoserver2_8/rest/workspaces/nvs_assets.xml, xan @ http://data.nanoos.org/geoserver2_8/rest/workspaces/xan.xml, basemaps @ http://data.nanoos.org/geoserver2_8/rest/workspaces/basemaps.xml, oa @ http://data.nanoos.org/geoserver2_8/rest/workspaces/oa.xml, crb @ http://data.nanoos.org/geoserver2_8/rest/workspaces/crb.xml, partners @ http://data.nanoos.org/geoserver2_8/rest/workspaces/partners.xml, nvs @ http://data.nanoos.org/geoserver2_8/rest/workspaces/nvs.xml, drb @ http://data.nanoos.org/geoserver2_8/rest/workspaces/drb.xml]

















In [17]:



    


# Get workspace
nvs = cat.get_workspace('nvs_assets')
print nvs.name



    


















    






nvs_assets

















In [18]:



    


# Create the geopackage datastore
gpkg_ds = cat.create_datastore('OOI_Assets', workspace=nvs)
# Edit the connection parameters
gpkg_ds.connection_parameters = {'Connection timeout': '20',
 'Evictor run periodicity': '300',
 'Evictor tests per run': '3',
 'Expose primary keys': 'false',
 'Max connection idle time': '300',
 'Test while idle': 'true',
 'database': 'file:data/geopackages/OOI_Assets.gpkg', # Point to location of geopackage relative to the geoserver data directory
 'dbtype': 'geopkg',
 'fetch size': '1000',
 'max connections': '10',
 'min connections': '1',
 'namespace': 'http://data.nanoos.org/geoserver2_8/nvs_assets', # Workspace URL
 'validate connections': 'true'}
# Save datastore
cat.save(gpkg_ds)



    


















    
Out[18]:








({'connection': 'keep-alive',
  'content-length': '0',
  'date': 'Wed, 01 Jun 2016 18:47:23 GMT',
  'location': 'http://data.nanoos.org/geoserver2_8/rest/workspaces/nvs_assets/datastores/OOI_Assets',
  'server': 'nginx/1.1.19',
  'status': '201'},
 '')

















In [19]:



    


# Set necessary variables for cURL
data_name = 'OOI_Assets'
wksp_name = nvs.name
ds_name = gpkg_ds.name
print ds_name



    


















    






OOI_Assets

















In [20]:



    


# Create layer from geopackage table
subprocess.call('curl -v -u ####:#### -XPOST -H "Content-type: text/xml" -d "<featureType><name>{0}</name></featureType>" http://data.nanoos.org/geoserver2_8/rest/workspaces/{1}/datastores/{2}/featuretypes'.format(data_name,wksp_name,ds_name), shell=True)



    


















    
Out[20]:








0

















In [21]:



    


# get the newly published layer w/o any projection
layer = cat.get_layer(data_name)

# retrieve resource to assign projection
rsrc = layer.resource

# assign Layer projection
rsrc.projection = 'EPSG:4326'

# save layer
cat.save(rsrc)



    


















    
Out[21]:








({'connection': 'keep-alive',
  'content-length': '0',
  'date': 'Wed, 01 Jun 2016 18:47:48 GMT',
  'server': 'nginx/1.1.19',
  'status': '200'},
 '')

















In [ ]:

Content source: rsignell-usgs/notebook

Similar notebooks:

notebook.community | gallery | about