In [1]:
from __future__ import division
from pyomo.environ import *
from pyomo.opt import SolverFactory
import googlemaps
from sqlalchemy import create_engine
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import os
import ast
import time as tm

In [2]:
def connection(user,passwd,dbname):
    str1 = ('postgresql+pg8000://' + user +':' + passw + '@switch-db2.erg.berkeley.edu:5432/' 
            + dbname + '?ssl=true&sslfactory=org.postgresql.ssl.NonValidatingFactory')
    engine = create_engine(str1)
    return engine

In [3]:
user = 'jdlara'
passw = 'Amadeus-2010'
dbname = 'apl_cec' 
engine_apl = connection(user,passw,dbname)

In [4]:
query = ('SELECT biosum.latlon.biosum_cond_id as biosum_id, \"PGE\".feeders.feeder_no as feeder_id, '+ 
         'biosum.latlon.lon as biosum_lon, biosum.latlon.lat as biosum_lat, ' +
         '\"PGE\".feeders.lon as feeders_lon, \"PGE\".feeders.lat as feeders_lat, '
         'ST_Distance(st_transform(biosum.latlon.geom,5070),st_transform(\"PGE\".feeders.geom,5070))/1000 ' + 
         'as distance_km FROM biosum.latlon, \"PGE\".feeders WHERE ST_DWithin(st_transform(biosum.latlon.geom,5070),' +
         'st_transform(\"PGE\".feeders.geom,5070), 160000);')

In [5]:
df_routes = pd.read_sql_query(query,engine_apl)

In [ ]:
biosum_coord = df_routes.biosum_lat.astype(str).str.cat(df_routes.biosum_lon.astype(str), sep=',')
biosum_coord = biosum_coord.values.tolist()
df_routes['biosum_coord_str'] = biosum_coord

feeder_coord = df_routes.feeders_lat.astype(str).str.cat(df_routes.feeders_lon.astype(str), sep=',')
feeder_coord = feeder_coord.values.tolist()
df_routes['feeder_coord_str'] = feeder_coord

df_routes['distance_google'] = pd.Series(np.random.randn(len(df_routes['biosum_id'])), index=df_routes.index)
df_routes['time_google'] = pd.Series(np.random.randn(len(df_routes['biosum_id'])), index=df_routes.index)
df_routes['data_state'] = pd.Series(np.zeros(len(df_routes['biosum_id'])), index=df_routes.index, dtype=np.int)

In [ ]:
df_routes = df_routes.drop(['biosum_lon','biosum_lat','feeders_lon','feeders_lat'], 1)
df_routes.to_sql('biosum_pge', engine_apl, schema='biosum', if_exists='replace',chunksize=100)

In [ ]:
gmaps = googlemaps.Client(key='AIzaSyAKlu6Ndp4RiMTgE2eiqoM3UnVZdUkZppU')
distance_table = {}
time_table = {}
biomass_list = []
substation_list = []
avoid_table = {}
fail_table = {}

In [19]:
count = 0;
for index, row in df_routes.iterrows():
    tm.sleep(0.81)
    sql_cmd = engine_apl.connect()
    query = ('select data_state from biosum.biosum_pge where ' + 
             'biosum_pge.biosum_coord_str = \'' + row['biosum_coord_str'] + '\' and ' +
             'biosum_pge.feeder_coord_str = \'' + row['feeder_coord_str'] + '\'')
    state = sql_cmd.execute(query)
    state = row['data_state']
    sql_cmd = engine_apl.dispose()
    if state == 0:
        matrx_distance = (gmaps.distance_matrix(row['biosum_coord_str'], row['feeder_coord_str'], 
                                                mode="driving", departure_time="now", traffic_model="pessimistic"))
        count = count + 1;
        error = matrx_distance['rows'][0]['elements'][0]['status']
        if error != 'OK':
            df_routes.set_value(index,'data_stae', -1)
            f = open('fail_table.dat', 'a')
            f.write(('Route data unavailable for ' + str(row['biosum_id']) + "," + str(row['feeder_id']) + "\n"))
            f.close()
        else:
            df_routes.set_value(index,'data_state',1)
            df_routes.set_value(index,'distance_google', (0.001 * matrx_distance['rows'][0]['elements'][0]['distance']['value']))
            df_routes.set_value(index,'time_google', (1 / 3600) *(matrx_distance['rows'][0]['elements'][0]['duration_in_traffic']['value']))
    else:
        continue
    
    if count == 10000:
        df_routes.to_csv('~/Desktop/backup.csv')
        print count
        count =0;
    else:
        continue


---------------------------------------------------------------------------
ProgrammingError                          Traceback (most recent call last)
<ipython-input-19-b3a3d1f43a8f> in <module>()
      2 for index, row in df_routes.iterrows():
      3     tm.sleep(0.81)
----> 4     sql_cmd = engine_apl.connect()
      5     query = ('select data_state from biosum.biosum_pge where ' + 
      6              'biosum_pge.biosum_coord_str = \'' + row['biosum_coord_str'] + '\' and ' +

/Library/Python/2.7/site-packages/sqlalchemy/engine/base.pyc in connect(self, **kwargs)
   2016         """
   2017 
-> 2018         return self._connection_cls(self, **kwargs)
   2019 
   2020     def contextual_connect(self, close_with_result=False, **kwargs):

/Library/Python/2.7/site-packages/sqlalchemy/engine/base.pyc in __init__(self, engine, connection, close_with_result, _branch_from, _execution_options, _dispatch, _has_events)
     70         else:
     71             self.__connection = connection \
---> 72                 if connection is not None else engine.raw_connection()
     73             self.__transaction = None
     74             self.__savepoint_seq = 0

/Library/Python/2.7/site-packages/sqlalchemy/engine/base.pyc in raw_connection(self, _connection)
   2102         """
   2103         return self._wrap_pool_connect(
-> 2104             self.pool.unique_connection, _connection)
   2105 
   2106 

/Library/Python/2.7/site-packages/sqlalchemy/engine/base.pyc in _wrap_pool_connect(self, fn, connection)
   2076             if connection is None:
   2077                 Connection._handle_dbapi_exception_noconnection(
-> 2078                     e, dialect, self)
   2079             else:
   2080                 util.reraise(*sys.exc_info())

/Library/Python/2.7/site-packages/sqlalchemy/engine/base.pyc in _handle_dbapi_exception_noconnection(cls, e, dialect, engine)
   1403             util.raise_from_cause(
   1404                 sqlalchemy_exception,
-> 1405                 exc_info
   1406             )
   1407         else:

/Library/Python/2.7/site-packages/sqlalchemy/util/compat.pyc in raise_from_cause(exception, exc_info)
    200     exc_type, exc_value, exc_tb = exc_info
    201     cause = exc_value if exc_value is not exception else None
--> 202     reraise(type(exception), exception, tb=exc_tb, cause=cause)
    203 
    204 if py3k:

/Library/Python/2.7/site-packages/sqlalchemy/engine/base.pyc in _wrap_pool_connect(self, fn, connection)
   2072         dialect = self.dialect
   2073         try:
-> 2074             return fn()
   2075         except dialect.dbapi.Error as e:
   2076             if connection is None:

/Library/Python/2.7/site-packages/sqlalchemy/pool.pyc in unique_connection(self)
    316 
    317         """
--> 318         return _ConnectionFairy._checkout(self)
    319 
    320     def _create_connection(self):

/Library/Python/2.7/site-packages/sqlalchemy/pool.pyc in _checkout(cls, pool, threadconns, fairy)
    711     def _checkout(cls, pool, threadconns=None, fairy=None):
    712         if not fairy:
--> 713             fairy = _ConnectionRecord.checkout(pool)
    714 
    715             fairy._pool = pool

/Library/Python/2.7/site-packages/sqlalchemy/pool.pyc in checkout(cls, pool)
    478     @classmethod
    479     def checkout(cls, pool):
--> 480         rec = pool._do_get()
    481         try:
    482             dbapi_connection = rec.get_connection()

/Library/Python/2.7/site-packages/sqlalchemy/pool.pyc in _do_get(self)
   1058                 except:
   1059                     with util.safe_reraise():
-> 1060                         self._dec_overflow()
   1061             else:
   1062                 return self._do_get()

/Library/Python/2.7/site-packages/sqlalchemy/util/langhelpers.pyc in __exit__(self, type_, value, traceback)
     58             exc_type, exc_value, exc_tb = self._exc_info
     59             self._exc_info = None   # remove potential circular references
---> 60             compat.reraise(exc_type, exc_value, exc_tb)
     61         else:
     62             if not compat.py3k and self._exc_info and self._exc_info[1]:

/Library/Python/2.7/site-packages/sqlalchemy/pool.pyc in _do_get(self)
   1055             if self._inc_overflow():
   1056                 try:
-> 1057                     return self._create_connection()
   1058                 except:
   1059                     with util.safe_reraise():

/Library/Python/2.7/site-packages/sqlalchemy/pool.pyc in _create_connection(self)
    321         """Called by subclasses to create a new ConnectionRecord."""
    322 
--> 323         return _ConnectionRecord(self)
    324 
    325     def _invalidate(self, connection, exception=None):

/Library/Python/2.7/site-packages/sqlalchemy/pool.pyc in __init__(self, pool)
    447     def __init__(self, pool):
    448         self.__pool = pool
--> 449         self.connection = self.__connect()
    450         self.finalize_callback = deque()
    451 

/Library/Python/2.7/site-packages/sqlalchemy/pool.pyc in __connect(self)
    605         try:
    606             self.starttime = time.time()
--> 607             connection = self.__pool._invoke_creator(self)
    608             self.__pool.logger.debug("Created new connection %r", connection)
    609             return connection

/Library/Python/2.7/site-packages/sqlalchemy/engine/strategies.pyc in connect(connection_record)
     95                         if connection is not None:
     96                             return connection
---> 97                 return dialect.connect(*cargs, **cparams)
     98 
     99             creator = pop_kwarg('creator', connect)

/Library/Python/2.7/site-packages/sqlalchemy/engine/default.pyc in connect(self, *cargs, **cparams)
    383 
    384     def connect(self, *cargs, **cparams):
--> 385         return self.dbapi.connect(*cargs, **cparams)
    386 
    387     def create_connect_args(self, url):

/Library/Python/2.7/site-packages/pg8000/__init__.pyc in connect(user, host, unix_sock, port, database, password, ssl, timeout, **kwargs)
    101     """
    102     return Connection(
--> 103         user, host, unix_sock, port, database, password, ssl, timeout)
    104 
    105 apilevel = "2.0"

/Library/Python/2.7/site-packages/pg8000/core.pyc in __init__(self, user, host, unix_sock, port, database, password, ssl, timeout)
   1610                 except Exception:
   1611                     pass
-> 1612                 raise e
   1613 
   1614         self.in_transaction = False

ProgrammingError: (pg8000.core.ProgrammingError) (u'FATAL', u'53300', u'remaining connection slots are reserved for non-replication superuser connections', u'postinit.c', u'660', u'InitPostgres', u'', u'')

In [ ]:
df_routes.to_sql('biosum_pge', engine_apl, schema='biosum', if_exists='replace',chunksize=100)

In [ ]:
df_routes

In [ ]: