In [ ]:
# This is grade A lazy attempt to stop github key parsers - need to implement env stuff properly...
gapikey1 = 'asdasd_AIzaSyA8ZTz_'
gapikey2 = 'dsfsdf_o3VhApo4CoehKa6m7_cFLKKBnXt'
gapikey = (gapikey1+gapikey2[::-1])[7:-7]
headers_base = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
'Referer':'https://www.costcotravel.com',
'Host':'www.costcotravel.com',
}
data_base = {
'rcs':'1',
'driverAge':'25',
'pickupZip':ZIP,
'pickupCityLocationTypeSearch':'2',
'dropoffZip':ZIP,
'dropoffCityLocationTypeSearch':'2',
'pickupCountry':'US',
'dropoffCountry':'US',
'pickupCityRadius':RADIUS,
'dropoffCityRadius':RADIUS,
'pickupAsAirport':'false',
'dropoffAsAirport':'false',
'pickupTime':TIME,
'dropoffTime':TIME,
'pickupLocationCode':ZIP,
'pickupLocationName':ZIP+' (CHICAGO, IL, US)',
'pickupLocationType':'zipCode',
'pickupLocationCityCode':'CHICAGO',
'pickupLocationStateCode':'',
'pickupLatitude':'',
'pickupLongitude':'',
'dropoffLocationCode':ZIP,
'dropoffLocationName':ZIP+' (CHICAGO, IL, US)',
'dropoffLocationType':'zipCode',
'dropoffLocationCityCode':'CHICAGO',
'dropoffLocationStateCode':'',
'dropoffLatitude':'',
'dropoffLongitude':'',
'fromHomePage':'true',
'fromCarVendorMainMenu':'true',
'carSearchInModifyFlow':'false',
'suppressOutput':'false',
}
In [ ]:
def getpriceasync(agencyid=None,span=None,idx=None):
headers = dict(headers_base)
data = dict(data_base)
data.update({'pickupDate':span[0].strftime("%m/%d/%Y"),'dropoffDate':span[1].strftime("%m/%d/%Y")})
s = requests.Session()
#print('{}: stage 1'.format(num))
r3 = s.get('https://www.costcotravel.com/h=4005',headers=headers)
cks = r3.cookies
#print(cks.keys())
#r3.text
res = re.search('\(\"Csrf-token\", \"(\S{128})\"\)',r3.text)
csrf = res.group(1)
#print(csrf)
ts = re.search('namespace.sessionTimestamp = (\d{10,20});',r3.text).group(1)
#print(ts)
headers.update({'X-Csrf-Token':csrf})
headers.update({'Referer':'https://www.costcotravel.com/h=4005'})
cks.set('Csrf-token',csrf)
if 'SESSION_TIME_OUT_DETECTED' in cks: cks.pop('SESSION_TIME_OUT_DETECTED')
cks.set('SESSION_TIMESTAMP',str(ts))
#print(cks.keys())
time.sleep(0.8)
#print('{}: stage 2'.format(num))
r = s.post('https://www.costcotravel.com/rentalCarSearch.act', data = data, headers = headers, cookies = cks)
#print(r.text)
res = re.findall(
('{\"id\":\"S(\d{1,3})\",\"type\":\"(city|airport)\",\"agency\":\"(\S{1,40})\",\"agencyCode\":\"(\S{1,8})\",'
'\"title\":\"(.{4,50})\",\"isOpen\":true,\"unAvailableMessage\":\"\",\"vendorCode\":\"(\S{2,3})\",\"address\"'
':\"(.{5,110})\",\"city\":\"(.{3,30})\",\"state\":\"(\S{2})\",\"zip\":\"'
'(\d{5})\",\"country\":\"US\",\"distance\":\"(\S{2,7})\",\"latitude\":(\S{2,10}),\"longitude\":(\S{2,10}),'),
r.text)
#print(len(res),res)
results = dict()
match = [v for v in res if v[3] == agencyid]
#print(match)
if match:
tp = match[0]
dictentry = {'num':int(tp[0]),'type':tp[1],'brand':tp[2],'code':tp[3],'name':tp[4],'bcode':tp[5],'addr':tp[6],
'city':tp[7],'state':tp[8],'zip':tp[9],'dist':tp[10],'lat':tp[11],'lon':tp[12],
'start':data['pickupDate'],'end':data['dropoffDate']}
#print(len(results),results.keys())
#print(results)
#print(tp)
#print(len(results))
time.sleep(1.0)
headers.update({'Referer':'https://www.costcotravel.com/h=3001'})
k = idx
v = dictentry
data2 = {
'cas':'Load_Forword_Navigation_From_Agency_Results',
#'carAgenciesForVendors':'[{"vendorId":"ET","agencyCodes":["E11576"]}]',
'carAgenciesForVendors':'[{{\"vendorId\":\"{}\",\"agencyCodes\":[\"{}\"]}}]'.format(v['bcode'],v['code']),
#'uid':'1504759332852_756.0475947513028'
}
#print('{}: stage 3'.format(num))
r2 = s.post('https://www.costcotravel.com/rentalCarAgencySelection.act',data=data2,cookies=cks,headers=headers)
#print(k,r2)
#print(r2.text)
prices = re.findall('<h3>(.{5,40})</h3></div></div></div><div style=\"height: 94px;\" class=\"col col-lg col-1 col-lg-1 text-center height-item test\"><a data-responsive-referrer=\"carMatrix\" data-category-id=\"\S{20,50}\" data-selected=\"(?:false|true)\" data-product-id=\"\S{20,50}\" data-price=\"(\S{4,30})\"',r2.text)
#print(prices)
#print(r2.text)
if len(prices) == 0:
print("{:03d} ND |".format(k),tp[0],tp[2:5],tp[10:])
#print(r2.text)
else:
#print("{:03d} OK |".format(k),tp[0],tp[2:5],tp[10:])
for pr in prices:
#print(pr)
try:
v[pr[0]] = float(pr[1])
except:
v[pr[0]] = pr[1]
#print(v)
return (k,v,len(res))
else:
print('NO MATCH')
return (idx,None,None)
In [ ]:
def getpriceasync_googletest(num):
s = requests.Session()
r = s.get('https://www.costcotravel.com/h=4005')
print('{}: static test'.format(num))
return (r.text)
In [ ]:
def getagencylist(span):
headers = dict(headers_base)
data = dict(data_base)
data.update({'pickupDate':span[0].strftime("%m/%d/%Y"),'dropoffDate':span[1].strftime("%m/%d/%Y")})
s = requests.Session()
#print('{}: stage 1'.format(num))
r3 = s.get('https://www.costcotravel.com/h=4005',headers=headers)
cks = r3.cookies
#print(cks.keys())
#r3.text
res = re.search('\(\"Csrf-token\", \"(\S{128})\"\)',r3.text)
csrf = res.group(1)
#print(csrf)
ts = re.search('namespace.sessionTimestamp = (\d{10,20});',r3.text).group(1)
#print(ts)
headers.update({'X-Csrf-Token':csrf})
headers.update({'Referer':'https://www.costcotravel.com/h=4005'})
cks.set('Csrf-token',csrf)
if 'SESSION_TIME_OUT_DETECTED' in cks: cks.pop('SESSION_TIME_OUT_DETECTED')
cks.set('SESSION_TIMESTAMP',str(ts))
#print(cks.keys())
time.sleep(1.0)
#print('{}: stage 2'.format(num))
r = s.post('https://www.costcotravel.com/rentalCarSearch.act', data = data, headers = headers, cookies = cks)
#print(r.text)
res = re.findall(
('{\"id\":\"S(\d{1,3})\",\"type\":\"(city|airport)\",\"agency\":\"(\S{1,40})\",\"agencyCode\":\"(\S{1,8})\",'
'\"title\":\"(.{4,50})\",\"isOpen\":true,\"unAvailableMessage\":\"\",\"vendorCode\":\"(\S{2,3})\",\"address\"'
':\"(.{5,110})\",\"city\":\"(.{3,30})\",\"state\":\"(\S{2})\",\"zip\":\"'
'(\d{5})\",\"country\":\"US\",\"distance\":\"(\S{2,7})\",\"latitude\":(\S{2,10}),\"longitude\":(\S{2,10}),'),
r.text)
#print(len(res),res)
results = dict()
for tp in res:
results[int(tp[0])] = {'num':int(tp[0]),'type':tp[1],'brand':tp[2],'code':tp[3],'name':tp[4],'bcode':tp[5],'addr':tp[6],
'city':tp[7],'state':tp[8],'zip':tp[9],'dist':tp[10],'lat':tp[11],'lon':tp[12]}
#print(len(results),results.keys())
#print(results)
#print(tp)
#print(len(results))
return (results)
In [ ]:
def computeTravelTimes(results):
addresses = []
for i in range(1,len(results)+1):
entry = results[i]
addresses.append(entry['addr']+', '+entry['city']+', '+entry['state']+', '+entry['zip']+', USA')
batchsize = 100
fullbatches = len(addresses)//batchsize
print(len(addresses))
destinations = []
for i in range(0,fullbatches):
#print(i)
destinations.append('|'.join(addresses[i*batchsize:(i+1)*batchsize]))
if len(addresses)%batchsize > 0:
destinations.append('|'.join(addresses[fullbatches*batchsize:]))
print([destinations[i].count('|')+1 for i in range(0,len(destinations))])
origin = '1369 East Hyde Park Blvd, Chicago, IL, 60615, USA'
for i, dest in enumerate(destinations):
gresponse = getGDistance(origin, dest)
time.sleep(0.5)
#print(gresponse)
if gresponse['status'] == 'OK':
elems = gresponse['rows'][0]['elements']
print('Got gmatrix response for batch {}|{}'.format(i,len(elems)))
for j,entry in enumerate(elems):
idx = i*batchsize+j+1
if (entry['status'] == 'OK'):
results[idx]['time'] = entry['duration']['value']
results[idx]['roaddist'] = entry['distance']['value']
#print(idx,results[idx]['time'],results[idx]['roaddist'])
else:
print('{} ENTRY FAILED'.format(idx))
else:
print('Gmatrix response for batch {} FAILED'.format(i))
def getGDistance(origin,destination,mode='transit'):
urlparams = {'units':'imperial','origins':origin,'destinations':destination,'key':gapikey,'mode':mode,
'departure_time':1511366822,'transit_routing_preference':'fewer_transfers'}
resp = requests.get('https://maps.googleapis.com/maps/api/distancematrix/json',params=urlparams)
rj = resp.json()
print(rj)
return rj
In [ ]:
def runparser(agencyids=None,threads=1):
assert agencyids is not None
startidx = 0
resultsfinal = {}
for agencyid in agencyids:
if FIXEDLENGTH:
# just want a sliding window
starts = generateDates(PICKUP,VARYSTART)
ends = generateDates(DROPOFF,VARYSTART)
assert len(starts) == len(ends)
spans = list(zip(starts,ends))
else:
# creates a cartesian product essentially
starts = generateDates(PICKUP,VARYSTART)
ends = generateDates(DROPOFF,VARYEND)
spans = [(x,y) for x in starts for y in ends]
print(len(spans))
resultstemp = dict()
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
# Requests uses blocking io, so doing things in single thread event loop doesn't help much
# -> have to do executor thingies and run separate threads
executor = concurrent.futures.ThreadPoolExecutor(max_workers=threads)
try:
loop.run_until_complete(runall(executor,resultstemp,agencyid,spans,startidx))
finally:
loop.close()
numresults = len(resultstemp)
startidx += numresults
resultsfinal.update(resultstemp)
return resultsfinal
async def runall(executor,resultstemp,agencyid,spans,startidx):
#print('start t2')
loop = asyncio.get_event_loop()
#print(pool)
blocking_tasks = [loop.run_in_executor(executor, getpriceasync, agencyid, spans[i], i+startidx) for i in range(0,len(spans))]
#print(blocking_tasks)
completed, pending = await asyncio.wait(blocking_tasks)
#print(completed,pending)
results = [t.result() for t in completed]
for i,r in enumerate(results):
resultstemp[r[0]] = r[1]
#print(i,r)
In [ ]:
def generateDates(center,variance):
dates = []
for i in range(-variance,variance+1):
dates.append(center+datetime.timedelta(days=i))
return dates