In [1]:
import pyes
from pyes.es import ES
import pytz
from datetime import datetime
from dateutil.parser import parse
from datetime import timedelta
import json
In [2]:
#change the default if you are not in Pacific time
#and want to use dates like 'today 8am'
def toUTC(suspectedDate,localTimeZone="US/Pacific"):
'''make a UTC date out of almost anything'''
utc=pytz.UTC
objDate=None
if type(suspectedDate)==str:
objDate=parse(suspectedDate,fuzzy=True)
elif type(suspectedDate)==datetime:
objDate=suspectedDate
if objDate.tzinfo is None:
objDate=pytz.timezone(localTimeZone).localize(objDate)
objDate=utc.normalize(objDate)
else:
objDate=utc.normalize(objDate)
if objDate is not None:
objDate=utc.normalize(objDate)
return objDate
In [3]:
#Set this to one of your ES servers:
es=ES(("http", "servername.goes.here", 9200))
#set a date range
begindateUTC=toUTC(datetime.now() - timedelta(minutes=15))
enddateUTC= toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
#set up some criteria (Queries are less usefull than filters)
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
#add as many 'must, must_not, should' criteria filters as you need
#to get the data you want
q = pyes.FilteredQuery(q,
pyes.BoolFilter(
must=[qDate,
pyes.TermFilter('_type', 'mozdefstats')
]
must_not=[],
should=[]
)
)
#in mozdef, events and events-previous
#are aliases to the current day and previous day
results=es.search(query=q,size=100,indices=['events','events-previous'])
#how many docs were found?
print(results.count())
In [5]:
#pyes has a iteration bug where
#walking the results pops the results from the collection
#so easiest way to capture results is _search_raw()
#which gives you the raw ES json
rawresults=results._search_raw()
In [ ]: