Time series

Date and Time Data Types and Tools



In [2]:

    
from datetime import datetime



In [4]:

    
now = datetime.now()
now









    Out[4]:





datetime.datetime(2017, 6, 28, 15, 11, 55, 925801)



In [5]:

    
now.year, now.month, now.day









    Out[5]:





(2017, 6, 28)



In [6]:

    
delta = datetime(2011, 1, 7) - datetime(2008, 6, 24, 8, 15)
delta









    Out[6]:





datetime.timedelta(926, 56700)



In [7]:

    
delta.days









    Out[7]:





926



In [8]:

    
delta.seconds









    Out[8]:





56700



In [9]:

    
from datetime import timedelta
start = datetime(2011, 1, 7)
start + timedelta(12)









    Out[9]:





datetime.datetime(2011, 1, 19, 0, 0)



In [10]:

    
start - 2 * timedelta(12)









    Out[10]:





datetime.datetime(2010, 12, 14, 0, 0)

Converting between string and datetime



In [11]:

    
stamp = datetime(2011, 1, 3)
str(stamp)









    Out[11]:





'2011-01-03 00:00:00'



In [13]:

    
stamp.strftime('%Y-%m-%d')









    Out[13]:





'2011-01-03'



In [14]:

    
value = '2011-01-03'
datetime.strptime(value, '%Y-%m-%d')









    Out[14]:





datetime.datetime(2011, 1, 3, 0, 0)



In [15]:

    
datestrs = ['7/6/2011', '8/6/2011']
[datetime.strptime(x, '%m/%d/%Y') for x in datestrs]









    Out[15]:





[datetime.datetime(2011, 7, 6, 0, 0), datetime.datetime(2011, 8, 6, 0, 0)]



In [16]:

    
from dateutil.parser import parse
parse('2011-01-03')









    Out[16]:





datetime.datetime(2011, 1, 3, 0, 0)



In [17]:

    
parse('Jan 31, 1997 10:45 PM')









    Out[17]:





datetime.datetime(1997, 1, 31, 22, 45)



In [18]:

    
parse('6/12/2011', dayfirst=True)









    Out[18]:





datetime.datetime(2011, 12, 6, 0, 0)



In [19]:

    
datestrs









    Out[19]:





['7/6/2011', '8/6/2011']



In [20]:

    
import pandas as pd



In [21]:

    
pd.to_datetime(datestrs)









    Out[21]:





DatetimeIndex(['2011-07-06', '2011-08-06'], dtype='datetime64[ns]', freq=None)



In [25]:

    
idx = pd.to_datetime(datestrs + [None])
idx









    Out[25]:





DatetimeIndex(['2011-07-06', '2011-08-06', 'NaT'], dtype='datetime64[ns]', freq=None)



In [26]:

    
pd.isnull(pd.to_datetime(datestrs + [None]))









    Out[26]:





array([False, False,  True], dtype=bool)

Time Series Basics



In [29]:

    
from pandas import Series, DataFrame
import numpy as np
dates = [datetime(2011, 1, 2), datetime(2011, 1, 5), datetime(2011, 1, 7),
         datetime(2011, 1, 8), datetime(2011, 1, 10), datetime(2011, 1, 12)]
ts = Series(np.random.randn(6), index=dates)
ts









    Out[29]:





2011-01-02    1.111272
2011-01-05    0.221653
2011-01-07    0.674424
2011-01-08   -0.749457
2011-01-10    1.922602
2011-01-12    0.849016
dtype: float64



In [30]:

    
type(ts)









    Out[30]:





pandas.core.series.Series



In [31]:

    
ts + ts[::2]









    Out[31]:





2011-01-02    2.222544
2011-01-05         NaN
2011-01-07    1.348848
2011-01-08         NaN
2011-01-10    3.845205
2011-01-12         NaN
dtype: float64



In [32]:

    
ts.index.dtype









    Out[32]:





dtype('<M8[ns]')



In [33]:

    
ts.index[0]









    Out[33]:





Timestamp('2011-01-02 00:00:00')

Indexing, selection, subsetting



In [34]:

    
ts['1/10/2011']









    Out[34]:





1.92260229416429



In [35]:

    
ts['20110110']









    Out[35]:





1.92260229416429



In [36]:

    
longer_ts = Series(np.random.randn(1000),
                   index=pd.date_range('1/1/2000', periods=1000))
longer_ts









    Out[36]:





2000-01-01   -0.429690
2000-01-02   -0.988769
2000-01-03   -0.024645
2000-01-04    2.430567
2000-01-05   -1.351163
2000-01-06    0.503308
2000-01-07    0.194014
2000-01-08   -0.697142
2000-01-09    0.191641
2000-01-10    0.330984
2000-01-11   -0.846752
2000-01-12    0.634315
2000-01-13   -0.893407
2000-01-14   -1.650352
2000-01-15   -0.791815
2000-01-16   -1.267505
2000-01-17   -0.762687
2000-01-18   -0.896668
2000-01-19    1.720866
2000-01-20    1.661988
2000-01-21    1.708285
2000-01-22   -0.330619
2000-01-23   -0.864698
2000-01-24   -0.200427
2000-01-25    0.509097
2000-01-26    1.675818
2000-01-27   -0.592886
2000-01-28    1.328173
2000-01-29   -1.123267
2000-01-30    1.319367
                ...   
2002-08-28   -0.909667
2002-08-29   -0.824423
2002-08-30   -0.705050
2002-08-31   -0.393563
2002-09-01   -0.628749
2002-09-02   -1.085462
2002-09-03    0.142824
2002-09-04    0.592543
2002-09-05    0.490188
2002-09-06    1.289383
2002-09-07   -1.250163
2002-09-08    0.590002
2002-09-09   -0.075362
2002-09-10    1.044168
2002-09-11   -0.088020
2002-09-12   -0.836371
2002-09-13    1.481651
2002-09-14    0.292208
2002-09-15    1.282557
2002-09-16    0.844125
2002-09-17    0.217562
2002-09-18    0.472906
2002-09-19    0.266230
2002-09-20    0.003674
2002-09-21    0.294167
2002-09-22    0.324055
2002-09-23   -0.567444
2002-09-24    1.445333
2002-09-25    1.348238
2002-09-26   -1.510428
Freq: D, dtype: float64



In [37]:

    
longer_ts['2001']









    Out[37]:





2001-01-01    1.423174
2001-01-02    2.655359
2001-01-03   -1.964971
2001-01-04   -0.751284
2001-01-05    0.835949
2001-01-06   -0.219525
2001-01-07   -0.433020
2001-01-08   -0.611016
2001-01-09   -0.803291
2001-01-10    0.805506
2001-01-11    0.291593
2001-01-12   -0.263987
2001-01-13    0.745169
2001-01-14   -1.085306
2001-01-15   -0.760404
2001-01-16    0.003680
2001-01-17   -0.501209
2001-01-18    0.000269
2001-01-19   -1.283880
2001-01-20    0.297868
2001-01-21   -0.701593
2001-01-22   -0.662293
2001-01-23   -0.243264
2001-01-24    0.617356
2001-01-25    0.606253
2001-01-26   -0.225016
2001-01-27    0.765918
2001-01-28   -0.122033
2001-01-29   -0.397498
2001-01-30    1.280038
                ...   
2001-12-02    0.848705
2001-12-03   -0.260151
2001-12-04   -0.373202
2001-12-05   -0.395652
2001-12-06   -0.264092
2001-12-07    1.370772
2001-12-08    0.269687
2001-12-09    1.174350
2001-12-10   -0.268601
2001-12-11    1.465140
2001-12-12   -0.796310
2001-12-13    0.162920
2001-12-14    0.211350
2001-12-15    3.106605
2001-12-16    1.905970
2001-12-17   -1.153820
2001-12-18   -1.115190
2001-12-19    0.395947
2001-12-20    0.166124
2001-12-21    0.451831
2001-12-22    0.792277
2001-12-23   -0.962898
2001-12-24    1.275114
2001-12-25   -1.523152
2001-12-26   -1.396730
2001-12-27   -0.437941
2001-12-28   -1.195787
2001-12-29    0.001429
2001-12-30    0.388284
2001-12-31    1.045000
Freq: D, dtype: float64



In [38]:

    
longer_ts['2001-05']









    Out[38]:





2001-05-01    0.042114
2001-05-02   -0.714940
2001-05-03    0.517785
2001-05-04   -0.587098
2001-05-05   -0.096823
2001-05-06   -0.976581
2001-05-07    1.278264
2001-05-08    1.890005
2001-05-09   -0.260056
2001-05-10    0.955862
2001-05-11    0.062605
2001-05-12   -0.317040
2001-05-13   -0.010676
2001-05-14   -0.893382
2001-05-15   -0.275795
2001-05-16    0.491210
2001-05-17    0.381374
2001-05-18    0.283364
2001-05-19    0.210997
2001-05-20    0.980941
2001-05-21    0.568281
2001-05-22   -0.434008
2001-05-23    0.746367
2001-05-24   -0.077535
2001-05-25   -1.610058
2001-05-26   -2.215233
2001-05-27   -0.064517
2001-05-28   -0.716334
2001-05-29   -1.110473
2001-05-30   -0.967791
2001-05-31   -0.976880
Freq: D, dtype: float64



In [39]:

    
ts[datetime(2011, 1, 7):]









    Out[39]:





2011-01-07    0.674424
2011-01-08   -0.749457
2011-01-10    1.922602
2011-01-12    0.849016
dtype: float64



In [40]:

    
ts









    Out[40]:





2011-01-02    1.111272
2011-01-05    0.221653
2011-01-07    0.674424
2011-01-08   -0.749457
2011-01-10    1.922602
2011-01-12    0.849016
dtype: float64



In [41]:

    
ts['1/6/2011':'1/11/2011']









    Out[41]:





2011-01-07    0.674424
2011-01-08   -0.749457
2011-01-10    1.922602
dtype: float64



In [42]:

    
ts.truncate(after='1/9/2011')









    Out[42]:





2011-01-02    1.111272
2011-01-05    0.221653
2011-01-07    0.674424
2011-01-08   -0.749457
dtype: float64



In [43]:

    
dates = pd.date_range('1/1/2000', periods=100, freq='W-WED')
long_df = DataFrame(np.random.randn(100, 4),
                    index=dates,
                    columns=['Colorado', 'Texas', 'New York', 'Ohio'])
long_df.ix['5-2001']

Time series with duplicate indices



In [44]:

    
dates = pd.DatetimeIndex(['1/1/2000', '1/2/2000', '1/2/2000', '1/2/2000',
                          '1/3/2000'])
dup_ts = Series(np.arange(5), index=dates)
dup_ts









    Out[44]:





2000-01-01    0
2000-01-02    1
2000-01-02    2
2000-01-02    3
2000-01-03    4
dtype: int64



In [49]:

    
dup_ts.index.is_unique









    Out[49]:





False



In [50]:

    
dup_ts['1/3/2000']









    Out[50]:





4



In [51]:

    
dup_ts['1/2/2000']









    Out[51]:





2000-01-02    1
2000-01-02    2
2000-01-02    3
dtype: int64



In [52]:

    
grouped = dup_ts.groupby(level=0)
grouped.mean()









    Out[52]:





2000-01-01    0
2000-01-02    2
2000-01-03    4
dtype: int64



In [53]:

    
grouped.count()









    Out[53]:





2000-01-01    1
2000-01-02    3
2000-01-03    1
dtype: int64

Date ranges, Frequencies, and Shifting



In [54]:

    
ts









    Out[54]:





2011-01-02    1.111272
2011-01-05    0.221653
2011-01-07    0.674424
2011-01-08   -0.749457
2011-01-10    1.922602
2011-01-12    0.849016
dtype: float64



In [55]:

    
ts.resample('D')









    Out[55]:





DatetimeIndexResampler [freq=<Day>, axis=0, closed=left, label=left, convention=start, base=0]

Generating date ranges



In [56]:

    
index = pd.date_range('4/1/2012', '6/1/2012')
index









    Out[56]:





DatetimeIndex(['2012-04-01', '2012-04-02', '2012-04-03', '2012-04-04',
               '2012-04-05', '2012-04-06', '2012-04-07', '2012-04-08',
               '2012-04-09', '2012-04-10', '2012-04-11', '2012-04-12',
               '2012-04-13', '2012-04-14', '2012-04-15', '2012-04-16',
               '2012-04-17', '2012-04-18', '2012-04-19', '2012-04-20',
               '2012-04-21', '2012-04-22', '2012-04-23', '2012-04-24',
               '2012-04-25', '2012-04-26', '2012-04-27', '2012-04-28',
               '2012-04-29', '2012-04-30', '2012-05-01', '2012-05-02',
               '2012-05-03', '2012-05-04', '2012-05-05', '2012-05-06',
               '2012-05-07', '2012-05-08', '2012-05-09', '2012-05-10',
               '2012-05-11', '2012-05-12', '2012-05-13', '2012-05-14',
               '2012-05-15', '2012-05-16', '2012-05-17', '2012-05-18',
               '2012-05-19', '2012-05-20', '2012-05-21', '2012-05-22',
               '2012-05-23', '2012-05-24', '2012-05-25', '2012-05-26',
               '2012-05-27', '2012-05-28', '2012-05-29', '2012-05-30',
               '2012-05-31', '2012-06-01'],
              dtype='datetime64[ns]', freq='D')



In [57]:

    
pd.date_range(start='4/1/2012', periods=20)









    Out[57]:





DatetimeIndex(['2012-04-01', '2012-04-02', '2012-04-03', '2012-04-04',
               '2012-04-05', '2012-04-06', '2012-04-07', '2012-04-08',
               '2012-04-09', '2012-04-10', '2012-04-11', '2012-04-12',
               '2012-04-13', '2012-04-14', '2012-04-15', '2012-04-16',
               '2012-04-17', '2012-04-18', '2012-04-19', '2012-04-20'],
              dtype='datetime64[ns]', freq='D')



In [58]:

    
pd.date_range(end='6/1/2012', periods=20)









    Out[58]:





DatetimeIndex(['2012-05-13', '2012-05-14', '2012-05-15', '2012-05-16',
               '2012-05-17', '2012-05-18', '2012-05-19', '2012-05-20',
               '2012-05-21', '2012-05-22', '2012-05-23', '2012-05-24',
               '2012-05-25', '2012-05-26', '2012-05-27', '2012-05-28',
               '2012-05-29', '2012-05-30', '2012-05-31', '2012-06-01'],
              dtype='datetime64[ns]', freq='D')



In [59]:

    
pd.date_range('1/1/2000', '12/1/2000', freq='BM')









    Out[59]:





DatetimeIndex(['2000-01-31', '2000-02-29', '2000-03-31', '2000-04-28',
               '2000-05-31', '2000-06-30', '2000-07-31', '2000-08-31',
               '2000-09-29', '2000-10-31', '2000-11-30'],
              dtype='datetime64[ns]', freq='BM')



In [60]:

    
pd.date_range('5/2/2012 12:56:31', periods=5)









    Out[60]:





DatetimeIndex(['2012-05-02 12:56:31', '2012-05-03 12:56:31',
               '2012-05-04 12:56:31', '2012-05-05 12:56:31',
               '2012-05-06 12:56:31'],
              dtype='datetime64[ns]', freq='D')



In [61]:

    
pd.date_range('5/2/2012 12:56:31', periods=5, normalize=True)









    Out[61]:





DatetimeIndex(['2012-05-02', '2012-05-03', '2012-05-04', '2012-05-05',
               '2012-05-06'],
              dtype='datetime64[ns]', freq='D')

Frequencies and Date Offsets



In [62]:

    
from pandas.tseries.offsets import Hour, Minute
hour = Hour()
hour









    Out[62]:





<Hour>



In [63]:

    
four_hours = Hour(4)
four_hours









    Out[63]:





<4 * Hours>



In [64]:

    
pd.date_range('1/1/2000', '1/3/2000 23:59', freq='4h')









    Out[64]:





DatetimeIndex(['2000-01-01 00:00:00', '2000-01-01 04:00:00',
               '2000-01-01 08:00:00', '2000-01-01 12:00:00',
               '2000-01-01 16:00:00', '2000-01-01 20:00:00',
               '2000-01-02 00:00:00', '2000-01-02 04:00:00',
               '2000-01-02 08:00:00', '2000-01-02 12:00:00',
               '2000-01-02 16:00:00', '2000-01-02 20:00:00',
               '2000-01-03 00:00:00', '2000-01-03 04:00:00',
               '2000-01-03 08:00:00', '2000-01-03 12:00:00',
               '2000-01-03 16:00:00', '2000-01-03 20:00:00'],
              dtype='datetime64[ns]', freq='4H')



In [65]:

    
Hour(2) + Minute(30)









    Out[65]:





<150 * Minutes>



In [66]:

    
pd.date_range('1/1/2000', periods=10, freq='1h30min')









    Out[66]:





DatetimeIndex(['2000-01-01 00:00:00', '2000-01-01 01:30:00',
               '2000-01-01 03:00:00', '2000-01-01 04:30:00',
               '2000-01-01 06:00:00', '2000-01-01 07:30:00',
               '2000-01-01 09:00:00', '2000-01-01 10:30:00',
               '2000-01-01 12:00:00', '2000-01-01 13:30:00'],
              dtype='datetime64[ns]', freq='90T')

Week of month dates



In [67]:

    
rng = pd.date_range('1/1/2012', '9/1/2012', freq='WOM-3FRI')
list(rng)









    Out[67]:





[Timestamp('2012-01-20 00:00:00', freq='WOM-3FRI'),
 Timestamp('2012-02-17 00:00:00', freq='WOM-3FRI'),
 Timestamp('2012-03-16 00:00:00', freq='WOM-3FRI'),
 Timestamp('2012-04-20 00:00:00', freq='WOM-3FRI'),
 Timestamp('2012-05-18 00:00:00', freq='WOM-3FRI'),
 Timestamp('2012-06-15 00:00:00', freq='WOM-3FRI'),
 Timestamp('2012-07-20 00:00:00', freq='WOM-3FRI'),
 Timestamp('2012-08-17 00:00:00', freq='WOM-3FRI')]

Shifting (leading and lagging) data



In [68]:

    
ts = Series(np.random.randn(4),
            index=pd.date_range('1/1/2000', periods=4, freq='M'))
ts









    Out[68]:





2000-01-31   -0.485025
2000-02-29    0.693190
2000-03-31   -1.280487
2000-04-30    0.426793
Freq: M, dtype: float64



In [69]:

    
ts.shift(2)









    Out[69]:





2000-01-31         NaN
2000-02-29         NaN
2000-03-31   -0.485025
2000-04-30    0.693190
Freq: M, dtype: float64



In [70]:

    
ts.shift(-2)









    Out[70]:





2000-01-31   -1.280487
2000-02-29    0.426793
2000-03-31         NaN
2000-04-30         NaN
Freq: M, dtype: float64



In [71]:

    
ts / ts.shift(1) - 1









    Out[71]:





2000-01-31         NaN
2000-02-29   -2.429184
2000-03-31   -2.847237
2000-04-30   -1.333305
Freq: M, dtype: float64



In [72]:

    
ts.shift(2, freq='M')









    Out[72]:





2000-03-31   -0.485025
2000-04-30    0.693190
2000-05-31   -1.280487
2000-06-30    0.426793
Freq: M, dtype: float64



In [73]:

    
ts.shift(3, freq='D')









    Out[73]:





2000-02-03   -0.485025
2000-03-03    0.693190
2000-04-03   -1.280487
2000-05-03    0.426793
dtype: float64



In [74]:

    
ts.shift(1, freq='3D')









    Out[74]:





2000-02-03   -0.485025
2000-03-03    0.693190
2000-04-03   -1.280487
2000-05-03    0.426793
dtype: float64



In [75]:

    
ts.shift(1, freq='90T')









    Out[75]:





2000-01-31 01:30:00   -0.485025
2000-02-29 01:30:00    0.693190
2000-03-31 01:30:00   -1.280487
2000-04-30 01:30:00    0.426793
Freq: M, dtype: float64

Shifting dates with offsets



In [76]:

    
from pandas.tseries.offsets import Day, MonthEnd
now = datetime(2011, 11, 17)
now + 3 * Day()









    Out[76]:





Timestamp('2011-11-20 00:00:00')



In [77]:

    
now + MonthEnd()









    Out[77]:





Timestamp('2011-11-30 00:00:00')



In [78]:

    
now + MonthEnd(2)









    Out[78]:





Timestamp('2011-12-31 00:00:00')



In [79]:

    
offset = MonthEnd()
offset.rollforward(now)









    Out[79]:





Timestamp('2011-11-30 00:00:00')



In [80]:

    
offset.rollback(now)









    Out[80]:





Timestamp('2011-10-31 00:00:00')



In [81]:

    
ts = Series(np.random.randn(20),
            index=pd.date_range('1/15/2000', periods=20, freq='4d'))
ts.groupby(offset.rollforward).mean()









    Out[81]:





2000-01-31    0.484219
2000-02-29   -0.106886
2000-03-31   -0.281010
dtype: float64



In [83]:

    
ts.resample('M').mean()









    Out[83]:





2000-01-31    0.484219
2000-02-29   -0.106886
2000-03-31   -0.281010
Freq: M, dtype: float64



In [ ]:

	Colorado	Texas	New York	Ohio
2001-05-02	-0.473243	0.814446	-0.475554	-0.042332
2001-05-09	0.363848	0.372462	0.999984	-1.011203
2001-05-16	-0.525765	-0.338188	0.801046	0.048134
2001-05-23	-0.222229	0.399359	1.232311	-0.074315
2001-05-30	-0.149293	-0.596474	1.870496	-0.051485