Chapter_10_part_01


Time series

Date and Time Data Types and Tools


In [2]:
from datetime import datetime

In [4]:
now = datetime.now()
now


Out[4]:
datetime.datetime(2017, 6, 28, 15, 11, 55, 925801)

In [5]:
now.year, now.month, now.day


Out[5]:
(2017, 6, 28)

In [6]:
delta = datetime(2011, 1, 7) - datetime(2008, 6, 24, 8, 15)
delta


Out[6]:
datetime.timedelta(926, 56700)

In [7]:
delta.days


Out[7]:
926

In [8]:
delta.seconds


Out[8]:
56700

In [9]:
from datetime import timedelta
start = datetime(2011, 1, 7)
start + timedelta(12)


Out[9]:
datetime.datetime(2011, 1, 19, 0, 0)

In [10]:
start - 2 * timedelta(12)


Out[10]:
datetime.datetime(2010, 12, 14, 0, 0)

Converting between string and datetime


In [11]:
stamp = datetime(2011, 1, 3)
str(stamp)


Out[11]:
'2011-01-03 00:00:00'

In [13]:
stamp.strftime('%Y-%m-%d')


Out[13]:
'2011-01-03'

In [14]:
value = '2011-01-03'
datetime.strptime(value, '%Y-%m-%d')


Out[14]:
datetime.datetime(2011, 1, 3, 0, 0)

In [15]:
datestrs = ['7/6/2011', '8/6/2011']
[datetime.strptime(x, '%m/%d/%Y') for x in datestrs]


Out[15]:
[datetime.datetime(2011, 7, 6, 0, 0), datetime.datetime(2011, 8, 6, 0, 0)]

In [16]:
from dateutil.parser import parse
parse('2011-01-03')


Out[16]:
datetime.datetime(2011, 1, 3, 0, 0)

In [17]:
parse('Jan 31, 1997 10:45 PM')


Out[17]:
datetime.datetime(1997, 1, 31, 22, 45)

In [18]:
parse('6/12/2011', dayfirst=True)


Out[18]:
datetime.datetime(2011, 12, 6, 0, 0)

In [19]:
datestrs


Out[19]:
['7/6/2011', '8/6/2011']

In [20]:
import pandas as pd

In [21]:
pd.to_datetime(datestrs)


Out[21]:
DatetimeIndex(['2011-07-06', '2011-08-06'], dtype='datetime64[ns]', freq=None)

In [25]:
idx = pd.to_datetime(datestrs + [None])
idx


Out[25]:
DatetimeIndex(['2011-07-06', '2011-08-06', 'NaT'], dtype='datetime64[ns]', freq=None)

In [26]:
pd.isnull(pd.to_datetime(datestrs + [None]))


Out[26]:
array([False, False,  True], dtype=bool)

Time Series Basics


In [29]:
from pandas import Series, DataFrame
import numpy as np
dates = [datetime(2011, 1, 2), datetime(2011, 1, 5), datetime(2011, 1, 7),
         datetime(2011, 1, 8), datetime(2011, 1, 10), datetime(2011, 1, 12)]
ts = Series(np.random.randn(6), index=dates)
ts


Out[29]:
2011-01-02    1.111272
2011-01-05    0.221653
2011-01-07    0.674424
2011-01-08   -0.749457
2011-01-10    1.922602
2011-01-12    0.849016
dtype: float64

In [30]:
type(ts)


Out[30]:
pandas.core.series.Series

In [31]:
ts + ts[::2]


Out[31]:
2011-01-02    2.222544
2011-01-05         NaN
2011-01-07    1.348848
2011-01-08         NaN
2011-01-10    3.845205
2011-01-12         NaN
dtype: float64

In [32]:
ts.index.dtype


Out[32]:
dtype('<M8[ns]')

In [33]:
ts.index[0]


Out[33]:
Timestamp('2011-01-02 00:00:00')

Indexing, selection, subsetting


In [34]:
ts['1/10/2011']


Out[34]:
1.92260229416429

In [35]:
ts['20110110']


Out[35]:
1.92260229416429

In [36]:
longer_ts = Series(np.random.randn(1000),
                   index=pd.date_range('1/1/2000', periods=1000))
longer_ts


Out[36]:
2000-01-01   -0.429690
2000-01-02   -0.988769
2000-01-03   -0.024645
2000-01-04    2.430567
2000-01-05   -1.351163
2000-01-06    0.503308
2000-01-07    0.194014
2000-01-08   -0.697142
2000-01-09    0.191641
2000-01-10    0.330984
2000-01-11   -0.846752
2000-01-12    0.634315
2000-01-13   -0.893407
2000-01-14   -1.650352
2000-01-15   -0.791815
2000-01-16   -1.267505
2000-01-17   -0.762687
2000-01-18   -0.896668
2000-01-19    1.720866
2000-01-20    1.661988
2000-01-21    1.708285
2000-01-22   -0.330619
2000-01-23   -0.864698
2000-01-24   -0.200427
2000-01-25    0.509097
2000-01-26    1.675818
2000-01-27   -0.592886
2000-01-28    1.328173
2000-01-29   -1.123267
2000-01-30    1.319367
                ...   
2002-08-28   -0.909667
2002-08-29   -0.824423
2002-08-30   -0.705050
2002-08-31   -0.393563
2002-09-01   -0.628749
2002-09-02   -1.085462
2002-09-03    0.142824
2002-09-04    0.592543
2002-09-05    0.490188
2002-09-06    1.289383
2002-09-07   -1.250163
2002-09-08    0.590002
2002-09-09   -0.075362
2002-09-10    1.044168
2002-09-11   -0.088020
2002-09-12   -0.836371
2002-09-13    1.481651
2002-09-14    0.292208
2002-09-15    1.282557
2002-09-16    0.844125
2002-09-17    0.217562
2002-09-18    0.472906
2002-09-19    0.266230
2002-09-20    0.003674
2002-09-21    0.294167
2002-09-22    0.324055
2002-09-23   -0.567444
2002-09-24    1.445333
2002-09-25    1.348238
2002-09-26   -1.510428
Freq: D, dtype: float64

In [37]:
longer_ts['2001']


Out[37]:
2001-01-01    1.423174
2001-01-02    2.655359
2001-01-03   -1.964971
2001-01-04   -0.751284
2001-01-05    0.835949
2001-01-06   -0.219525
2001-01-07   -0.433020
2001-01-08   -0.611016
2001-01-09   -0.803291
2001-01-10    0.805506
2001-01-11    0.291593
2001-01-12   -0.263987
2001-01-13    0.745169
2001-01-14   -1.085306
2001-01-15   -0.760404
2001-01-16    0.003680
2001-01-17   -0.501209
2001-01-18    0.000269
2001-01-19   -1.283880
2001-01-20    0.297868
2001-01-21   -0.701593
2001-01-22   -0.662293
2001-01-23   -0.243264
2001-01-24    0.617356
2001-01-25    0.606253
2001-01-26   -0.225016
2001-01-27    0.765918
2001-01-28   -0.122033
2001-01-29   -0.397498
2001-01-30    1.280038
                ...   
2001-12-02    0.848705
2001-12-03   -0.260151
2001-12-04   -0.373202
2001-12-05   -0.395652
2001-12-06   -0.264092
2001-12-07    1.370772
2001-12-08    0.269687
2001-12-09    1.174350
2001-12-10   -0.268601
2001-12-11    1.465140
2001-12-12   -0.796310
2001-12-13    0.162920
2001-12-14    0.211350
2001-12-15    3.106605
2001-12-16    1.905970
2001-12-17   -1.153820
2001-12-18   -1.115190
2001-12-19    0.395947
2001-12-20    0.166124
2001-12-21    0.451831
2001-12-22    0.792277
2001-12-23   -0.962898
2001-12-24    1.275114
2001-12-25   -1.523152
2001-12-26   -1.396730
2001-12-27   -0.437941
2001-12-28   -1.195787
2001-12-29    0.001429
2001-12-30    0.388284
2001-12-31    1.045000
Freq: D, dtype: float64

In [38]:
longer_ts['2001-05']


Out[38]:
2001-05-01    0.042114
2001-05-02   -0.714940
2001-05-03    0.517785
2001-05-04   -0.587098
2001-05-05   -0.096823
2001-05-06   -0.976581
2001-05-07    1.278264
2001-05-08    1.890005
2001-05-09   -0.260056
2001-05-10    0.955862
2001-05-11    0.062605
2001-05-12   -0.317040
2001-05-13   -0.010676
2001-05-14   -0.893382
2001-05-15   -0.275795
2001-05-16    0.491210
2001-05-17    0.381374
2001-05-18    0.283364
2001-05-19    0.210997
2001-05-20    0.980941
2001-05-21    0.568281
2001-05-22   -0.434008
2001-05-23    0.746367
2001-05-24   -0.077535
2001-05-25   -1.610058
2001-05-26   -2.215233
2001-05-27   -0.064517
2001-05-28   -0.716334
2001-05-29   -1.110473
2001-05-30   -0.967791
2001-05-31   -0.976880
Freq: D, dtype: float64

In [39]:
ts[datetime(2011, 1, 7):]


Out[39]:
2011-01-07    0.674424
2011-01-08   -0.749457
2011-01-10    1.922602
2011-01-12    0.849016
dtype: float64

In [40]:
ts


Out[40]:
2011-01-02    1.111272
2011-01-05    0.221653
2011-01-07    0.674424
2011-01-08   -0.749457
2011-01-10    1.922602
2011-01-12    0.849016
dtype: float64

In [41]:
ts['1/6/2011':'1/11/2011']


Out[41]:
2011-01-07    0.674424
2011-01-08   -0.749457
2011-01-10    1.922602
dtype: float64

In [42]:
ts.truncate(after='1/9/2011')


Out[42]:
2011-01-02    1.111272
2011-01-05    0.221653
2011-01-07    0.674424
2011-01-08   -0.749457
dtype: float64

In [43]:
dates = pd.date_range('1/1/2000', periods=100, freq='W-WED')
long_df = DataFrame(np.random.randn(100, 4),
                    index=dates,
                    columns=['Colorado', 'Texas', 'New York', 'Ohio'])
long_df.ix['5-2001']


Out[43]:
Colorado Texas New York Ohio
2001-05-02 -0.473243 0.814446 -0.475554 -0.042332
2001-05-09 0.363848 0.372462 0.999984 -1.011203
2001-05-16 -0.525765 -0.338188 0.801046 0.048134
2001-05-23 -0.222229 0.399359 1.232311 -0.074315
2001-05-30 -0.149293 -0.596474 1.870496 -0.051485

Time series with duplicate indices


In [44]:
dates = pd.DatetimeIndex(['1/1/2000', '1/2/2000', '1/2/2000', '1/2/2000',
                          '1/3/2000'])
dup_ts = Series(np.arange(5), index=dates)
dup_ts


Out[44]:
2000-01-01    0
2000-01-02    1
2000-01-02    2
2000-01-02    3
2000-01-03    4
dtype: int64

In [49]:
dup_ts.index.is_unique


Out[49]:
False

In [50]:
dup_ts['1/3/2000']


Out[50]:
4

In [51]:
dup_ts['1/2/2000']


Out[51]:
2000-01-02    1
2000-01-02    2
2000-01-02    3
dtype: int64

In [52]:
grouped = dup_ts.groupby(level=0)
grouped.mean()


Out[52]:
2000-01-01    0
2000-01-02    2
2000-01-03    4
dtype: int64

In [53]:
grouped.count()


Out[53]:
2000-01-01    1
2000-01-02    3
2000-01-03    1
dtype: int64

Date ranges, Frequencies, and Shifting


In [54]:
ts


Out[54]:
2011-01-02    1.111272
2011-01-05    0.221653
2011-01-07    0.674424
2011-01-08   -0.749457
2011-01-10    1.922602
2011-01-12    0.849016
dtype: float64

In [55]:
ts.resample('D')


Out[55]:
DatetimeIndexResampler [freq=<Day>, axis=0, closed=left, label=left, convention=start, base=0]

Generating date ranges


In [56]:
index = pd.date_range('4/1/2012', '6/1/2012')
index


Out[56]:
DatetimeIndex(['2012-04-01', '2012-04-02', '2012-04-03', '2012-04-04',
               '2012-04-05', '2012-04-06', '2012-04-07', '2012-04-08',
               '2012-04-09', '2012-04-10', '2012-04-11', '2012-04-12',
               '2012-04-13', '2012-04-14', '2012-04-15', '2012-04-16',
               '2012-04-17', '2012-04-18', '2012-04-19', '2012-04-20',
               '2012-04-21', '2012-04-22', '2012-04-23', '2012-04-24',
               '2012-04-25', '2012-04-26', '2012-04-27', '2012-04-28',
               '2012-04-29', '2012-04-30', '2012-05-01', '2012-05-02',
               '2012-05-03', '2012-05-04', '2012-05-05', '2012-05-06',
               '2012-05-07', '2012-05-08', '2012-05-09', '2012-05-10',
               '2012-05-11', '2012-05-12', '2012-05-13', '2012-05-14',
               '2012-05-15', '2012-05-16', '2012-05-17', '2012-05-18',
               '2012-05-19', '2012-05-20', '2012-05-21', '2012-05-22',
               '2012-05-23', '2012-05-24', '2012-05-25', '2012-05-26',
               '2012-05-27', '2012-05-28', '2012-05-29', '2012-05-30',
               '2012-05-31', '2012-06-01'],
              dtype='datetime64[ns]', freq='D')

In [57]:
pd.date_range(start='4/1/2012', periods=20)


Out[57]:
DatetimeIndex(['2012-04-01', '2012-04-02', '2012-04-03', '2012-04-04',
               '2012-04-05', '2012-04-06', '2012-04-07', '2012-04-08',
               '2012-04-09', '2012-04-10', '2012-04-11', '2012-04-12',
               '2012-04-13', '2012-04-14', '2012-04-15', '2012-04-16',
               '2012-04-17', '2012-04-18', '2012-04-19', '2012-04-20'],
              dtype='datetime64[ns]', freq='D')

In [58]:
pd.date_range(end='6/1/2012', periods=20)


Out[58]:
DatetimeIndex(['2012-05-13', '2012-05-14', '2012-05-15', '2012-05-16',
               '2012-05-17', '2012-05-18', '2012-05-19', '2012-05-20',
               '2012-05-21', '2012-05-22', '2012-05-23', '2012-05-24',
               '2012-05-25', '2012-05-26', '2012-05-27', '2012-05-28',
               '2012-05-29', '2012-05-30', '2012-05-31', '2012-06-01'],
              dtype='datetime64[ns]', freq='D')

In [59]:
pd.date_range('1/1/2000', '12/1/2000', freq='BM')


Out[59]:
DatetimeIndex(['2000-01-31', '2000-02-29', '2000-03-31', '2000-04-28',
               '2000-05-31', '2000-06-30', '2000-07-31', '2000-08-31',
               '2000-09-29', '2000-10-31', '2000-11-30'],
              dtype='datetime64[ns]', freq='BM')

In [60]:
pd.date_range('5/2/2012 12:56:31', periods=5)


Out[60]:
DatetimeIndex(['2012-05-02 12:56:31', '2012-05-03 12:56:31',
               '2012-05-04 12:56:31', '2012-05-05 12:56:31',
               '2012-05-06 12:56:31'],
              dtype='datetime64[ns]', freq='D')

In [61]:
pd.date_range('5/2/2012 12:56:31', periods=5, normalize=True)


Out[61]:
DatetimeIndex(['2012-05-02', '2012-05-03', '2012-05-04', '2012-05-05',
               '2012-05-06'],
              dtype='datetime64[ns]', freq='D')

Frequencies and Date Offsets


In [62]:
from pandas.tseries.offsets import Hour, Minute
hour = Hour()
hour


Out[62]:
<Hour>

In [63]:
four_hours = Hour(4)
four_hours


Out[63]:
<4 * Hours>

In [64]:
pd.date_range('1/1/2000', '1/3/2000 23:59', freq='4h')


Out[64]:
DatetimeIndex(['2000-01-01 00:00:00', '2000-01-01 04:00:00',
               '2000-01-01 08:00:00', '2000-01-01 12:00:00',
               '2000-01-01 16:00:00', '2000-01-01 20:00:00',
               '2000-01-02 00:00:00', '2000-01-02 04:00:00',
               '2000-01-02 08:00:00', '2000-01-02 12:00:00',
               '2000-01-02 16:00:00', '2000-01-02 20:00:00',
               '2000-01-03 00:00:00', '2000-01-03 04:00:00',
               '2000-01-03 08:00:00', '2000-01-03 12:00:00',
               '2000-01-03 16:00:00', '2000-01-03 20:00:00'],
              dtype='datetime64[ns]', freq='4H')

In [65]:
Hour(2) + Minute(30)


Out[65]:
<150 * Minutes>

In [66]:
pd.date_range('1/1/2000', periods=10, freq='1h30min')


Out[66]:
DatetimeIndex(['2000-01-01 00:00:00', '2000-01-01 01:30:00',
               '2000-01-01 03:00:00', '2000-01-01 04:30:00',
               '2000-01-01 06:00:00', '2000-01-01 07:30:00',
               '2000-01-01 09:00:00', '2000-01-01 10:30:00',
               '2000-01-01 12:00:00', '2000-01-01 13:30:00'],
              dtype='datetime64[ns]', freq='90T')

Week of month dates


In [67]:
rng = pd.date_range('1/1/2012', '9/1/2012', freq='WOM-3FRI')
list(rng)


Out[67]:
[Timestamp('2012-01-20 00:00:00', freq='WOM-3FRI'),
 Timestamp('2012-02-17 00:00:00', freq='WOM-3FRI'),
 Timestamp('2012-03-16 00:00:00', freq='WOM-3FRI'),
 Timestamp('2012-04-20 00:00:00', freq='WOM-3FRI'),
 Timestamp('2012-05-18 00:00:00', freq='WOM-3FRI'),
 Timestamp('2012-06-15 00:00:00', freq='WOM-3FRI'),
 Timestamp('2012-07-20 00:00:00', freq='WOM-3FRI'),
 Timestamp('2012-08-17 00:00:00', freq='WOM-3FRI')]

Shifting (leading and lagging) data


In [68]:
ts = Series(np.random.randn(4),
            index=pd.date_range('1/1/2000', periods=4, freq='M'))
ts


Out[68]:
2000-01-31   -0.485025
2000-02-29    0.693190
2000-03-31   -1.280487
2000-04-30    0.426793
Freq: M, dtype: float64

In [69]:
ts.shift(2)


Out[69]:
2000-01-31         NaN
2000-02-29         NaN
2000-03-31   -0.485025
2000-04-30    0.693190
Freq: M, dtype: float64

In [70]:
ts.shift(-2)


Out[70]:
2000-01-31   -1.280487
2000-02-29    0.426793
2000-03-31         NaN
2000-04-30         NaN
Freq: M, dtype: float64

In [71]:
ts / ts.shift(1) - 1


Out[71]:
2000-01-31         NaN
2000-02-29   -2.429184
2000-03-31   -2.847237
2000-04-30   -1.333305
Freq: M, dtype: float64

In [72]:
ts.shift(2, freq='M')


Out[72]:
2000-03-31   -0.485025
2000-04-30    0.693190
2000-05-31   -1.280487
2000-06-30    0.426793
Freq: M, dtype: float64

In [73]:
ts.shift(3, freq='D')


Out[73]:
2000-02-03   -0.485025
2000-03-03    0.693190
2000-04-03   -1.280487
2000-05-03    0.426793
dtype: float64

In [74]:
ts.shift(1, freq='3D')


Out[74]:
2000-02-03   -0.485025
2000-03-03    0.693190
2000-04-03   -1.280487
2000-05-03    0.426793
dtype: float64

In [75]:
ts.shift(1, freq='90T')


Out[75]:
2000-01-31 01:30:00   -0.485025
2000-02-29 01:30:00    0.693190
2000-03-31 01:30:00   -1.280487
2000-04-30 01:30:00    0.426793
Freq: M, dtype: float64

Shifting dates with offsets


In [76]:
from pandas.tseries.offsets import Day, MonthEnd
now = datetime(2011, 11, 17)
now + 3 * Day()


Out[76]:
Timestamp('2011-11-20 00:00:00')

In [77]:
now + MonthEnd()


Out[77]:
Timestamp('2011-11-30 00:00:00')

In [78]:
now + MonthEnd(2)


Out[78]:
Timestamp('2011-12-31 00:00:00')

In [79]:
offset = MonthEnd()
offset.rollforward(now)


Out[79]:
Timestamp('2011-11-30 00:00:00')

In [80]:
offset.rollback(now)


Out[80]:
Timestamp('2011-10-31 00:00:00')

In [81]:
ts = Series(np.random.randn(20),
            index=pd.date_range('1/15/2000', periods=20, freq='4d'))
ts.groupby(offset.rollforward).mean()


Out[81]:
2000-01-31    0.484219
2000-02-29   -0.106886
2000-03-31   -0.281010
dtype: float64

In [83]:
ts.resample('M').mean()


Out[83]:
2000-01-31    0.484219
2000-02-29   -0.106886
2000-03-31   -0.281010
Freq: M, dtype: float64

In [ ]: