This is a personal project around the causal impact analysis of grey disruptions using windowed PCA. Sky is the limit


In [2]:
%matplotlib inline
import os
import sys

import pandas as pd
import pandas_datareader.data as web
import numpy as np

import statsmodels.formula.api as smf
import statsmodels.tsa.api as smt
import statsmodels.api as sm
import scipy.stats as scs
from arch import arch_model

import matplotlib.pyplot as plt
import matplotlib as mpl

#print('Machine: {}\n'.format(os.uname()))
#print(sys.version)

def tsplot(title, y, lags=None, figsize=(10, 8), style='bmh'):
    if not isinstance(y, pd.Series):
        y = pd.Series(y)
    with plt.style.context(style):    
        fig = plt.figure(figsize=figsize)
        #mpl.rcParams['font.family'] = 'Ubuntu Mono'
        layout = (3, 2)
        ts_ax = plt.subplot2grid(layout, (0, 0), colspan=2)
        acf_ax = plt.subplot2grid(layout, (1, 0))
        pacf_ax = plt.subplot2grid(layout, (1, 1))
        qq_ax = plt.subplot2grid(layout, (2, 0))
        pp_ax = plt.subplot2grid(layout, (2, 1))
        
        y.plot(ax=ts_ax)
        title_with_infos="{:s} $(\mu={:.3f}, \sigma={:.3f})$".format(title,y.mean(),y.std())
        ts_ax.set_title(title_with_infos)
        smt.graphics.plot_acf(y, lags=lags, ax=acf_ax, alpha=0.5)
        smt.graphics.plot_pacf(y, lags=lags, ax=pacf_ax, alpha=0.5)
        sm.qqplot(y, line='s', ax=qq_ax)
        qq_ax.set_title('QQ Plot')        
        scs.probplot(y, sparams=(y.mean(), y.std()), plot=pp_ax)

        plt.tight_layout()
        #print("@tsplot(); $\mu={:.3f}$, variance={:.3f}, sigma={:.3f}".format(y.mean(), y.var(), y.std()))
    return


/Users/sbergeron/anaconda/lib/python2.7/site-packages/statsmodels/compat/pandas.py:56: FutureWarning: The pandas.core.datetools module is deprecated and will be removed in a future version. Please use the pandas.tseries module instead.
  from pandas.core import datetools

In [3]:
np.random.seed(1)
n_samples = 1000

# plot of discrete white noise
randser = np.random.normal(size=n_samples)
tsplot('Discrete white noise', randser, lags=30)



In [4]:
x = w = randser
for t in range(n_samples):
    x[t] = x[t-1] + w[t]

_ = tsplot('Random walk', x, lags=30)
_ = tsplot('First difference of random walk', np.diff(x), lags=30)



In [26]:
df = pd.read_csv('../Specter/loganalyzer/resources/cache/cxseed/scheduling.schedule.builder.CatchupAsOfTodaySimulation/nmon/rawdata/cxseed-a-back.csv')
df.plot(x='Time', y=['NETETOTAL total-KB/s','NETETOTAL total-read-KB/s','NETETOTAL total-write-KB/s'], style='o-')


Out[26]:
<matplotlib.axes._subplots.AxesSubplot at 0x1135a7750>

In [24]:
#pd.set_option('display.max_rows', len(df))
#print(df[['NETETOTAL total-KB/s','NETETOTAL total-read-KB/s','NETETOTAL total-write-KB/s']])
#list(df.columns.values)


    NETETOTAL total-KB/s  NETETOTAL total-read-KB/s  \
0                 1396.5                      613.0   
1                  405.0                      164.1   
2                   46.0                       17.4   
3                   33.7                       17.2   
4                   29.1                        8.8   
5                   38.4                       16.3   
6                    7.3                        2.0   
7                   28.4                        7.9   
8                  801.8                      352.2   
9                  807.5                      354.7   
10                2739.7                     1196.5   
11                3413.0                     1466.6   
12                4201.9                     1838.9   
13                4230.5                     1858.3   
14                4225.6                     1857.5   
15                4286.0                     1883.3   
16                4175.1                     1834.0   
17                4151.2                     1824.0   
18                4132.7                     1815.4   
19                4000.6                     1757.5   
20                4199.0                     1846.1   
21                4185.9                     1839.0   
22                4192.1                     1841.7   
23                4322.6                     1899.2   
24                4246.3                     1865.6   
25                4098.4                     1800.6   
26                4158.2                     1828.3   
27                4215.8                     1852.4   
28                3950.0                     1735.5   
29                4137.3                     1818.0   
30                3852.2                     1692.6   
31                4001.7                     1758.4   
32                4126.8                     1814.8   
33                4118.4                     1809.9   
34                4270.0                     1876.6   
35                4160.5                     1828.3   
36                4146.2                     1821.8   
37                4219.4                     1854.2   
38                4246.6                     1867.5   
39                4093.9                     1799.3   
40                4093.3                     1798.5   
41                4215.8                     1852.7   
42                4060.4                     1784.3   
43                4086.1                     1795.8   
44                7009.8                     4339.9   
45               10480.0                     1952.7   
46                4164.4                     1825.9   
47                7837.2                     4878.0   
48               11328.1                     2679.4   
49                6979.9                     1742.7   
50                4152.8                     1808.9   
51                4340.0                     1880.9   
52                3987.6                     1732.4   
53                4093.8                     1780.1   
54                4046.1                     1761.2   
55                3942.5                     1724.1   
56                4139.8                     1813.1   
57                4000.4                     1751.1   
58                4024.1                     1762.4   
59                4162.2                     1823.8   
60                3907.2                     1711.7   
61                3877.6                     1699.6   
62                4089.8                     1794.2   
63                3983.7                     1746.6   
64                4111.7                     1803.1   
65                3991.8                     1751.0   
66                4095.0                     1796.1   
67                4141.8                     1817.4   
68                4092.8                     1797.1   
69                4034.4                     1770.1   
70                3964.2                     1739.3   
71                4213.4                     1849.5   
72                4232.8                     1857.6   
73                4017.2                     1763.3   
74                4381.4                     1924.7   
75                3942.7                     1730.7   
76                4169.7                     1830.3   
77                4023.9                     1766.9   
78                4093.2                     1797.2   
79                4228.2                     1856.6   
80                4061.6                     1784.7   
81                4235.5                     1859.8   
82                4037.2                     1771.9   
83                3006.3                     1309.6   
84                1918.6                      842.2   
85                1842.3                      808.4   
86                1411.7                      612.4   
87                 213.5                       91.0   
88                  99.9                       43.2   
89                  85.6                       34.7   

    NETETOTAL total-write-KB/s  
0                        783.5  
1                        240.9  
2                         28.6  
3                         16.5  
4                         20.3  
5                         22.1  
6                          5.3  
7                         20.5  
8                        449.6  
9                        452.8  
10                      1543.2  
11                      1946.4  
12                      2363.0  
13                      2372.2  
14                      2368.1  
15                      2402.7  
16                      2341.1  
17                      2327.2  
18                      2317.3  
19                      2243.1  
20                      2352.9  
21                      2346.9  
22                      2350.4  
23                      2423.4  
24                      2380.7  
25                      2297.8  
26                      2329.9  
27                      2363.4  
28                      2214.5  
29                      2319.3  
30                      2159.6  
31                      2243.3  
32                      2312.0  
33                      2308.5  
34                      2393.4  
35                      2332.2  
36                      2324.4  
37                      2365.2  
38                      2379.1  
39                      2294.6  
40                      2294.8  
41                      2363.1  
42                      2276.1  
43                      2290.3  
44                      2669.9  
45                      8527.3  
46                      2338.5  
47                      2959.2  
48                      8648.7  
49                      5237.2  
50                      2343.9  
51                      2459.1  
52                      2255.2  
53                      2313.7  
54                      2284.9  
55                      2218.4  
56                      2326.7  
57                      2249.3  
58                      2261.7  
59                      2338.4  
60                      2195.5  
61                      2178.0  
62                      2295.6  
63                      2237.1  
64                      2308.6  
65                      2240.8  
66                      2298.9  
67                      2324.4  
68                      2295.7  
69                      2264.3  
70                      2224.9  
71                      2363.9  
72                      2375.2  
73                      2253.9  
74                      2456.7  
75                      2212.0  
76                      2339.4  
77                      2257.0  
78                      2296.0  
79                      2371.6  
80                      2276.9  
81                      2375.7  
82                      2265.3  
83                      1696.7  
84                      1076.4  
85                      1033.9  
86                       799.3  
87                       122.5  
88                        56.7  
89                        50.9  

In [ ]: