spot_analysis_corr


Spot resources Analytics

Here we perform some initial process and analysis on the dataset.


With static dataset, e.g. load the grabbed data.


In [197]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn
%matplotlib inline
import time
import dateutil.parser

In [3]:
# The filtered file make sure to use the fi
filename = 'data/aws-recent-filtered.csv'
#filename = './data/subset.csv'

In [4]:
start = time.time()
# parse the data file and extra the results
df = pd.read_csv(filename)
print(time.time()-start)
df.head(3)


149.24496603012085
Out[4]:
TimeStamp AvailabilityZone InstanceType SpotPrice
0 2016-12-12 10:44:30 ap-northeast-1a t1.micro 0.0135
1 2016-12-12 10:44:30 ap-northeast-1a t1.micro 0.0035
2 2016-12-12 10:43:50 ap-northeast-1a t1.micro 0.0134

In [5]:
df = df.dropna()
df = df[df.AvailabilityZone != "AvailabilityZone"]
df['TimeStamp'] = pd.to_datetime(df.TimeStamp)
df.index = df.TimeStamp
df = df.drop('TimeStamp', axis=1)
df= df.sort_index()
df.head(3)


Out[5]:
AvailabilityZone InstanceType SpotPrice
TimeStamp
2016-10-06 10:44:39 eu-west-1b p2.16xlarge 184.96
2016-10-06 10:44:39 eu-west-1c p2.16xlarge 184.96
2016-10-06 10:44:39 eu-west-1a p2.16xlarge 184.96


In [6]:
print (len(df))
print (df['InstanceType'].unique())
print (df['AvailabilityZone'].unique())
print (df['SpotPrice'].unique())


94577818
['p2.16xlarge' 'c3.large' 'i2.4xlarge' 'i2.2xlarge' 'm2.4xlarge' 'm3.large'
 'm4.4xlarge' 'm4.xlarge' 't1.micro' 'g2.8xlarge' 'd2.2xlarge' 'c1.xlarge'
 'm1.small' 'i2.xlarge' 'm1.xlarge' 'c4.2xlarge' 'm1.large' 'c4.large'
 'c4.xlarge' 'm3.medium' 'm4.16xlarge' 'p2.8xlarge' 'm1.medium'
 'x1.16xlarge' 'x1.32xlarge' 'c3.4xlarge' 'r3.4xlarge' 'c1.medium'
 'c4.4xlarge' 'i2.8xlarge' 'hi1.4xlarge' 'd2.8xlarge' 'r3.2xlarge'
 'd2.4xlarge' 'm4.10xlarge' 'c3.8xlarge' 'r3.8xlarge' 'c3.2xlarge'
 'c3.xlarge' 'r3.large' 'c4.8xlarge' 'm3.xlarge' 'm4.large' 'd2.xlarge'
 'm2.2xlarge' 'm2.xlarge' 'r3.xlarge' 'm3.2xlarge' 'm4.2xlarge'
 'cr1.8xlarge' 'p2.xlarge' 'cc2.8xlarge' 'cg1.4xlarge' 'cc1.4xlarge'
 'g2.2xlarge' 'r4.large' 'r4.xlarge' 'r4.4xlarge' 'r4.16xlarge'
 'r4.8xlarge' 'r4.2xlarge']
['eu-west-1b' 'eu-west-1c' 'eu-west-1a' 'us-west-2c' 'ap-northeast-2c'
 'ap-northeast-2a' 'us-west-1b' 'us-west-1c' 'us-west-2b' 'us-west-2a'
 'ap-southeast-2c' 'eu-central-1b' 'ap-southeast-2b' 'ap-southeast-2a'
 'eu-central-1a' 'us-east-1a' 'us-east-1e' 'sa-east-1c' 'sa-east-1a'
 'ap-southeast-1a' 'ap-southeast-1b' 'us-east-1d' 'ap-northeast-1c'
 'ap-northeast-1a' 'us-east-1c']
[  1.84960000e+02   1.04100000e-01   1.13600000e+00 ...,   3.05060000e+00
   1.83800000e-02   1.18380000e-01]

Hypothesis #2

For each machine type there exists a region that is more favorable to use, as the market volatility is very low and the prices tend to stay cheaper than the other regions.

With in proving this hypothesis users will be able to find the best region they should be bidding in, as long as latency is not an issue for them.

Data Science tools & Techniques: We can use clustering and classification methods.


In [7]:
def corrGraph(title, df):
    corr_df = df.corr()
    mask = np.zeros_like(corr_df)
    mask[np.triu_indices_from(mask)] = True
    seaborn.heatmap(corr_df, cmap='RdYlGn_r', vmax=1.0, vmin=-1.0 , mask = mask, linewidths=2.5)
    plt.yticks(rotation=0) 
    plt.title(title)
    plt.xticks(rotation=90) 
    plt.show()

In [13]:
# Some info about the data
df = df.truncate(before='2016-10-13 00:00:00', after='2016-12-11 00:00:00')
print (df.index.min())
print (df.index.max())
print(df.index.max()- df.index.min()) 

df.head(3)


2016-10-13 00:00:00
2016-12-11 00:00:00
59 days 00:00:00
Out[13]:
AvailabilityZone InstanceType SpotPrice
TimeStamp
2016-10-13 us-east-1e m3.xlarge 0.1395
2016-10-13 ap-northeast-1a r3.8xlarge 0.4891
2016-10-13 ap-northeast-1a r3.8xlarge 0.5891

In [68]:
depa = df.groupby(['AvailabilityZone', 'InstanceType'])

In [75]:
#depa = awsResampler(df)
# Initialize dictionary of all combos of  empty dfs we want to graph and corr
zonedfs={}
typedfs={}
for item in df['InstanceType'].unique():
    typedfs.update({item: pd.DataFrame()})
for item in df['AvailabilityZone'].unique():
    zonedfs.update({item: pd.DataFrame()})

#Fill zonedfs with dataframes of all machines in that zone pricing
for name, group in depa:
    #We have to create the data frame by merging the rows and the first run we need our first row
    if zonedfs[name[0]].empty:
        #RESAMPLE DATA HOURLY getting the avg of the hour
        zonedfs[name[0]] = group
        zonedfs[name[0]] = zonedfs[name[0]].resample('H').mean()
        zonedfs[name[0]] = zonedfs[name[0]].fillna(method="ffill")
        #zonedfs[name[0]] = zonedfs[name[0]].drop('InstanceType', axis=1).drop(['AvailabilityZone'],axis=1)
        #Remove index so that you can merge
        zonedfs[name[0]] = zonedfs[name[0]].reset_index()
        #print(zonedfs[name[0]].head(40) )
        zonedfs[name[0]].rename(columns = {'SpotPrice':name[1]}, inplace = True)

    else:
        #RESAMPLE DATA HOURLY getting the avg of the hour
        group1 = group.resample('H').mean()
        group1 = group1.fillna(method="ffill")
        #print(zonedfs[name[0]].head(20) )
        #group1 = group.drop('InstanceType', axis=1).drop(['AvailabilityZone'],axis=1)
        group1.rename(columns = {'SpotPrice':name[1]}, inplace = True)    
        #Remove index so that you can merge
        group1 = group1.reset_index()
        #print(group1.head(20))
        zonedfs[name[0]] = zonedfs[name[0]].merge(group1,how='right')


#Fill typedfs with dataframes of all machines in that zone pricing
for name, group in depa:
    if typedfs[name[1]].empty:
        typedfs[name[1]] = group
        typedfs[name[1]] = typedfs[name[1]].resample('H').mean()
        typedfs[name[1]] = typedfs[name[1]].fillna(method="ffill")
        #typedfs[name[1]] = typedfs[name[1]].drop('InstanceType', axis=1).drop(['AvailabilityZone'],axis=1)
        typedfs[name[1]].rename(columns = {'SpotPrice':name[0]}, inplace = True)
        typedfs[name[1]] = typedfs[name[1]].reset_index()
    else:
        group1 = group.resample('H').mean()
        group1 = group1.fillna(method="ffill")
        #group1 = group.drop('InstanceType', axis=1).drop(['AvailabilityZone'],axis=1)
        group1.rename(columns = {'SpotPrice':name[0]}, inplace = True)
        group1 = group1.reset_index()
        typedfs[name[1]] = typedfs[name[1]].merge(group1,how='right')

In [76]:
# Reindex time by timestamp
for key in typedfs:
    typedfs[key].index = typedfs[key].TimeStamp
    typedfs[key]       = typedfs[key].drop(['TimeStamp'],axis=1)

for key in zonedfs:
    zonedfs[key].index = zonedfs[key].TimeStamp
    zonedfs[key]       = zonedfs[key].drop(['TimeStamp'],axis=1)

In [96]:
typedfs['r4.large'].head(3)


Out[96]:
ap-southeast-2b ap-southeast-2c eu-central-1a eu-central-1b eu-west-1a eu-west-1b eu-west-1c us-east-1a us-east-1c us-east-1d us-east-1e us-west-1b us-west-1c us-west-2a us-west-2b us-west-2c
TimeStamp
2016-12-05 18:00:00 0.08 0.08 0.08 0.08 0.078 0.078 0.078 0.063000 0.063000 0.063000 0.063000 0.078 0.078 0.077 0.077 0.077
2016-12-05 19:00:00 0.08 0.08 0.08 0.08 0.078 0.078 0.078 0.077067 0.077067 0.081667 0.077267 0.078 0.078 0.077 0.077 0.077
2016-12-05 20:00:00 0.08 0.08 0.08 0.08 0.078 0.078 0.078 0.077033 0.077033 0.097500 0.077133 0.078 0.078 0.077 0.077 0.077

Correlations of timeseries


In [82]:
#NORMALIZE THE DATA HERE


tN=typedfs.copy()
zN=zonedfs.copy()

for key in tN:
    #typedfs[key].index = typedfs[key].TimeStamp
    #typedfs[key]       = typedfs[key].drop(['TimeStamp'],axis=1)
    #Normalize data
        #typedfs[key] = typedfs[key].apply(lambda row: np.log(row).diff(), axis=0 )
    tN[key] = tN[key].diff(axis=0)
    corrGraph(key, tN[key])
    
for key in zonedfsN:
    #zonedfs[key].index = zonedfs[key].TimeStamp
    #zonedfs[key]       = zonedfs[key].drop(['TimeStamp'],axis=1)
    #Normalize data
        #zonedfs[key] = zonedfs[key].apply(lambda row: np.log(row).diff(), axis=0 )
    zN[key] = zN[key].diff(axis=0)
    corrGraph(key, zN[key])