Using the sklearn hidden markov models to disaggregate energy signals

In [1]:

    

import sys
sys.path.append('../../') # or non-Unix equivalent (add wikienergy/ to path)
from disaggregator import PecanStreetDatasetAdapter as psda
from disaggregator import fhmm
from disaggregator import utils
from disaggregator import appliance
from collections import OrderedDict
import itertools
import numpy as np
from hmmlearn import hmm
import matplotlib.pyplot as plt
from disaggregator import evaluation_metrics as metric
import pickle
%matplotlib inline
import pylab
from pylab import plot, show, title, xlabel, ylabel, subplot
pylab.rcParams['figure.figsize'] = 12, 6


    

    




/usr/local/lib/python2.7/dist-packages/pandas/io/excel.py:626: UserWarning: Installed openpyxl is not supported at this time. Use >=1.6.1 and <2.0.0.
  .format(openpyxl_compat.start_ver, openpyxl_compat.stop_ver))

In [2]:

    

#Load Datasets
devices_types={}
devices_types_unsampled={}
db_url='postgresql://USERNAME:PASSWORD@db.wiki-energy.org:5432/postgres'
psda.set_url(db_url)
schema = 'shared'
tables= psda.get_table_names(schema)
print tables


    

    




[u'validated_01_2014', u'validated_02_2014', u'validated_04_2014', u'validated_05_2014', u'validated_03_2014']

In [5]:

    

#Gets id's associated with a device and generates types for each ids
ids_for_devices={}
table=tables[3]
ids_device_name='air1'
ids_for_devices[ids_device_name]=psda.get_dataids_with_real_values(schema,table,ids_device_name)


    

In [6]:

    

num_houses=1000


    

In [7]:

    

device_name='air1'
devices_types_unsampled[device_name]=psda.generate_type_for_appliance_by_dataids(schema,table,device_name,ids_for_devices[ids_device_name][:num_houses])


    

    




select air1,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=26
select air1,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=59

In [204]:

    

device_name='car1'
devices_types_unsampled[device_name]=psda.generate_type_for_appliance_by_dataids(schema,table,device_name,ids_for_devices[ids_device_name][:num_houses])


    

    




select car1,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=26
select car1,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=59

In [10]:

    

device_name='use'
devices_types_unsampled[device_name]=psda.generate_type_for_appliance_by_dataids(schema,table,device_name,ids_for_devices[ids_device_name][:num_houses])


    

    




select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=26
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=59
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=86
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=93
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=94
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=280
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=410
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=434
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=484
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=499
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=580
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=624
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=661
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=739
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=744
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=774
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=821
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=871
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=936
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1086
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1167
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1283
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1334
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1450
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1507
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1617
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1632
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1681
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1696
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1714
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1718
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1782
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1790
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1830
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1953
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1994
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2034
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2094
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2129
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2156
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2158
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2171
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2242
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2365
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2378
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2470
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2575
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2606
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2638
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2641
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2769
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2787
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2814
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2829
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2845
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2864
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2945
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2953
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2974
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3044
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3092
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3134
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3192
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3221
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3263
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3367
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3394
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3456
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3482
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3504
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3531
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3649
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3652
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3723
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3736
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3778
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3795
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3893
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4135
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4154
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4298
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4313
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4352
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4373
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4505
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4526
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4641
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4767
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4874
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4922
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4956
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4957
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4998
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5026
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5109
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5209
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5218
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5275
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5357
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5395
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5545
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5568
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5677
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5785
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5814
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5852
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5874
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5889
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5938
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5949
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5972
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=6139
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=6165
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=6412
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=6636
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=6673
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=6730
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=6826
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=6836
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=6910
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=6941
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7062
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7319
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7390
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7531
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7536
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7617
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7731
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7769
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7788
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7800
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7850
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7863
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7875
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7940
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7951
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=8046
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=8061
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=8079
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=8084
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=8142
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=8188
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=8197
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=8201
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=8292
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=8342
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=8419
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=8645
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=8669
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=8741
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=8956
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9019
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9036
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9121
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9141
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9160
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9343
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9356
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9484
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9488
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9499
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9555
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9578
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9609
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9613
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9643
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9654
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9701
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9729
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9737
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9771
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9830
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9875
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9915
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9922
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9926
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9932
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9934
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9938
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9939
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9982
select use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9983

In [8]:

    

#Resamples the data
sample_rate='1T'
length='D'
devices_types_unsplit={}
for key in devices_types_unsampled:
    devices_types_unsplit[key]=devices_types_unsampled[key].resample(sample_rate)
    devices_types[key]=devices_types_unsplit[key].split_by(length)
    #devices_types[key]=devices_types_unsplit[key]
    print key


    

    




air1

In [28]:

    

#Initialize Variables for Model
devices_models={}
devices_data={}
devices_type_names=[]
pi_prior={}
a_prior={}
mean_prior={}
cov_prior={}
best_model=OrderedDict()


    

In [25]:

    

## 1 Feature
device_type_name='air1'
key_for_model_name='dataid'
pi_prior[device_type_name]=np.array([0.5,0.5])
a_prior[device_type_name]=np.array([[0.98,0.02],[0.02,0.98]])
mean_prior[device_type_name]=np.array([[0],[200]])
cov_prior[device_type_name]=np.tile(1, (2, 1, 1))
devices_models[device_type_name]=fhmm.generate_HMMs_from_type(devices_types[device_type_name],pi_prior[device_type_name],a_prior[device_type_name],mean_prior[device_type_name],cov_prior[device_type_name],key_for_model_name)


    

In [ ]:

    

best_model[device_type_name]=fhmm.get_best_instance_model(devices_models[device_type_name],devices_types[device_type_name],key_for_model_name)


    

In [29]:

    

with open('../../data/air/4/air1_4 15T.pkl','rb') as f:
    devices_models=pickle.load(f)
best_model[device_type_name]=fhmm.get_best_instance_model(devices_models[device_type_name],devices_types[device_type_name],key_for_model_name)


    

    




---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-29-7fd294fde453> in <module>()
      1 with open('../../data/air/4/air1_4 15T.pkl','rb') as f:
      2     devices_models=pickle.load(f)
----> 3 best_model[device_type_name]=fhmm.get_best_instance_model(devices_models[device_type_name],devices_types[device_type_name],key_for_model_name)

KeyError: 'air1'

In [329]:

    

#Create FHMM
type_models={}
for device_type_name in devices_models:
    type_models[device_type_name]=devices_models[device_type_name][best_model[device_type_name]]
model_fhmm,means_fhmm=fhmm.generate_FHMM_from_HMMs(type_models)


    

    




[(3.4856369036770385e-09,), (0.024116635683468335,), (3.410641404865159,)]
3

In [336]:

    

#Generate Test Data
test_data={}
house_id=1
trace_num=0
for device_type_name in devices_models:
        test_data[device_type_name]=utils.trace_series_to_numpy_array(devices_types[device_type_name].instances[house_id].traces[trace_num].series)
power_total=utils.trace_series_to_numpy_array(devices_types['use'].instances[house_id].traces[trace_num].series)
power_total_minus_bottom=[]
for i in power_total:
    power_total_minus_bottom.append(i-power_total.min())


    

In [311]:

    

#Predict and Plot FHMM Results
plt.plot(power_total_minus_bottom,label='total')
plt.title('Aggregated Energy without constant power')
plt.ylabel('Energy (Wh)')
plt.xlabel('Time')
for i,device_type in enumerate(type_models):
    plt.figure(1)
    plt.plot(test_data[device_type],label=device_type)
    plt.legend(bbox_to_anchor=(0., 1.05, 1, .102), loc=3,
       ncol=2, mode="expand", borderaxespad=1.)
    plt.figure()
[decoded_states, decoded_power]=fhmm.predict_with_FHMM(model_fhmm,means_fhmm,test_data,power_total_minus_bottom)


    

    













    






<matplotlib.figure.Figure at 0x7f9b8e2c9b50>






    






<matplotlib.figure.Figure at 0x7f9b8de50550>






    













    

In [312]:

    

#Evaluate
for device_name in devices_types:
    if device_name is not 'use':
        truth_states=metric.guess_truth_from_power(test_data[device_name],.0001)
        for i,num in enumerate(test_data[device_name]):
            if decoded_states[device_name][i]>1:
                decoded_states[device_name][i]=1
        eval_metrics=metric.get_positive_negative_stats(truth_states,decoded_states[device_name])
        print device_name
        print 'precision:' + str(metric.get_precision(eval_metrics['tp'],eval_metrics['fp'])*100)+'%'
        print 'recall:' + str(metric.get_sensitivity(eval_metrics['tp'],eval_metrics['fn'])*100)+'%'
        print 'diff in power:' + str(metric.sum_error(test_data[device_name],decoded_power[device_name]))+'Wh'
        print


    

    




air1
precision:70.0636942675%
recall:22.7743271222%
diff in power:760.096959772Wh

car1
precision:100.0%
recall:37.5694444444%
diff in power:1326.82833156Wh