

In [1]:

    
%matplotlib inline
import sys
import os.path
sys.path.append(os.path.abspath(os.path.join(os.pardir,os.pardir)))
import disaggregator as da
import disaggregator.PecanStreetDatasetAdapter as psda
import pickle
import numpy as np
import pdb
import matplotlib.pyplot as plt

db_url = "postgresql://USERNAME:PASSWORD@db.wiki-energy.org:5432/postgres"
psda.set_url(db_url)

schema = 'shared'
tables = [u'validated_01_2014',
          u'validated_02_2014',
          u'validated_03_2014',
          u'validated_04_2014',
          u'validated_05_2014',]

Dataset Creation

This notebook focuses on creating machine-learning datasets amenable to use for appliance detection. It builds upon the PecanStreetDatasetAdpater functions and may integrate its results into those functions.

Goals

This notebook

develops an interface for direct creation of usable datasets.
focuses on case study of EV detection.
focuses on the shared dataset.



In [2]:

    
# Get dataids

# uncomment the following line to to regenerate this list
# common_use_ids = da.utils.get_common_ids([psda.get_dataids_with_real_values(schema,table,'use') for table in tables])

Description of an ideal dataset:

The ideal dataset consists of as much differentiating information as possible about what it looks like for a device to be present. The dataset should be a big numpy array of data and a big numpy array of labels.

Description of some useful functions

Function that takes an appliance name and a dataid and gets all windows of data in the available months.



In [3]:

    
train,valid,test = psda.get_appliance_detection_arrays(schema,tables,'car1',672,96,10)









    



Fetching dataids
Training
Fetching samples for key = 1
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=661
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=661
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=661
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=661
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=661
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=1714
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=1714
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=1714
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1714
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=1714
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=1953
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=1953
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=1953
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1953
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=1953
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=2470
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=2470
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=2470
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2470
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=2470
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=2638
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=2638
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=2638
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2638
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=2638
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=2769
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=2769
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=2769
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2769
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=2769
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=2814
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=2814
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=2814
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2814
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=2814
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=3192
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=3192
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=3192
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3192
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=3192
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=3367
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=3367
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=3367
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3367
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=3367
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=3723
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=3723
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=3723
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3723
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=3723
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=4135
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=4135
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=4135
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4135
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=4135
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=4526
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=4526
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=4526
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4526
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=4526
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=5109
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=5109
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=5109
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5109
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=5109
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=6139
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=6139
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=6139
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=6139
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=6139
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=6836
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=6836
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=6836
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=6836
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=6836
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=6910
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=6910
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=6910
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=6910
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=6910
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=7850
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=7850
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=7850
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7850
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=7850
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=7875
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=7875
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=7875
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7875
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=7875
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=8669
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=8669
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=8669
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=8669
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=8669
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=9932
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=9932
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=9932
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9932
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=9932
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=9934
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=9934
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=9934
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9934
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=9934






    



---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-3-6d74d108f908> in <module>()
----> 1 train,valid,test = psda.get_appliance_detection_arrays(schema,tables,'car1',672,96,10)

/Users/philngo/Documents/Work/DSSG/wikienergy/disaggregator/PecanStreetDatasetAdapter.py in get_appliance_detection_arrays(schema, tables, appliance, window_length, window_stride, drop_percentile, verbose)
    575         active_appliance_windows = get_use_for_active_windows(
    576                 schema,tables,[appliance],appliance_ids,
--> 577                 window_length,window_stride,drop_percentile)
    578         if verbose:
    579             print "Fetching smaples for key = 0"

/Users/philngo/Documents/Work/DSSG/wikienergy/disaggregator/PecanStreetDatasetAdapter.py in get_use_for_active_windows(schema, tables, appliances, dataids, window_length, window_stride, drop_percentile, sample_rate)
    512                 window_stride)
    513         appliance_windows = []
--> 514         if instances.empty():
    515             if drop_percentile is not 0:
    516                 print "Warning: ignoring drop_percentile"

AttributeError: 'list' object has no attribute 'empty'



In [ ]: