In [1]:
%matplotlib inline
import sys
import os.path
sys.path.append(os.path.abspath(os.path.join(os.pardir,os.pardir)))
import disaggregator as da
import disaggregator.PecanStreetDatasetAdapter as psda
import pickle
import numpy as np
import pdb
import matplotlib.pyplot as plt
db_url = "postgresql://USERNAME:PASSWORD@db.wiki-energy.org:5432/postgres"
psda.set_url(db_url)
schema = 'shared'
tables = [u'validated_01_2014',
u'validated_02_2014',
u'validated_03_2014',
u'validated_04_2014',
u'validated_05_2014',]
This notebook focuses on creating machine-learning datasets amenable to use for appliance detection. It builds upon the PecanStreetDatasetAdpater
functions and may integrate its results into those functions.
This notebook
In [2]:
# Get dataids
# uncomment the following line to to regenerate this list
# common_use_ids = da.utils.get_common_ids([psda.get_dataids_with_real_values(schema,table,'use') for table in tables])
Description of an ideal dataset:
The ideal dataset consists of as much differentiating information as possible about what it looks like for a device to be present. The dataset should be a big numpy array of data and a big numpy array of labels.
Description of some useful functions
Function that takes an appliance name and a dataid and gets all windows of data in the available months.
In [3]:
train,valid,test = psda.get_appliance_detection_arrays(schema,tables,'car1',672,96,10)
Fetching dataids
Training
Fetching samples for key = 1
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=661
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=661
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=661
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=661
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=661
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=1714
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=1714
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=1714
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1714
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=1714
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=1953
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=1953
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=1953
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=1953
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=1953
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=2470
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=2470
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=2470
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2470
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=2470
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=2638
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=2638
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=2638
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2638
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=2638
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=2769
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=2769
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=2769
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2769
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=2769
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=2814
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=2814
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=2814
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=2814
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=2814
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=3192
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=3192
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=3192
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3192
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=3192
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=3367
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=3367
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=3367
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3367
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=3367
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=3723
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=3723
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=3723
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=3723
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=3723
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=4135
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=4135
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=4135
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4135
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=4135
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=4526
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=4526
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=4526
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=4526
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=4526
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=5109
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=5109
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=5109
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=5109
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=5109
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=6139
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=6139
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=6139
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=6139
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=6139
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=6836
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=6836
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=6836
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=6836
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=6836
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=6910
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=6910
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=6910
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=6910
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=6910
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=7850
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=7850
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=7850
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7850
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=7850
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=7875
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=7875
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=7875
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=7875
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=7875
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=8669
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=8669
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=8669
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=8669
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=8669
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=9932
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=9932
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=9932
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9932
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=9932
select car1,use,localminute from "PecanStreet_SharedData".validated_01_2014 where dataid=9934
select car1,use,localminute from "PecanStreet_SharedData".validated_02_2014 where dataid=9934
select car1,use,localminute from "PecanStreet_SharedData".validated_03_2014 where dataid=9934
select car1,use,localminute from "PecanStreet_SharedData".validated_04_2014 where dataid=9934
select car1,use,localminute from "PecanStreet_SharedData".validated_05_2014 where dataid=9934
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-3-6d74d108f908> in <module>()
----> 1 train,valid,test = psda.get_appliance_detection_arrays(schema,tables,'car1',672,96,10)
/Users/philngo/Documents/Work/DSSG/wikienergy/disaggregator/PecanStreetDatasetAdapter.py in get_appliance_detection_arrays(schema, tables, appliance, window_length, window_stride, drop_percentile, verbose)
575 active_appliance_windows = get_use_for_active_windows(
576 schema,tables,[appliance],appliance_ids,
--> 577 window_length,window_stride,drop_percentile)
578 if verbose:
579 print "Fetching smaples for key = 0"
/Users/philngo/Documents/Work/DSSG/wikienergy/disaggregator/PecanStreetDatasetAdapter.py in get_use_for_active_windows(schema, tables, appliances, dataids, window_length, window_stride, drop_percentile, sample_rate)
512 window_stride)
513 appliance_windows = []
--> 514 if instances.empty():
515 if drop_percentile is not 0:
516 print "Warning: ignoring drop_percentile"
AttributeError: 'list' object has no attribute 'empty'
In [ ]:
Content source: dssg/wikienergy
Similar notebooks: