Convert data to NILMTK format and load into NILMTK



In [2]:

    
!! pip install -U Pillow









    Out[2]:





['Collecting Pillow',
 '  Downloading https://files.pythonhosted.org/packages/4e/d9/468422371e6fcf02d6a162ee30db4552221de8b2b3ff837363bf54cbb347/Pillow-6.1.0-cp36-cp36m-win_amd64.whl (2.0MB)',
 'Installing collected packages: Pillow',
 '  Found existing installation: Pillow 6.0.0',
 '    Uninstalling Pillow-6.0.0:',
 '      Successfully uninstalled Pillow-6.0.0',
 'Successfully installed Pillow-6.1.0']

NILMTK uses an open file format based on the HDF5 binary file format to store both the power data and the metadata. The very first step when using NILMTK is to convert your dataset to the NILMTK HDF5 file format.

NOTE: If you are on Windows, remember to escape the back-slashes, use forward-slashs, or use raw-strings when passing paths in Python, e.g. one of the following would work:

convert_redd('c:\\data\\REDD\\low_freq', r'c:\\data\\redd.h5')
convert_redd('c:/data/REDD/low_freq', 'c:/data/redd.h5')
convert_redd(r'c:\data\REDD\low_freq', r'c:\data\redd.h5')

REDD

Converting the REDD dataset is easy:



In [3]:

    
from nilmtk.dataset_converters import convert_redd
convert_redd('../datasets/REDD/low_freq', '../datasets/REDD/low_freq.h5')









    



C:\Users\diego\anaconda3\envs\doutorado\lib\site-packages\PIL\Image.py:116: RuntimeWarning: The _imaging extension was built for another version of Pillow or PIL:
Core version: 6.1.0
Pillow version: 6.0.0
  elif "Symbol not found: _PyUnicodeUCS2_" in str(v):






    



---------------------------------------------------------------------------
ImportError                               Traceback (most recent call last)
<ipython-input-3-9ec0312a8704> in <module>
----> 1 from nilmtk.dataset_converters import convert_redd
      2 convert_redd('../datasets/REDD/low_freq', '../datasets/REDD/low_freq.h5')

~\anaconda3\envs\doutorado\lib\site-packages\nilmtk\__init__.py in <module>
      9 from nilmtk.version import version as __version__
     10 from nilmtk.timeframe import TimeFrame
---> 11 from nilmtk.elecmeter import ElecMeter
     12 from nilmtk.datastore import DataStore, HDFDataStore, CSVDataStore, Key
     13 from nilmtk.metergroup import MeterGroup

~\anaconda3\envs\doutorado\lib\site-packages\nilmtk\elecmeter.py in <module>
      6 import pandas as pd
      7 from six import iteritems
----> 8 from .preprocessing import Clip
      9 from .stats import TotalEnergy, GoodSections, DropoutRate
     10 from .hashable import Hashable

~\anaconda3\envs\doutorado\lib\site-packages\nilmtk\preprocessing\__init__.py in <module>
----> 1 from .clip import Clip
      2 from .apply import Apply

~\anaconda3\envs\doutorado\lib\site-packages\nilmtk\preprocessing\clip.py in <module>
      2 from warnings import warn
      3 from ..node import Node
----> 4 from ..utils import index_of_column_name
      5 
      6 class Clip(Node):

~\anaconda3\envs\doutorado\lib\site-packages\nilmtk\utils.py in <module>
     11 import datetime
     12 import pytz
---> 13 from nilmtk.datastore import HDFDataStore, CSVDataStore
     14 import warnings
     15 

~\anaconda3\envs\doutorado\lib\site-packages\nilmtk\datastore\__init__.py in <module>
      1 from .datastore import DataStore, MAX_MEM_ALLOWANCE_IN_BYTES
----> 2 from .hdfdatastore import HDFDataStore
      3 from .csvdatastore import CSVDataStore
      4 from .key import Key

~\anaconda3\envs\doutorado\lib\site-packages\nilmtk\datastore\hdfdatastore.py in <module>
      5 from os.path import isfile
      6 from nilmtk.timeframe import TimeFrame
----> 7 from nilmtk.timeframegroup import TimeFrameGroup
      8 from .datastore import DataStore, MAX_MEM_ALLOWANCE_IN_BYTES
      9 from nilmtk.docinherit import doc_inherit

~\anaconda3\envs\doutorado\lib\site-packages\nilmtk\timeframegroup.py in <module>
      1 from __future__ import print_function, division
----> 2 import matplotlib.pyplot as plt
      3 import pandas as pd
      4 from datetime import timedelta
      5 

~\anaconda3\envs\doutorado\lib\site-packages\matplotlib\pyplot.py in <module>
   2353     dict.__setitem__(rcParams, "backend", rcsetup._auto_backend_sentinel)
   2354 # Set up the backend.
-> 2355 switch_backend(rcParams["backend"])
   2356 
   2357 # Just to be safe.  Interactive mode can be turned on without

~\anaconda3\envs\doutorado\lib\site-packages\matplotlib\pyplot.py in switch_backend(newbackend)
    219         else "matplotlib.backends.backend_{}".format(newbackend.lower()))
    220 
--> 221     backend_mod = importlib.import_module(backend_name)
    222     Backend = type(
    223         "Backend", (matplotlib.backends._Backend,), vars(backend_mod))

~\anaconda3\envs\doutorado\lib\importlib\__init__.py in import_module(name, package)
    124                 break
    125             level += 1
--> 126     return _bootstrap._gcd_import(name[level:], package, level)
    127 
    128 

~\anaconda3\envs\doutorado\lib\site-packages\ipykernel\pylab\backend_inline.py in <module>
      7 
      8 import matplotlib
----> 9 from matplotlib.backends.backend_agg import new_figure_manager, FigureCanvasAgg # analysis: ignore
     10 from matplotlib import colors
     11 from matplotlib._pylab_helpers import Gcf

~\anaconda3\envs\doutorado\lib\site-packages\matplotlib\backends\backend_agg.py in <module>
     43 
     44 if _has_pil:
---> 45     from PIL import Image
     46 
     47 backend_version = 'v2.2'

~\anaconda3\envs\doutorado\lib\site-packages\PIL\Image.py in <module>
     99             "The _imaging extension was built for another version of Pillow or PIL:\n"
    100             "Core version: %s\n"
--> 101             "Pillow version: %s" % (getattr(core, "PILLOW_VERSION", None), __version__)
    102         )
    103 

ImportError: The _imaging extension was built for another version of Pillow or PIL:
Core version: 6.1.0
Pillow version: 6.0.0

Now redd.h5 holds all the REDD power data and all the relevant metadata. In NILMTK v0.2 this conversion only uses a tiny fraction of the system memory (unlike NILMTK v0.1 which would guzzle ~1 GByte of RAM just to do the dataset conversion!).

Of course, if you want to run convert_redd on your own machine then you first need to download REDD, decompress it and pass the relevant source_directory and output_filename to convert_redd().

Other datasets

At the time of writing, NILMTK contains converters for 8 datasets.

Contributing a new converter is easy and highly encouraged! Learn how to write a dataset converter.

Open HDF5 in NILMTK



In [2]:

    
from nilmtk import DataSet
from nilmtk.utils import print_dict

redd = DataSet('/data/redd.h5')

At this point, all the metadata has been loaded into memory but none of the power data has been loaded. This is our first encounter with a fundamental difference between NILMTK v0.1 and v0.2+: NILMTK v0.1 used to eagerly load the entire dataset into memory before you did any actual work on the data. NILMTK v0.2+ is lazy! It won't load data into memory until you tell it what you want to do with the data (and, even then, large dataset will be loaded in chunks that fit into memory). This allows NILMTK v0.2+ to work with arbitrarily large datasets (datasets too large to fit into memory) without choking your system.

Exploring the `DataSet` object

Let's have a quick poke around to see what's in this redd object...

There is a lot of metadata associated with the dataset, including information about the two models of meter device the authors used to record REDD:



In [3]:

    
print_dict(redd.metadata)









    




name: REDD
long_name: The Reference Energy Disaggregation Data set
creators: Kolter, Zico
Johnson, Matthew
publication_date: 2011
institution: Massachusetts Institute of Technology (MIT)
contact: zkolter@cs.cmu.edu
description: Several weeks of power data for 6 different homes.
subject: Disaggregated power demand from domestic buildings.
number_of_buildings: 6
timezone: US/Eastern
geo_location: locality: Massachusetts
country: US
latitude: 42.360091
longitude: -71.09416
related_documents: http://redd.csail.mit.edu
J. Zico Kolter and Matthew J. Johnson. REDD: A public data set for energy disaggregation research. In proceedings of the SustKDD workshop on Data Mining Applications in Sustainability, 2011. http://redd.csail.mit.edu/kolter-kddsust11.pdf
schema: https://github.com/nilmtk/nilm_metadata/tree/v0.2
meter_devices: eMonitor: model: eMonitor
manufacturer: Powerhouse Dynamics
manufacturer_url: http://powerhousedynamics.com
description: Measures circuit-level power demand.  Comes with 24 CTs. This FAQ page suggests the eMonitor measures real (active) power: http://www.energycircle.com/node/14103  although the REDD readme.txt says all channels record apparent power.
sample_period: 3
max_sample_period: 50
measurements: {'physical_quantity': 'power', 'type': 'active', 'upper_limit': 5000, 'lower_limit': 0}
wireless: False
REDD_whole_house: description: REDD's DIY power meter used to measure whole-home AC waveforms at high frequency.  To quote from their paper: "CTs from TED (http://www.theenergydetective.com) to measure current in the power mains, a Pico TA041 oscilloscope probe (http://www.picotechnologies.com) to measure voltage for one of the two phases in the home, and a National Instruments NI-9239 analog to digital converter to transform both these analog signals to digital readings. This A/D converter has 24 bit resolution with noise of approximately 70 µV, which determines the noise level of our current and voltage readings: the TED CTs are rated for 200 amp circuits and a maximum of 3 volts, so we are able to differentiate between currents of approximately ((200))(70 × 10−6)/(3) = 4.66mA, corresponding to power changes of about 0.5 watts. Similarly, since we use a 1:100 voltage stepdown in the oscilloscope probe, we can detect voltage differences of about 7mV."
sample_period: 1
max_sample_period: 30
measurements: {'physical_quantity': 'power', 'type': 'apparent', 'upper_limit': 50000, 'lower_limit': 0}
wireless: False

We also have all the buildings available as an OrderedDict (indexed from 1 not 0 because every dataset we are aware of starts numbering buildings from 1 not 0)



In [4]:

    
print_dict(redd.buildings)









    




1: Building(instance=1, dataset='REDD')
2: Building(instance=2, dataset='REDD')
3: Building(instance=3, dataset='REDD')
4: Building(instance=4, dataset='REDD')
5: Building(instance=5, dataset='REDD')
6: Building(instance=6, dataset='REDD')

Each building has a little bit of metadata associated with it (there isn't much building-specific metadata in REDD):



In [5]:

    
print_dict(redd.buildings[1].metadata)









    




instance: 1
original_name: house_1
dataset: REDD

Each building has an elec attribute which is a MeterGroup object (much more about those soon!)



In [6]:

    
redd.buildings[1].elec









    Out[6]:





MeterGroup(meters=
  ElecMeter(instance=1, building=1, dataset='REDD', site_meter, appliances=[])
  ElecMeter(instance=2, building=1, dataset='REDD', site_meter, appliances=[])
  ElecMeter(instance=5, building=1, dataset='REDD', appliances=[Appliance(type='fridge', instance=1)])
  ElecMeter(instance=6, building=1, dataset='REDD', appliances=[Appliance(type='dish washer', instance=1)])
  ElecMeter(instance=7, building=1, dataset='REDD', appliances=[Appliance(type='sockets', instance=1)])
  ElecMeter(instance=8, building=1, dataset='REDD', appliances=[Appliance(type='sockets', instance=2)])
  ElecMeter(instance=9, building=1, dataset='REDD', appliances=[Appliance(type='light', instance=1)])
  ElecMeter(instance=11, building=1, dataset='REDD', appliances=[Appliance(type='microwave', instance=1)])
  ElecMeter(instance=12, building=1, dataset='REDD', appliances=[Appliance(type='unknown', instance=1)])
  ElecMeter(instance=13, building=1, dataset='REDD', appliances=[Appliance(type='electric space heater', instance=1)])
  ElecMeter(instance=14, building=1, dataset='REDD', appliances=[Appliance(type='electric stove', instance=1)])
  ElecMeter(instance=15, building=1, dataset='REDD', appliances=[Appliance(type='sockets', instance=3)])
  ElecMeter(instance=16, building=1, dataset='REDD', appliances=[Appliance(type='sockets', instance=4)])
  ElecMeter(instance=17, building=1, dataset='REDD', appliances=[Appliance(type='light', instance=2)])
  ElecMeter(instance=18, building=1, dataset='REDD', appliances=[Appliance(type='light', instance=3)])
  ElecMeter(instance=19, building=1, dataset='REDD', appliances=[Appliance(type='unknown', instance=2)])
  MeterGroup(meters=
    ElecMeter(instance=3, building=1, dataset='REDD', appliances=[Appliance(type='electric oven', instance=1)])
    ElecMeter(instance=4, building=1, dataset='REDD', appliances=[Appliance(type='electric oven', instance=1)])
  )
  MeterGroup(meters=
    ElecMeter(instance=10, building=1, dataset='REDD', appliances=[Appliance(type='washer dryer', instance=1)])
    ElecMeter(instance=20, building=1, dataset='REDD', appliances=[Appliance(type='washer dryer', instance=1)])
  )
)

Yup, that's where all the meat lies!

Convert data to NILMTK format and load into NILMTK

REDD

Other datasets

Open HDF5 in NILMTK

Exploring the DataSet object

Exploring the `DataSet` object