In [132]:
#Power data classification/regression with CNN
import numpy as np
import tensorflow as tf
import pandas as pd
import time as time
import matplotlib.pyplot as plt
import pydot
import csv as csv
%matplotlib inline
print("TensorFlow version:",tf.__version__)
In [133]:
#Read total rows in csv file without loading into memory
def data_set_size(csv_file):
with open(csv_file) as csvfile:
csv_rows = 0
for _ in csvfile:
csv_rows += 1
print("Numer of rows in csv:",csv_rows)
return csv_rows-1 #Remove header from count and return
csv_file = "./MISO_power_data_classification_labels.csv"
n_train = data_set_size(csv_file)
print("Training data set size:",n_train)
In [134]:
#Python generator to supply batches of traning data during training with loading full data set to memory
def power_data_generator(batch_size=10,gen_type='training'):
valid_size = max(1,np.int(0.2*batch_size))
#print(valid_size)
while 1:
df_input=pd.read_csv('./MISO_power_data_input.csv',usecols =['Wind_MWh','Actual_Load_MWh'],chunksize =24*(batch_size+valid_size), iterator=True)
df_target=pd.read_csv('./MISO_power_data_classification_labels.csv',usecols =['LowPower','MedPower','HighPower','LowVar','MedVar','HighVar','LowWind','HighWind'],chunksize =batch_size+valid_size, iterator=True)
for chunk, chunk2 in zip(df_input,df_target):
InputX = chunk.as_matrix()
InputX = np.resize(InputX,(batch_size+valid_size,24,2,1))
InputX.astype('float32', copy=False)
InputY = chunk2.as_matrix()
InputY = np.resize(InputY,(batch_size+valid_size,8))
InputY.astype('float32', copy=False)
if gen_type =='training':
yield (InputX[0:batch_size],InputY[0:batch_size])
elif gen_type =='validation':
yield (InputX[batch_size:batch_size+valid_size],InputY[batch_size:batch_size+valid_size])
#yield (InputX,InputY)
elif gen_type =='inference':
yield InputX
In [135]:
train_generator= power_data_generator(batch_size=2,gen_type='training')
valid_generator= power_data_generator(batch_size=2,gen_type='validation')
#next(train_generator)
In [137]:
#Define model using Keras
Yclasses = 8 #Number of output classes
#bias0 = tf.contrib.keras.initializers.glorot_uniform()
#bias0 = tf.contrib.keras.initializers.RandomUniform()
bias0 = 'zeros'
#datagen= tf.contrib.keras.preprocessing.image.ImageDataGImageDataGenerator()
max_power = 100000.0 #For normalizing
model = tf.contrib.keras.models.Sequential()
model.add(tf.contrib.keras.layers.InputLayer(input_shape=(24,2,1),name='InputLayer'))
model.add(tf.contrib.keras.layers.Lambda(lambda x:x/max_power,name='Normalizing'))
model.add(tf.contrib.keras.layers.Conv2D(filters=4,kernel_size=(6,2),strides=(2,2),activation='relu',bias_initializer=bias0,
name='ConvLayer1'))
model.add(tf.contrib.keras.layers.Conv2D(filters=6,kernel_size=(6,1),strides=(2,2),activation='relu',bias_initializer=bias0,
name='ConvLayer2'))
model.add(tf.contrib.keras.layers.Flatten(name='Flatten'))
model.add(tf.contrib.keras.layers.Dense(units=8,bias_initializer=bias0,name='FeedForward1'))
model.add(tf.contrib.keras.layers.Activation('relu',name='ReLU1'))
model.add(tf.contrib.keras.layers.Dense(units=3,bias_initializer=bias0,name='FeedForward2'))
model.add(tf.contrib.keras.layers.Activation('relu',name='ReLU2'))
model.add(tf.contrib.keras.layers.Dense(units=Yclasses,activation='sigmoid',bias_initializer=bias0,name='OutputLayer'))
model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['binary_accuracy'])
model.summary()
log_folder ="./log/"
summary = tf.contrib.keras.callbacks.TensorBoard(log_dir=log_folder,histogram_freq =1,write_graph=True)
In [131]:
#tf.contrib.keras.utils.plot_model(model=model)
samples_per_batch = 5
train_generator= power_data_generator(batch_size=samples_per_batch,gen_type='training')
valid_generator= power_data_generator(batch_size=samples_per_batch,gen_type='validation')
number_of_batches = np.int32(n_train/(samples_per_batch+max(1,np.int32(0.2*samples_per_batch))))
#Training starts
t = time.time()
model.fit_generator(train_generator, steps_per_epoch= number_of_batches,epochs=10,validation_data=valid_generator, validation_steps=number_of_batches)
print("Time: %.3f minutes" % ((time.time() - t)/60))
model.save('model.h5')
print("Model Saved")
In [127]:
inference_generator= power_data_generator(batch_size=1,gen_type='inference') #Generator for getting only input data
In [130]:
test_input = next(inference_generator) #Get next batch of input data from data set
model.predict_on_batch(test_input) #Probability predictions for giving input
Out[130]: