In [1]:
%matplotlib inline
In [2]:
import pandas as pd
import numpy as np
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense, LeakyReLU, Dropout, ReLU, GRU, TimeDistributed, Conv2D, MaxPooling2D, Flatten
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.callbacks import EarlyStopping
from jlab import load_test_data, get_test_detector_plane
In [3]:
X_train = pd.read_csv('MLchallenge2_training.csv')
X_test = load_test_data('test_in.csv')
eval_planes = get_test_detector_plane(X_test)
# Also, load our truth values
y_true = pd.read_csv('test_prediction.csv', names=['x', 'y', 'px', 'py', 'pz'],
header=None)
In [4]:
X_test.head()
Out[4]:
In [5]:
y_true.head()
Out[5]:
pad_sequences to prepend with zeros appropriately
In [10]:
N_SAMPLES = len(X_train)
N_DETECTORS = 25
N_KINEMATICS = 6
SHAPE = (N_SAMPLES, N_DETECTORS-1, N_KINEMATICS)
In [11]:
X_train_list = []
y_train_array = np.ndarray(shape=(N_SAMPLES, N_KINEMATICS-1))
for ix in range(N_SAMPLES):
seq_len = np.random.choice(range(8, 25))
track = X_train.iloc[ix].values.reshape(N_DETECTORS, N_KINEMATICS)
X_train_list.append(track[0:seq_len])
# Store the kinematics of the next in the sequence
# Ignore the 3rd one, which is z
y_train_array[ix] = track[seq_len][[0,1,3,4,5]]
In [12]:
for track in X_train_list[:10]:
print(len(track))
In [13]:
X_train_list = pad_sequences(X_train_list, dtype=float)
In [14]:
for track in X_train_list[:10]:
print(len(track))
In [15]:
X_train_array = np.array(X_train_list)
X_train_array.shape
Out[15]:
In [16]:
y_train_array.shape
Out[16]:
In [17]:
N_TEST_SAMPLES = len(X_test)
In [18]:
y_test_array = y_true.values
In [19]:
X_test_list = []
for ix in range(N_TEST_SAMPLES):
seq_len = get_test_detector_plane(X_test.iloc[ix])
track = X_test.iloc[ix].values.reshape(N_DETECTORS, N_KINEMATICS)
X_test_list.append(track[0:seq_len])
In [20]:
X_test_list = pad_sequences(X_test_list, dtype=float)
X_test_array = np.array(X_test_list)
In [21]:
X_test_array.shape
Out[21]:
In [22]:
y_test_array.shape
Out[22]:
In [23]:
y_true.values.shape
Out[23]:
In [1]:
import pandas as pd
import numpy as np
from math import floor
from tensorflow.keras.preprocessing.sequence import pad_sequences
from sklearn.model_selection import train_test_split
In [2]:
data = pd.read_csv('MLchallenge2_training.csv')
In [4]:
# Z values are constant -- what are they?
Z_VALS = data[['z'] + [f'z{i}' for i in range(1, 25)]].loc[0].values
# Z-distance from one timestep to another is set; calculate it
Z_DIST = [Z_VALS[i+1] - Z_VALS[i] for i in range(0, 24)] + [0.0]
In [18]:
# Number of timesteps
N_DETECTORS = 25
# Provided number of kinematics
N_KINEMATICS = 6
# Number of features after engineering them all
N_FEATURES = 13
In [4]:
def get_detector_meta(kin_array, det_id):
# Is there a large gap after this detector?
# 0 is for padded timesteps
# 1 is for No, 2 is for Yes
mind_the_gap = int(det_id % 6 == 0) + 1
# Detector group: 1 (origin), 2, 3, 4, or 5
det_grp = floor((det_id-1) / 6) + 2
# Detectors numbered 1-6 (origin is 6)
# (Which one in the group of six is it?)
det_rank = ((det_id-1) % 6) + 1
# Distance to the next detector?
z_dist = Z_DIST[det_id]
# Transverse momentum (x-y component)
pt = np.sqrt(np.square(kin_array[3]) + np.square(kin_array[4]))
# Total momentum
p_tot = np.sqrt(np.square(kin_array[3])
+ np.square(kin_array[4])
+ np.square(kin_array[5]))
# Put all the calculated features together
det_meta = np.array([det_id, mind_the_gap, det_grp, det_rank,
z_dist, pt, p_tot])
# Return detector data plus calculated features
return np.concatenate([kin_array, det_meta], axis=None)
In [5]:
def tracks_to_time_series(X):
"""Convert training dataframe to multivariate time series training set
Pivots each track to a series ot timesteps. Then randomly truncates them
to be identical to the provided test set. The step after the truncated
step is saved as the target.
Truncated sequence are front-padded with zeros.
Parameters
----------
X : pandas.DataFrame
Returns
-------
(numpy.ndarray, numpy.ndarray)
Tuple of the training data and labels
"""
X_ts_list = []
n_samples = len(X)
y_array = np.ndarray(shape=(n_samples, N_KINEMATICS-1))
for ix in range(n_samples):
# Randomly choose how many detectors the track went through
track_len = np.random.choice(range(8, 25))
# Reshape into ts-like
track = X.iloc[ix].values.reshape(N_DETECTORS, N_KINEMATICS)
#eng_track = np.zeros(shape=(N_DETECTORS, N_FEATURES))
#for i in range(0, N_DETECTORS):
# eng_track[i] = get_detector_meta(track[i], i)
# Truncate the track to only N detectors
X_ts_list.append(track[0:track_len])
# Store the kinematics of the next in the sequence
# Ignore the 3rd one, which is z
y_array[ix] = track[track_len][[0,1,3,4,5]]
# Pad the training sequence
X_ts_list = pad_sequences(X_ts_list, dtype=float)
X_ts_array = np.array(X_ts_list)
return X_ts_array, y_array
In [6]:
X, y = tracks_to_time_series(data)
In [7]:
X[3]
Out[7]:
In [8]:
y[3]
Out[8]:
In [9]:
X_train, X_test, y_train, y_test = train_test_split(X, y)
In [10]:
len(X_train), len(X_test)
Out[10]:
In [7]:
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import GRU, Dense, LeakyReLU, Dropout
from tensorflow.keras.callbacks import EarlyStopping
import joblib
In [20]:
def lrelu(x):
return LeakyReLU()(x)
def gru_model(gru_units=35, dense_units=100,
dropout_rate=0.25):
"""Model definition.
Three layers of Gated Recurrent Units (GRUs), utilizing
LeakyReLU activations, finally passing GRU block output
to a dense layer, passing its output to the final output
layer, with a touch of dropout in between.
Bon apetit.
Parameters
----------
gru_units : int
dense_units : int
dropout_rate : float
Returns
-------
tensorflow.keras.models.Sequential
"""
model = Sequential()
model.add(GRU(gru_units, activation=lrelu,
input_shape=(N_DETECTORS-1, N_KINEMATICS),
return_sequences=True))
model.add(GRU(gru_units, activation=lrelu,
return_sequences=True))
model.add(GRU(gru_units, activation=lrelu))
model.add(Dense(dense_units, activation=lrelu))
model.add(Dropout(dropout_rate))
model.add(Dense(N_KINEMATICS-1))
model.compile(loss='mse', optimizer='adam')
return model
In [21]:
model = gru_model()
In [22]:
model.summary()
In [16]:
from tensorflow.keras.utils import plot_model
plot_model(model, to_file='gru_model.png', show_shapes=True)
Out[16]:
In [17]:
es = EarlyStopping(monitor='val_loss', mode='min',
patience=5, restore_best_weights=True)
history = model.fit(
x=X_train,
y=y_train,
validation_data=(X_test, y_test),
callbacks=[es],
epochs=50,
)
model.save("gru_model.h5")
joblib.dump(history.history, "gru_model.history")
In [8]:
history = joblib.load("dannowitz_jlab2_model_20191031.history")
In [11]:
import matplotlib.pyplot as plt
# Plot training & validation loss values
plt.plot(history['loss'])
plt.plot(history['val_loss'])
plt.title('Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper right')
plt.show()
In [42]:
pred = pd.read_csv('data/submission/dannowitz_jlab2_submission_20191112.csv', header=None)
truth = pd.read_csv('data/ANSWERS.csv', header=None)
In [43]:
# Calculate square root of the mean squared error
# Then apply weights and sum them all up
sq_error = (truth - pred).applymap(np.square)
mse = sq_error.sum() / len(truth)
rmse = np.sqrt(mse)
rms_weighted = rmse / [0.03, 0.03, 0.01, 0.01, 0.011]
score = rms_weighted.sum()
score
Out[43]:
You can slice and dice the stats however you want, but it helps to be able to see your predictions at work.
In [161]:
def lstm_model():
model = Sequential()
model.add(LSTM(200, activation=LeakyReLU(), input_shape=(N_DETECTORS-1, N_KINEMATICS)))
model.add(Dense(100, activation=LeakyReLU()))
model.add(Dropout(0.25))
model.add(Dense(N_KINEMATICS-1, activation='linear'))
model.compile(loss='mse', optimizer='adam')
return model
In [150]:
model = lstm_model()
model.summary()
In [143]:
history = model.fit(x=X_train_array, y=y_train_array, validation_data=(X_test_array, y_test_array), epochs=5)
Out[143]:
In [151]:
history = model.fit(x=X_train_array, y=y_train_array,
validation_data=(X_test_array, y_test_array),
epochs=50, use_multiprocessing=True)
In [156]:
model = lstm_model()
es = EarlyStopping(monitor='val_loss', mode='min')
history = model.fit(x=X_train_array, y=y_train_array,
validation_data=(X_test_array, y_test_array),
callbacks=[es], epochs=20, use_multiprocessing=True)
In [157]:
model.save("lstm100-dense100-dropout025-epochs20-early-stopping.h5")
In [162]:
def lstm_model_lin():
model = Sequential()
model.add(LSTM(200, activation=LeakyReLU(), input_shape=(N_DETECTORS-1, N_KINEMATICS)))
model.add(Dense(100, activation=LeakyReLU()))
model.add(Dropout(0.25))
model.add(Dense(N_KINEMATICS-1, activation='linear'))
model.compile(loss='mse', optimizer='adam')
return model
lin_act_model = lstm_model_lin()
es = EarlyStopping(monitor='val_loss', mode='min')
history = lin_act_model.fit(x=X_train_array[:10000], y=y_train_array[:10000],
validation_data=(X_test_array, y_test_array),
callbacks=[es], epochs=20, use_multiprocessing=True)
In [164]:
def lstm_model_adam():
model = Sequential()
model.add(LSTM(200, activation=LeakyReLU(), input_shape=(N_DETECTORS-1, N_KINEMATICS)))
model.add(Dense(100, activation=LeakyReLU()))
model.add(Dropout(0.25))
model.add(Dense(N_KINEMATICS-1))
model.compile(loss='mse', optimizer='adam')
return model
adam_model = lstm_model_adam()
es = EarlyStopping(monitor='val_loss', mode='min')
history = adam_model.fit(x=X_train_array[:10000], y=y_train_array[:10000],
validation_data=(X_test_array, y_test_array),
callbacks=[es], epochs=20, use_multiprocessing=True)
In [166]:
def lstm_model_dropout50():
model = Sequential()
model.add(LSTM(200, activation=LeakyReLU(), input_shape=(N_DETECTORS-1, N_KINEMATICS)))
model.add(Dense(100, activation=LeakyReLU()))
model.add(Dropout(0.50))
model.add(Dense(N_KINEMATICS-1))
model.compile(loss='mse', optimizer='adam')
return model
dropout50_model = lstm_model_dropout50()
es = EarlyStopping(monitor='val_loss', mode='min')
history = dropout50_model.fit(x=X_train_array[:10000], y=y_train_array[:10000],
validation_data=(X_test_array, y_test_array),
callbacks=[es], epochs=20, use_multiprocessing=True)
In [172]:
def lstm_model_nodropout():
model = Sequential()
model.add(LSTM(200, activation=LeakyReLU(), input_shape=(N_DETECTORS-1, N_KINEMATICS)))
model.add(Dense(100, activation=LeakyReLU()))
model.add(Dense(N_KINEMATICS-1))
model.compile(loss='mse', optimizer='adam')
return model
nodropout_model = lstm_model_nodropout()
es = EarlyStopping(monitor='val_loss', mode='min')
history = nodropout_model.fit(x=X_train_array[:10000], y=y_train_array[:10000],
validation_data=(X_test_array, y_test_array),
callbacks=[es], epochs=20, use_multiprocessing=True)
In [167]:
def lstm_model_relu():
model = Sequential()
model.add(LSTM(200, activation='relu', input_shape=(N_DETECTORS-1, N_KINEMATICS)))
model.add(Dense(100, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(N_KINEMATICS-1))
model.compile(loss='mse', optimizer='adam')
return model
relu_model = lstm_model_relu()
es = EarlyStopping(monitor='val_loss', mode='min')
history = relu_model.fit(x=X_train_array[:10000], y=y_train_array[:10000],
validation_data=(X_test_array, y_test_array),
callbacks=[es], epochs=20, use_multiprocessing=True)
In [170]:
def model_gru():
model = Sequential()
model.add(GRU(200, activation=LeakyReLU(), input_shape=(N_DETECTORS-1, N_KINEMATICS)))
model.add(Dense(100, activation=LeakyReLU()))
model.add(Dropout(0.25))
model.add(Dense(N_KINEMATICS-1))
model.compile(loss='mse', optimizer='adam')
return model
gru_model = model_gru()
es = EarlyStopping(monitor='val_loss', mode='min')
history = gru_model.fit(x=X_train_array[:10000], y=y_train_array[:10000],
validation_data=(X_test_array, y_test_array),
callbacks=[es], epochs=20, use_multiprocessing=True)
In [243]:
def model_v2():
model = Sequential()
model.add(GRU(200, activation=LeakyReLU(), input_shape=(N_DETECTORS-1, N_KINEMATICS)))
model.add(Dense(100, activation=LeakyReLU()))
model.add(Dropout(0.25))
model.add(Dense(N_KINEMATICS-1))
model.compile(loss='mse', optimizer='adam')
return model
v2_model = model_v2()
es = EarlyStopping(monitor='val_loss', mode='min')
history = v2_model.fit(x=X_train_array, y=y_train_array,
validation_data=(X_test_array, y_test_array),
callbacks=[es], epochs=8, use_multiprocessing=True)
In [ ]:
from tensorflow.keras.back
In [26]:
def model_v2_deep():
model = Sequential()
model.add(GRU(30, activation=LeakyReLU(), input_shape=(N_DETECTORS-1, N_KINEMATICS),
return_sequences=True))
model.add(GRU(30, activation=LeakyReLU(), return_sequences=True))
model.add(GRU(30, activation=LeakyReLU()))
model.add(Dense(100, activation=LeakyReLU()))
model.add(Dropout(0.25))
model.add(Dense(N_KINEMATICS-1))
model.compile(loss='mse', optimizer='adam')
return model
v2_model_deep = model_v2_deep()
v2_model_deep.summary()
In [ ]:
es = EarlyStopping(monitor='val_loss', mode='min', patience=2, restore_best_weights=True)
history = v2_model_deep.fit(x=X_train_array, y=y_train_array,
validation_data=(X_test_array, y_test_array),
callbacks=[es],
epochs=8, use_multiprocessing=True)
In [242]:
def model_v2_dbl_gru():
model = Sequential()
model.add(GRU(200, activation=LeakyReLU(), input_shape=(N_DETECTORS-1, N_KINEMATICS),
return_sequences=True))
model.add(GRU(200, activation=LeakyReLU(), input_shape=(N_DETECTORS-1, N_KINEMATICS)))
model.add(Dense(100, activation=LeakyReLU()))
model.add(Dropout(0.25))
model.add(Dense(N_KINEMATICS-1))
model.compile(loss='mse', optimizer='adam')
return model
v2_model_dbl_gru = model_v2_dbl_gru()
es = EarlyStopping(monitor='val_loss', mode='min')
history = v2_model_dbl_gru.fit(x=X_train_array[:20000], y=y_train_array[:20000],
validation_data=(X_test_array, y_test_array),
#callbacks=[es],
epochs=10, use_multiprocessing=True)
In [177]:
def model_v2_2x_dropout():
model = Sequential()
model.add(GRU(200, activation=LeakyReLU(), input_shape=(N_DETECTORS-1, N_KINEMATICS)))
model.add(Dropout(0.25))
model.add(Dense(100, activation=LeakyReLU()))
model.add(Dropout(0.25))
model.add(Dense(N_KINEMATICS-1))
model.compile(loss='mse', optimizer='adam')
return model
v2_model_dbl_dropout = model_v2_2x_dropout()
es = EarlyStopping(monitor='val_loss', mode='min')
history = v2_model_dbl_dropout.fit(x=X_train_array[:20000], y=y_train_array[:20000],
validation_data=(X_test_array, y_test_array),
callbacks=[es], epochs=20, use_multiprocessing=True)
In [200]:
def model_v2_big_gru():
model = Sequential()
model.add(GRU(400, activation=LeakyReLU(), input_shape=(N_DETECTORS-1, N_KINEMATICS)))
model.add(Dense(100, activation=LeakyReLU()))
model.add(Dropout(0.25))
model.add(Dense(N_KINEMATICS-1))
model.compile(loss='mse', optimizer='adam')
return model
v2_model_big_gru = model_v2_big_gru()
es = EarlyStopping(monitor='val_loss', mode='min')
history = v2_model_big_gru.fit(x=X_train_array[:20000], y=y_train_array[:20000],
validation_data=(X_test_array, y_test_array),
#callbacks=[es],
epochs=10, use_multiprocessing=True)
In [181]:
v2_model_big_gru.fit(x=X_train_array[:20000], y=y_train_array[:20000],
validation_data=(X_test_array, y_test_array),
#callbacks=[es],
epochs=15, use_multiprocessing=True, initial_epoch=10)
Out[181]:
In [221]:
X_train_array.shape
Out[221]:
In [240]:
def cnn_gru():
model = Sequential()
model.add(Conv1D(filters=5, kernel_size=2, strides=1, input_shape=(N_DETECTORS-1, N_KINEMATICS)))
#model.add(MaxPooling1D())
model.add(GRU(200, activation=LeakyReLU()))
model.add(Dense(100, activation=LeakyReLU()))
model.add(Dropout(0.25))
model.add(Dense(N_KINEMATICS-1))
model.compile(loss='mse', optimizer='adam')
return model
cnn_model = cnn_gru()
cnn_model.summary()
In [241]:
#es = EarlyStopping(monitor='val_loss', mode='min')
history = cnn_model.fit(x=X_train_array[:20000], y=y_train_array[:20000],
validation_data=(X_test_array, y_test_array),
epochs=10, use_multiprocessing=True)
In [251]:
history.history
Out[251]:
In [1]:
from train import train
from predict import predict
In [2]:
model = train(frac=1.00, filename="dannowitz_jlab2_model", epochs=100, ret_model=True)
In [ ]:
preds = predict(model_filename="dannowitz_jlab2_model.h5",
data_filename="test_in (1).csv",
output_filename="danowitz_jlab2_submission.csv")