In [ ]:
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import validation_curve
from sklearn.tree import DecisionTreeRegressor
from sklearn.utils import shuffle
In [ ]:
n_samples = 1000
def f(X, noise=0.4, random_state=0):
x = X[:, 0]
rng = np.random.RandomState(random_state)
return np.sin(2 * np.pi * x) + rng.randn(x.shape[0]) * noise
def make_data(n, random_state=0):
rng = np.random.RandomState(random_state)
n_sin = int(0.9 * n)
n_zeros = int(n - n_sin)
X1 = rng.uniform(0, 1, size=(n_sin, 1))
y1 = f(X1, random_state=random_state)
X2 = rng.uniform(-1, 0, size=(n_zeros, 1))
y2 = np.zeros(n_zeros, dtype=y1.dtype)
return shuffle(np.vstack([X1, X2]), np.concatenate([y1, y2]),
random_state=rng)
X_train, y_train = make_data(200, random_state=0)
X_test, y_test = make_data(10000, random_state=1)
plt.scatter(X_train[:, 0], y_train)
plt.savefig('noisy_sin_dataset.svg')
In [ ]:
param_range = [2, 4, 8, 16, 32, 64]
tree = DecisionTreeRegressor()
train_scores, test_scores = validation_curve(
DecisionTreeRegressor(),
X_train, y_train, param_name="max_leaf_nodes", param_range=param_range,
cv=5, scoring="neg_mean_squared_error", n_jobs=1)
train_scores_mean = -np.mean(train_scores, axis=1)
train_scores_std = np.std(train_scores, axis=1)
test_scores_mean = -np.mean(test_scores, axis=1)
test_scores_std = np.std(test_scores, axis=1)
plt.figure(figsize=(8, 6))
plt.title("Decision Tree")
plt.xlabel("number of nodes")
plt.ylabel("Score")
lw = 2
plt.plot(param_range, train_scores_mean, label="Training score",
color="darkorange", lw=lw)
plt.fill_between(param_range, train_scores_mean - train_scores_std,
train_scores_mean + train_scores_std, alpha=0.2,
color="darkorange", lw=lw)
plt.plot(param_range, test_scores_mean, label="Cross-validation score",
color="navy", lw=lw)
plt.fill_between(param_range, test_scores_mean - test_scores_std,
test_scores_mean + test_scores_std, alpha=0.2,
color="navy", lw=lw)
plt.ylim(0, 0.8)
plt.xlim(param_range[0], param_range[-1])
plt.legend(loc="best");
In [ ]:
for i, max_leaf_nodes in enumerate(param_range):
fig, (left, right) = plt.subplots(ncols=2, figsize=(12, 6))
right.scatter(X_train[:, 0], y_train, alpha=0.3)
left.set_title("Decision Tree")
left.set_xlabel("number of nodes")
left.set_ylabel("Score")
lw = 2
left.plot(param_range[:i + 1], train_scores_mean[:i + 1], label="Training score",
color="darkorange", marker='o', lw=lw)
left.fill_between(param_range[:i + 1], train_scores_mean[:i + 1] - train_scores_std[:i + 1],
train_scores_mean[:i + 1] + train_scores_std[:i + 1], alpha=0.2,
color="darkorange", lw=lw)
left.plot(param_range[:i + 1], test_scores_mean[:i + 1], label="Cross-validation score",
color="navy", marker='o', lw=lw)
left.fill_between(param_range[:i + 1], test_scores_mean[:i + 1] - test_scores_std[:i + 1],
test_scores_mean[:i + 1] + test_scores_std[:i + 1], alpha=0.2,
color="navy", lw=lw)
left.set_ylim(0, 0.8)
left.set_xlim(param_range[0], param_range[-1])
left.legend(loc="best");
tree.max_leaf_nodes = max_leaf_nodes
tree.fit(X_train, y_train)
x = np.linspace(-1.2, 1.2)
right.plot(x, tree.predict(x[:, None]))
right.set_xlabel('x')
right.set_ylabel('y = model(x)')
right.set_title('Model predictions')
right.set_ylim(-2, 2)
fig.savefig(f'noisy_sin_tree_{i:03d}.svg')
plt.close(fig)
In [ ]:
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import Ridge
from sklearn.pipeline import Pipeline
poly = Pipeline([
('poly_features', PolynomialFeatures()),
('ridge', Ridge(alpha=1e-6)),
])
param_range = [0, 5, 10, 15, 20, 25]
train_scores, test_scores = validation_curve(
poly, X_train, y_train, param_name="poly_features__degree",
param_range=param_range, cv=5, scoring="neg_mean_squared_error", n_jobs=8)
train_scores_mean = -np.mean(train_scores, axis=1)
train_scores_std = np.std(train_scores, axis=1)
test_scores_mean = -np.mean(test_scores, axis=1)
test_scores_std = np.std(test_scores, axis=1)
plt.figure(figsize=(8, 6))
plt.title("Polynomial Regression")
plt.xlabel("degree")
plt.ylabel("Score")
lw = 2
plt.plot(param_range, train_scores_mean, label="Training score",
color="darkorange", lw=lw)
plt.fill_between(param_range, train_scores_mean - train_scores_std,
train_scores_mean + train_scores_std, alpha=0.2,
color="darkorange", lw=lw)
plt.plot(param_range, test_scores_mean, label="Cross-validation score",
color="navy", lw=lw)
plt.fill_between(param_range, test_scores_mean - test_scores_std,
test_scores_mean + test_scores_std, alpha=0.2,
color="navy", lw=lw)
plt.ylim(0, 0.8)
plt.xlim(param_range[0], param_range[-1])
plt.legend(loc="best");
In [ ]:
for i, degree in enumerate(param_range):
fig, (left, right) = plt.subplots(ncols=2, figsize=(12, 6))
right.scatter(X_train[:, 0], y_train, alpha=0.3)
left.set_title("Polynomial Regression")
left.set_xlabel("degree")
left.set_ylabel("Score")
lw = 2
left.plot(param_range[:i + 1], train_scores_mean[:i + 1], label="Training score",
color="darkorange", marker='o', lw=lw)
left.fill_between(param_range[:i + 1], train_scores_mean[:i + 1] - train_scores_std[:i + 1],
train_scores_mean[:i + 1] + train_scores_std[:i + 1], alpha=0.2,
color="darkorange", lw=lw)
left.plot(param_range[:i + 1], test_scores_mean[:i + 1], label="Cross-validation score",
color="navy", marker='o', lw=lw)
left.fill_between(param_range[:i + 1], test_scores_mean[:i + 1] - test_scores_std[:i + 1],
test_scores_mean[:i + 1] + test_scores_std[:i + 1], alpha=0.2,
color="navy", lw=lw)
left.set_ylim(0, 0.8)
left.set_xlim(param_range[0], param_range[-1])
left.legend(loc="best");
poly.set_params(poly_features__degree=degree)
poly.fit(X_train, y_train)
x = np.linspace(-1.2, 1.2)
right.plot(x, poly.predict(x[:, None]))
right.set_xlabel('x')
right.set_ylabel('y = model(x)')
right.set_title('Model predictions')
right.set_ylim(-2, 2)
fig.savefig(f'noisy_sin_poly_{i:03d}.svg')
plt.close(fig)
In [ ]:
from sklearn.neural_network import MLPRegressor
# mlp = MLPRegressor(solver='lbfgs', max_iter=5000, alpha=0)
mlp = MLPRegressor(solver='adam', learning_rate_init=0.1, learning_rate='adaptive',
alpha=0, max_iter=5000, n_iter_no_change=100, random_state=0)
param_range = [16, 32, 64, 128, 256, 1024]
train_scores, test_scores = validation_curve(
mlp, X_train, y_train, param_name="hidden_layer_sizes",
param_range=param_range, cv=5, scoring="neg_mean_squared_error", n_jobs=2)
train_scores_mean = -np.mean(train_scores, axis=1)
train_scores_std = np.std(train_scores, axis=1)
test_scores_mean = -np.mean(test_scores, axis=1)
test_scores_std = np.std(test_scores, axis=1)
In [ ]:
plt.figure(figsize=(8, 6))
plt.title("1 hidden layers MLP (no regularization)")
plt.xlabel("width")
plt.ylabel("Score")
lw = 2
plt.plot(param_range, train_scores_mean, label="Training score",
color="darkorange", lw=lw)
plt.fill_between(param_range, train_scores_mean - train_scores_std,
train_scores_mean + train_scores_std, alpha=0.2,
color="darkorange", lw=lw)
plt.plot(param_range, test_scores_mean, label="Cross-validation score",
color="navy", lw=lw)
plt.fill_between(param_range, test_scores_mean - test_scores_std,
test_scores_mean + test_scores_std, alpha=0.2,
color="navy", lw=lw)
plt.ylim(0, 0.8)
plt.xlim(param_range[0], 256)
plt.legend(loc="best");
In [ ]:
for i, h in enumerate(param_range[:-1]):
fig, (left, right) = plt.subplots(ncols=2, figsize=(12, 6))
right.scatter(X_train[:, 0], y_train, alpha=0.3)
left.set_title("Single hidden layer MLP, no regularization")
left.set_xlabel("hidden units")
left.set_ylabel("Score")
lw = 2
left.plot(param_range[:i + 1], train_scores_mean[:i + 1], label="Training score",
color="darkorange", marker='o', lw=lw)
left.fill_between(param_range[:i + 1], train_scores_mean[:i + 1] - train_scores_std[:i + 1],
train_scores_mean[:i + 1] + train_scores_std[:i + 1], alpha=0.2,
color="darkorange", lw=lw)
left.plot(param_range[:i + 1], test_scores_mean[:i + 1], label="Cross-validation score",
color="navy", marker='o', lw=lw)
left.fill_between(param_range[:i + 1], test_scores_mean[:i + 1] - test_scores_std[:i + 1],
test_scores_mean[:i + 1] + test_scores_std[:i + 1], alpha=0.2,
color="navy", lw=lw)
left.set_ylim(0, 0.8)
left.set_xlim(param_range[0], param_range[-2])
left.legend(loc="best");
mlp.set_params(hidden_layer_sizes=[h])
mlp.fit(X_train, y_train)
x = np.linspace(-1.2, 1.2)
right.plot(x, mlp.predict(x[:, None]))
right.set_xlabel('x')
right.set_ylabel('y = model(x)')
right.set_title('Model predictions')
right.set_ylim(-2, 2)
fig.savefig(f'noisy_sin_mlp_{i:03d}.svg')
plt.close(fig)
In [ ]:
plt.figure(figsize=(16, 6))
plt.title("1 hidden layers MLP (no regularization)")
plt.xlabel("width")
plt.ylabel("Score")
lw = 2
plt.plot(param_range, train_scores_mean, label="Training score",
color="darkorange", lw=lw)
plt.fill_between(param_range, train_scores_mean - train_scores_std,
train_scores_mean + train_scores_std, alpha=0.2,
color="darkorange", lw=lw)
plt.plot(param_range, test_scores_mean, label="Cross-validation score",
color="navy", lw=lw)
plt.fill_between(param_range, test_scores_mean - test_scores_std,
test_scores_mean + test_scores_std, alpha=0.2,
color="navy", lw=lw)
plt.ylim(0, 0.8)
plt.xlim(param_range[0], param_range[-1])
plt.legend(loc="best");
plt.savefig(f'noisy_sin_mlp_full_learning_curve.svg')
In [ ]: