import matplotlib.pyplot as plt
import numpy as np
from sklearn import ensemble,datasets,model_selection
def load_data_regression():
diabetes = datasets.load_diabetes()
return model_selection.train_test_split(diabetes.data,diabetes.target,test_size=0.3,random_state = 0)
def test_GradientBoostingRegression(*data):
X_train,X_test,Y_train,Y_test = data
regr = ensemble.GradientBoostingRegressor()
regr.fit(X_train,Y_train)
print("Training score:%f"%regr.score(X_train,Y_train))
print("Testing score:%f"%regr.score(X_test,Y_test))
X_train,X_test,Y_train,Y_test = load_data_regression()
test_GradientBoostingRegression(X_train,X_test,Y_train,Y_test)
Training score:0.876188
Testing score:0.275905
考察个体回归树的数量对GBRT的影响
def test_GradientBoostingRegression_estimators_num(*data):
X_train,X_test,Y_train,Y_test = data
nums = np.arange(1,100,step=2)
fig = plt.figure()
ax = fig.add_subplot(1,1,1)
train_scores = []
test_scores = []
for i,num in enumerate(nums):
clf = ensemble.GradientBoostingRegressor(n_estimators=num)
clf.fit(X_train,Y_train)
train_scores.append(clf.score(X_train,Y_train))
test_scores.append(clf.score(X_test,Y_test))
ax.plot(nums,train_scores,label="training score")
ax.plot(nums,test_scores,label="testing score")
ax.set_xlabel("estimators num")
ax.set_ylabel("score")
ax.legend(loc="lower right")
ax.set_title("GradientBoostRegression")
plt.show()
X_train,X_test,Y_train,Y_test = load_data_regression()
test_GradientBoostingRegression_estimators_num(X_train,X_test,Y_train,Y_test)
output_6_0.png
运行结果分析：
- 随着个体回归树数量的增长，GBRT的性能对与训练集一直在提高。
- 但是对于测试集测试得分先快速上升后基本缓慢下降。
考察个体回归树的最大深度对于集成回归器预测性能的影响
def test_GradientBoostingRegression_max_depth(*data):
X_train,X_test,Y_train,Y_test = data
maxdepths = np.arange(1,20+1)
fig = plt.figure()
ax = fig.add_subplot(1,1,1)
train_scores = []
test_scores = []
for i,maxdepth in enumerate(maxdepths):
clf = ensemble.GradientBoostingRegressor(max_depth=maxdepth,max_leaf_nodes=None)
clf.fit(X_train,Y_train)
train_scores.append(clf.score(X_train,Y_train))
test_scores.append(clf.score(X_test,Y_test))
ax.plot(maxdepths,train_scores,label="Training score")
ax.plot(maxdepths,test_scores,label="Testing score")
ax.set_xlabel("maxdepth num")
ax.set_ylabel("score")
ax.legend(loc="lower right")
ax.set_ylim(-1,1.05)
ax.set_title("GradientBoostingRegression")
plt.show()
X_train,X_test,Y_train,Y_test = load_data_regression()
test_GradientBoostingRegression_max_depth(X_train,X_test,Y_train,Y_test)
output_10_0.png
运行结果分析：
- 随着个体回归树的最大深度增长，对训练集的拟合越来越好，但是对测试集的拟合越来越差，产生了过拟合现象。
考察学习率对于GBRT的预测性能的影响
def test_GradientBoostingRegression_learning_rate(*data):
X_train,X_test,Y_train,Y_test = data
learing_rates = np.linspace(0.01,1.0)
fig = plt.figure()
ax = fig.add_subplot(1,1,1)
train_scores = []
test_scores = []
for i,learing_rate in enumerate(learing_rates):
clf = ensemble.GradientBoostingRegressor(learning_rate=learing_rate)
clf.fit(X_train,Y_train)
train_scores.append(clf.score(X_train,Y_train))
test_scores.append(clf.score(X_test,Y_test))
ax.plot(learing_rates,train_scores,label="Training score")
ax.plot(learing_rates,test_scores,label="Testing score")
ax.set_xlabel("learing_rate")
ax.set_ylabel("score")
ax.legend(loc="best")
ax.set_ylim(-1,1.05)
ax.set_title("GradientBoostingRegression_learning_rate")
plt.show()
X_train,X_test,Y_train,Y_test = load_data_regression()
test_GradientBoostingRegression_learning_rate(X_train,X_test,Y_train,Y_test)
output_14_0.png
运行结果分析：
- GBRT对预测集的预测得分随着学习率的增长而一直降低
考察subsample的影响
def test_GradientBoostingRegression_subsample(*data):
X_train,X_test,Y_train,Y_test = data
subsamples = np.linspace(0.01,1.0)
fig = plt.figure()
ax = fig.add_subplot(1,1,1)
train_scores = []
test_scores = []
for i,subsample in enumerate(subsamples):
clf = ensemble.GradientBoostingRegressor(subsample=subsample,n_estimators=100)
clf.fit(X_train,Y_train)
train_scores.append(clf.score(X_train,Y_train))
test_scores.append(clf.score(X_test,Y_test))
ax.plot(subsamples,train_scores,label="Training score")
ax.plot(subsamples,test_scores,label="Testing score")
ax.set_xlabel("subsample")
ax.set_ylabel("score")
ax.legend(loc="best")
ax.set_ylim(-1,1.05)
ax.set_title("GradientBoostingRegression_subsample")
plt.show()
X_train,X_test,Y_train,Y_test = load_data_regression()
test_GradientBoostingRegression_subsample(X_train,X_test,Y_train,Y_test)
output_18_0.png
运行结果分析：
- 本问题中，subsample对GBRT预测的影响不大，主要对GBRT的训练拟合能力起作用。
考察max_features参数的影响
def test_GradientBoostingRegression_max_features(*data):
X_train,X_test,Y_train,Y_test = data
max_features = np.linspace(0.01,1.0)
fig = plt.figure()
ax = fig.add_subplot(1,1,1)
train_scores = []
test_scores = []
for i,features in enumerate(max_features):
clf = ensemble.GradientBoostingRegressor(max_features=features,n_estimators=100)
clf.fit(X_train,Y_train)
train_scores.append(clf.score(X_train,Y_train))
test_scores.append(clf.score(X_test,Y_test))
ax.plot(max_features,train_scores,label="Training score")
ax.plot(max_features,test_scores,label="Testing score")
ax.set_xlabel("max_features")
ax.set_ylabel("score")
ax.legend(loc="best")
ax.set_ylim(0,1.05)
ax.set_title("GradientBoostingRegression_max_features")
plt.show()
X_train,X_test,Y_train,Y_test = load_data_regression()
test_GradientBoostingRegression_max_features(X_train,X_test,Y_train,Y_test)
output_22_0.png
运行结果分析：
- GBRT对于特征集合的选取不是很敏感
考察损失函数的影响
def test_GradientBoostingRegression_loss(*data):
X_train,X_test,Y_train,Y_test = data
fig = plt.figure(figsize=(15,15))
nums = np.arange(1,200,step=2)
###ls:平方损失函数
###lad:绝对值损失函数
###huber:上述两者的加权结合
losses = ["ls","lad","huber"]
ax = fig.add_subplot(2,1,1)
alphas = np.linspace(0.01,1.0,endpoint=False,num=5)
for alpha in alphas:
test_scores = []
train_scores = []
for num in nums:
regr = ensemble.GradientBoostingRegressor(n_estimators=num,loss="huber",alpha=alpha)
regr.fit(X_train,Y_train)
train_scores.append(regr.score(X_train,Y_train))
test_scores.append(regr.score(X_test,Y_test))
ax.plot(nums,train_scores,label="Training score:alpha=%f"%alpha)
ax.plot(nums,test_scores,label="Testing score:alpha=%f"%alpha)
ax.set_xlabel("estimator num")
ax.set_ylabel("score")
ax.legend(loc="best",framealpha=0.4)
ax.set_ylim(0,1.05)
ax.set_title("loss=huber")
plt.suptitle("GradientBoostRegressor")
######绘制ls和lad#######
ax = fig.add_subplot(2,1,2)
for loss in ["ls","lad"]:
test_scores = []
train_scores = []
for num in nums:
regr = ensemble.GradientBoostingRegressor(n_estimators=num,loss=loss)
regr.fit(X_train,Y_train)
train_scores.append(regr.score(X_train,Y_train))
test_scores.append(regr.score(X_test,Y_test))
ax.plot(nums,train_scores,label="Training score:loss=%s"%loss)
ax.plot(nums,test_scores,label="Testing score:loss=%s"%loss)
ax.set_xlabel("estimator num")
ax.set_ylabel("score")
ax.legend(loc="best",framealpha=0.4)
ax.set_ylim(0,1.05)
ax.set_title("loss=ls,lad")
plt.suptitle("GradientBoostRegressor")
plt.show()
output.png
