【Python】贝叶斯超参数优化

bayes_opt

https://www.bilibili.com/video/BV1aa41167oS

import numpy as np
from sklearn.ensemble import RandomForestRegressor as RFR
from sklearn.model_selection import KFold, cross_validate
# 基于bayes_opt库实现贝叶斯优化
from bayes_opt import BayesianOptimization

1. 定义目标函数

'''
1. 目标函数的输入必须是具体的超参数，不能是超参数空间
2. 目标函数的超参数输入值只能是浮点数，不支持整数和字符串。所以字符串型的超参数不能直接传。
3. bayes_opt只支持寻找目标函数最大值.RMSE越小越好,-RMSE越大越好.若是准确率、auc等则保持原样.
'''
def bayesopt_objective(n_estimators,max_depth,max_features,min_impurity_decrease):
    # 定义评估器
    # 调整超参数输入
    reg = RFR(n_estimators=int(n_estimators)
             ,max_depth=int(max_depth)
             ,max_features=int(max_features)
             ,min_impurity_decrease=min_impurity_decrease
             ,random_state=1412
             ,verbose=False #自己定义
             ,n_jobs=-1
             )
    # 定义损失输出,5折交叉验证的结果,输出-RMSE
    cv = KFold(n_splits=5, shuffle=True, random_state=1412)
    validation_loss = cross_validate(reg
                                    ,X,y
                                    ,scoring='neg_root_mean_squared_error'
                                    ,cv=cv
                                    ,verbose=False
                                    ,n_jobs=-1
                                    ,error_score='raise' # 默认nan.如果出错,输出原因
                                    )
    # 最终优化拟合的是交叉验证的值
    return np.mean(validation_loss['test_score'])

2. 定义参数空间

'''
双向闭区间(两边都能取到)
只有上下界,没有步长,自动取浮点数
因此bayes_opt参数空间取值更密,迭代次数更多
'''
param_grid_simple = {'n_estimators':(80,100)
                    ,'max_depth':(10,25)
                    ,'max_features':(10,20)
                    ,'min_impurity_decrease':(0,1)
                    }

3. 优化目标函数的具体流程

'''
涉及随机性,如随机抽取点作为观测点、随机抽取观测点进行采集函数计算
优化算法无法被复现,每次运行必然不一样
最佳超参数组合可以复现,记住优化算法实例化对象的最佳参数组合和最佳分数,交叉验证分割已定义随机种子
'''
def param_bayes_opt(init_points, n_iter):
    # 定义优化器
    opt = BayesianOptimization(bayesopt_objective # 待优化的目标函数
                              ,param_grid_simple # 参数空间
                              ,random_state=1412 # 随机数种子,虽然无法控制
                              )
    # 使用优化器,只支持最大化
    opt.maximize(init_points=init_points # 抽取多少个初始观测值
                 ,n_iter=n_iter # 一共观测/迭代多少次
                 )
    # 优化完成,取出最佳分数和最佳分数
    param_best = opt.max['params']
    score_best = opt.max['target']
    print('\n\n','best_params: ',param_best,
         '\n\n','best_cvscore: ',score_best)
    
    return param_best, score_best

4. 定义验证函数(非必须)

'''
目标函数中设置了随机数种子,交叉验证是规定的,
贝叶斯优化给出的最佳分数一定与与验证的分数相同
'''
def bayes_opt_validation(params_best):
        reg = RFR(n_estimators=int(n_estimators)
             ,max_depth=int(max_depth)
             ,max_features=int(max_features)
             ,min_impurity_decrease=min_impurity_decrease
             ,random_state=1412
             ,verbose=False
             ,n_jobs=-1
             )
        cv = KFold(n_splits=5, shuffle=True, random_state=1412)
        validation_loss = cross_validate(reg
                                        ,X,y
                                        ,scoring='neg_root_mean_squared_error'
                                        ,cv=cv
                                        ,verbose=False
                                        ,n_jobs=-1
                                        ,error_score='raise'
                                        )
        return np.mean(validation_loss['test_score'])

from sklearn.datasets import load_digits

data = load_digits()
X = data.data
y = data.target
print(X.shape,y.shape)

(1797, 64) (1797,)

5. 调用

import time
start = time.time()
params_best, score_best = param_bayes_opt(20,280) # 20初始观测点,迭代280次
print('It takes %s minutes'%((time.time()-start)/60))
validation_score = bayes_opt_validation(params_best)
print('\n validation_score: ',validation_score)

    |   iter    |  target   | max_depth | max_fe... | min_im... | n_esti... |
    -------------------------------------------------------------------------
    | [0m 1       [0m | [0m-1.357   [0m | [0m 23.2    [0m | [0m 17.52   [0m | [0m 0.06379 [0m | [0m 88.79   [0m |
    | [0m 2       [0m | [0m-2.735   [0m | [0m 14.8    [0m | [0m 17.61   [0m | [0m 0.9214  [0m | [0m 97.58   [0m |
    | [0m 3       [0m | [0m-1.941   [0m | [0m 15.86   [0m | [0m 15.56   [0m | [0m 0.2661  [0m | [0m 87.98   [0m |
    | [0m 4       [0m | [0m-1.384   [0m | [0m 14.05   [0m | [0m 16.84   [0m | [0m 0.06744 [0m | [0m 89.72   [0m |
    | [0m 5       [0m | [0m-2.726   [0m | [0m 18.71   [0m | [0m 19.17   [0m | [0m 0.9315  [0m | [0m 83.7    [0m |
    | [0m 6       [0m | [0m-2.483   [0m | [0m 17.7    [0m | [0m 19.58   [0m | [0m 0.7127  [0m | [0m 89.18   [0m |
    | [0m 7       [0m | [0m-2.154   [0m | [0m 14.21   [0m | [0m 12.62   [0m | [0m 0.3381  [0m | [0m 91.51   [0m |
    | [0m 8       [0m | [0m-2.559   [0m | [0m 23.23   [0m | [0m 10.89   [0m | [0m 0.6078  [0m | [0m 95.06   [0m |
    | [0m 9       [0m | [0m-2.776   [0m | [0m 14.89   [0m | [0m 14.0    [0m | [0m 0.9487  [0m | [0m 80.16   [0m |
    | [95m 10      [0m | [95m-1.258   [0m | [95m 11.52   [0m | [95m 12.58   [0m | [95m 0.03276 [0m | [95m 92.56   [0m |
    | [0m 11      [0m | [0m-1.99    [0m | [0m 13.14   [0m | [0m 13.31   [0m | [0m 0.2563  [0m | [0m 98.24   [0m |
    | [0m 12      [0m | [0m-2.263   [0m | [0m 17.94   [0m | [0m 11.48   [0m | [0m 0.3778  [0m | [0m 82.09   [0m |
    | [0m 13      [0m | [0m-2.594   [0m | [0m 16.02   [0m | [0m 17.03   [0m | [0m 0.7735  [0m | [0m 88.31   [0m |
    | [0m 14      [0m | [0m-2.345   [0m | [0m 13.92   [0m | [0m 15.04   [0m | [0m 0.529   [0m | [0m 93.66   [0m |
    | [0m 15      [0m | [0m-2.318   [0m | [0m 12.51   [0m | [0m 13.69   [0m | [0m 0.4482  [0m | [0m 99.9    [0m |
    | [0m 16      [0m | [0m-2.371   [0m | [0m 17.73   [0m | [0m 10.05   [0m | [0m 0.4143  [0m | [0m 82.79   [0m |
    | [0m 17      [0m | [0m-2.785   [0m | [0m 16.6    [0m | [0m 10.84   [0m | [0m 0.9134  [0m | [0m 88.37   [0m |
    | [0m 18      [0m | [0m-2.671   [0m | [0m 21.92   [0m | [0m 15.0    [0m | [0m 0.8219  [0m | [0m 85.86   [0m |
    | [0m 19      [0m | [0m-1.369   [0m | [0m 14.07   [0m | [0m 11.38   [0m | [0m 0.05068 [0m | [0m 91.53   [0m |
    | [0m 20      [0m | [0m-2.581   [0m | [0m 10.35   [0m | [0m 17.38   [0m | [0m 0.7624  [0m | [0m 99.19   [0m |
    


    ---------------------------------------------------------------------------

    StopIteration                             Traceback (most recent call last)

    File E:\Anaconda3\envs\DL\lib\site-packages\bayes_opt\bayesian_optimization.py:179, in BayesianOptimization.maximize(self, init_points, n_iter, acq, kappa, kappa_decay, kappa_decay_delay, xi, **gp_params)
        178 try:
    --> 179     x_probe = next(self._queue)
        180 except StopIteration:
    

    File E:\Anaconda3\envs\DL\lib\site-packages\bayes_opt\bayesian_optimization.py:25, in Queue.__next__(self)
         24 if self.empty:
    ---> 25     raise StopIteration("Queue is empty, no more objects to retrieve.")
         26 obj = self._queue[0]
    

    StopIteration: Queue is empty, no more objects to retrieve.

    
    During handling of the above exception, another exception occurred:
    

    TypeError                                 Traceback (most recent call last)

    Input In [53], in <cell line: 3>()
          1 import time
          2 start = time.time()
    ----> 3 params_best, score_best = param_bayes_opt(20,280) # 20初始观测点,迭代280次
          4 print('It takes %s minutes'%((time.time()-start)/60))
          5 validation_score = bayes_opt_validation(params_best)
    

    Input In [50], in param_bayes_opt(init_points, n_iter)
          9 opt = BayesianOptimization(bayesopt_objective # 待优化的目标函数
         10                           ,param_grid_simple # 参数空间
         11                           ,random_state=1412 # 随机数种子,虽然无法控制
         12                           )
         13 # 使用优化器,只支持最大化
    ---> 14 opt.maximize(init_points=init_points # 抽取多少个初始观测值
         15              ,n_iter=n_iter # 一共观测/迭代多少次
         16              )
         17 # 优化完成,取出最佳分数和最佳分数
         18 param_best = opt.max['params']
    

    File E:\Anaconda3\envs\DL\lib\site-packages\bayes_opt\bayesian_optimization.py:182, in BayesianOptimization.maximize(self, init_points, n_iter, acq, kappa, kappa_decay, kappa_decay_delay, xi, **gp_params)
        180 except StopIteration:
        181     util.update_params()
    --> 182     x_probe = self.suggest(util)
        183     iteration += 1
        185 self.probe(x_probe, lazy=False)
    

    File E:\Anaconda3\envs\DL\lib\site-packages\bayes_opt\bayesian_optimization.py:131, in BayesianOptimization.suggest(self, utility_function)
        128     self._gp.fit(self._space.params, self._space.target)
        130 # Finding argmax of the acquisition function.
    --> 131 suggestion = acq_max(
        132     ac=utility_function.utility,
        133     gp=self._gp,
        134     y_max=self._space.target.max(),
        135     bounds=self._space.bounds,
        136     random_state=self._random_state
        137 )
        139 return self._space.array_to_params(suggestion)
    

    File E:\Anaconda3\envs\DL\lib\site-packages\bayes_opt\util.py:65, in acq_max(ac, gp, y_max, bounds, random_state, n_warmup, n_iter)
         62     continue
         64 # Store it if better than previous minimum(maximum).
    ---> 65 if max_acq is None or -res.fun[0] >= max_acq:
         66     x_max = res.x
         67     max_acq = -res.fun[0]
    

    TypeError: 'float' object is not subscriptable

hyperopt

https://blog.youkuaiyun.com/u012735708/article/details/84820101

import numpy as np

from sklearn.datasets import load_digits
from sklearn.ensemble import RandomForestRegressor as RFR
from sklearn.ensemble import GradientBoostingRegressor as GBR
from sklearn.model_selection import cross_validate, KFold

# 导入优化算法
import hyperopt
from hyperopt import hp, fmin, tpe, Trials, partial
from hyperopt.early_stop import no_progress_loss

data = load_digits()
X = data.data
y = data.target
print(X.shape,y.shape)

    (1797, 64) (1797,)

1. 建立benchmark

在当前环境、数据集下各种算法得到的结果判断优化是否有效

2. 定义参数init需要的算法

benchmark中效果最好的算法

rf = RFR(n_estimators=89, max_depth=22, 
         max_features=14, min_impurity_decrease=0, 
         random_state=1412, verbose=False, n_jobs=-1)

3. 目标函数

# 目标函数
def hyper_objective(params):
    reg = GBR(n_estimators=int(params['n_estimators']) # int()保证整数值
             ,learning_rate=params['lr']
             ,criterion=params['criterion']
             ,loss=params['loss']
             ,max_depth=int(params['max_depth'])
             ,max_features=params['max_features']
             ,subsample=params['subsample']
             ,min_impurity_decrease=params['min_impurity_decrease']
             ,init=rf
             ,random_state=1412 #GBR中只能控制特征抽样,不控制样本抽样
             ,verbose=False # 打印进程
             )
    cv = KFold(n_splits=5, shuffle=True, random_state=1412)
    validation_loss = cross_validate(reg
                                    ,X,y
                                    # 负跟均方误差 绝对值越小越好
                                    ,scoring='neg_root_mean_squared_error'
                                    ,cv=cv
                                    ,verbose=False
                                    ,n_jobs=-1
                                    ,error_score='raise'
                                    )
    # fmin寻找目标函数最小值,所以评估指标越小越好
    return np.mean(abs(validation_loss['test_score']))

4. 参数空间

param_grid_simple = {'n_estimators': hp.quniform('n_estimators',25,200,25)
                    ,'lr': hp.quniform('learning_rate',0.05,2.05,0.05)
                    ,'criterion': hp.choice('criterion',['friedman_mse','squared_error','mse','mae'])
                    ,'loss': hp.choice('loss',['squared_error','absolute_error','huber','quantile'])
                    ,'max_depth': hp.quniform('max_depth',2,30,2)
                    ,'subsample': hp.quniform('sub_sample',0.1,0.8,0.1)
                    ,'max_features': hp.choice('max_features',['log2','sqrt',16,32,64,'auto'])
                    ,'min_impurity_decrease': hp.quniform('min_impurity_decrease',0,5,1)
                    }

## 计算参数空间大小
len(range(25,200,25))*len(np.arange(0.05,2.05,0.05))*4*4*len(range(2,30,2))*len(np.arange(0.1,0.8,0.1))*6*len(range(0,5,1))

    13171200
    

5. 优化函数(优化目标函数的具体流程)

def param_hyperopt(max_evals=100):
    # 保存迭代过程
    trials = Trials()
    # 设置提前停止
    # 如果连续100次迭代损失函数值没有下降则停止
    early_stop_fn = no_progress_loss(100)
    # 定义代理模型
    params_best = fmin(hyper_objective
                     ,space=param_grid_simple
                     ,algo=tpe.suggest # 指定搜索算法
                     ,max_evals=max_evals # 最大评估次数
                     ,verbose=True
                     ,trials=trials
                     ,early_stop_fn=early_stop_fn
                     )
    # 打印最优参数,fmin会自动打印最佳分数
    print('\n','\n','best params:',params_best,
         '\n')
    return params_best, trials
    

6. 验证函数(可选)

def hyperopt_validation(params):
    reg = GBR(n_estimators=int(params['n_estimators']) # int()保证整数值
             ,learning_rate=params['lr']
             ,criterion=params['criterion']
             ,loss=params['loss']
             ,max_depth=int(params['max_depth'])
             ,max_features=params['max_features']
             ,subsample=params['subsample']
             ,min_impurity_decrease=params['min_impurity_decrease']
             ,init=rf
             ,random_state=1412 #GBR中只能控制特征抽样,不控制样本抽样
             ,verbose=False # 打印进程
             )
    cv = KFold(n_splits=5, shuffle=True, random_state=1412)
    validation_loss = cross_validate(reg
                                    ,X,y
                                    # 负跟均方误差 绝对值越小越好
                                    ,scoring='neg_root_mean_squared_error'
                                    ,cv=cv
                                    ,verbose=False
                                    ,n_jobs=-1
                                    ,error_score='raise'
                                    )
    return np.mean(abs(validation_loss['test_score']))

7. 训练贝叶斯优化器

params_best, trials = param_hyperopt(30)
'''
    100%|████████████████████████████████████████████████| 30/30 [02:22<00:00,  4.76s/trial, best loss: 0.9290570901285291]
    
     
     best params: {'criterion': 1, 'learning_rate': 0.4, 'loss': 2, 'max_depth': 6.0, 'max_features': 3, 'min_impurity_decrease': 0.0, 'n_estimators': 175.0, 'sub_sample': 0.6000000000000001} 
'''
# 多搜索几次，对于常选中的参数可以在下次搜索中确定取值(修改目标函数)
# 对于参数范围可进一步缩小(按上一次搜索取值向两边扩)

params_best
'''
    {'criterion': 1,
     'learning_rate': 0.4,
     'loss': 2,
     'max_depth': 6.0,
     'max_features': 3,
     'min_impurity_decrease': 0.0,
     'n_estimators': 175.0,
     'sub_sample': 0.6000000000000001}
'''

# Trials用来记录每次eval的时候，具体使用了什么参数以及相关的返回值
print('trials: ',trials.trials[:1])
'''
 trials:  [
 {'state': 2, 'tid': 0, 'spec': None, 
 'result': {'loss': 1.0419885867093803, 'status': 'ok'}, 
 'misc': {'tid': 0, 'cmd': ('domain_attachment', 'FMinIter_Domain'), 'workdir': None, 'idxs': {'criterion': [0], 'learning_rate': [0], 'loss': [0], 'max_depth': [0], 'max_features': [0], 'min_impurity_decrease': [0], 'n_estimators': [0], 'sub_sample': [0]}, 
 'vals': {'criterion': [2], 'learning_rate': [1.9000000000000001], 'loss': [1], 'max_depth': [10.0], 'max_features': [3], 'min_impurity_decrease': [3.0], 'n_estimators': [125.0], 'sub_sample': [0.5]}
 }, 
 'exp_key': None, 
 'owner': None, 
 'version': 0, 
 'book_time': datetime.datetime(2022, 8, 26, 0, 45, 22, 349000), 
 'refresh_time': datetime.datetime(2022, 8, 26, 0, 45, 25, 226000)
 } ]
'''
    

print('每次迭代loss和status:',trials.results[:2])
print('每次迭代loss:',trials.losses()[:2])
print('每次迭代status:',trials.statuses()[:2])
'''
    每次迭代loss和status: [{'loss': 1.0419885867093803, 'status': 'ok'}, {'loss': 0.9290570901285291, 'status': 'ok'}]
    每次迭代loss: [1.0419885867093803, 0.9290570901285291]
    每次迭代status: ['ok', 'ok']
'''