ray-rllib使用实践 -- 1

环境： 

        python: 3.12.8

        ray: 2.40.0

        gymnasium: 1.0.0

用 PPOConfig配置并创建算法，并训练， 保存checkpoint 

from genericpath import exists
from ray.rllib.algorithms.ppo import PPOConfig 
from ray.tune.logger import pretty_print 
import os 

algo = (
    PPOConfig()
    .env_runners(num_env_runners=1) ## 并行或分布式的几个进程
    .resources(num_gpus=0)
    .environment(env="CartPole-v1")
    .build()
)

## 训练并保存 checkpoint 
for i in range(5):
    result = algo.train()
    print(f"episode_{i}")
    # print(pretty_print(result))  ## result中包含了许多信息，可以选择性的提取使用或打印

    if i % 2 == 0:
        checkpoint_dir = f"F:/codes/RLlib_study/outputs/checkpoint_episode_{i}"
        os.makedirs(checkpoint_dir,exist_ok=True)
        
        algo.save_checkpoint(checkpoint_dir)
        print(f"saved checkpoint to {checkpoint_dir}")
        # checkpoint_dir = algo.save().checkpoint.path 
        # print(f"Checkpoint saved in directory {checkpoint_dir}")

 加载预训练的模型继续训练并保存模型

from ray.rllib.algorithms.algorithm import Algorithm 

checkpoint_path = "F:/codes/RLlib_study/outputs/checkpoint_episode_2"
algo = Algorithm.from_checkpoint(checkpoint_path)
print(f"成功加载 checkpoint from {checkpoint_path} ")

## 继续训练并保存 checkpoint 
for i in range(6,10,1):
    result = algo.train()
    print(f"episode_{i}")
    # print(pretty_print(result))
    if i % 2 == 0:
        checkpoint_dir = f"F:/codes/RLlib_study/outputs/checkpoint_episode_{i}"
        os.makedirs(checkpoint_dir,exist_ok=True)
        algo.save_checkpoint(checkpoint_dir)
        print(f"saved checkpoint to {checkpoint_dir}")

参考：RLlib一：RLlib入门-优快云博客

Getting Started with RLlib — Ray 3.0.0.dev0