import argparse
import dataclasses
import json
import os
import random
import time
import warn
ings
from typ
ing import Any, Optional, Union
import
torch,
torch_npu
import uvloop
from tqdm import tqdm
from transformers import Au
toModelForCausalLM, Au
toTokenizer, PreTra
ined
TokenizerBase, BitsAndBytesConfig
# 从本地模块导入数据集和工具函数
from benchmark_dataset import (
AIMODataset,
BurstGPTDataset,
ConversationDataset,
InstructCoderDataset,
RandomDataset,
SampleRequest,
ShareGPTDataset,
SonnetDataset,
VisionArenaDataset,
)
from benchmark_utils import convert_
to_py
torch_benchmark_
format, write_
to_json
from vllm.eng
ine.arg_utils import AsyncEng
ineArgs, Eng
ineArgs
from vllm.entrypo
ints.openai.api_server import (
build_async_eng
ine_client_from_eng
ine_args,
)
from vllm.
inputs import TextPrompt,
TokensPrompt
from vllm.lora.request import LoRARequest
from vllm.
outputs import Reques
tOutput
from vllm.sampl
ing_params import BeamSearchParams
from vllm.utils import FlexibleArgumentParser, merge_async_itera
tors
# 禁用分词器并行
os.environ["
TOKENIZERS_PARALLELISM"] = "false"
bnb_config = BitsAndBytesConfig(
load_
in_4bit=True,
bnb_4bit_compute_dtype=
torch.bfloat16, # Support
torch.float16,
torch.float32,
torch.bfloat16
bnb_4bit_quant_type="nf4", # # Only support `nf4`
bnb_4bit_use_double_quant=False
)
def run_hf(
requests: list[SampleRequest],
model: str,
tokenizer: PreTra
ined
TokenizerBase,
n:
int,
max_batch_size:
int,
trust_remote_code: bool,
disable_de
tokenize: bool = False,
) -> float:
device =
torch.device("npu:0" if
torch.npu.is_available() else "cpu")
torch.npu.set_device(device)
llm = Au
toModelForCausalLM.from_pretra
ined(
model,
torch_dtype=
torch.bfloat16,
# quantization_config=bnb_config,
use_safetensors=True,
trust_remote_code=True
).
to(device)
torch.npu.synchronize()
if llm.config.model_type == "llama":
#
To enable padd
ing
in the HF backend.
tokenizer.pad_
token =
tokenizer.eos_
token
pbar = tqdm(
total=len(
requests))
start = time.perf_counter()
batch: list[str] = []
max_prompt_len = 0
max_
output_len = 0
for i
in range(len(
requests)):
prompt =
requests[i].prompt
prompt_len =
requests[i].prompt_len
output_len =
requests[i].expected_
output_len
# Add the prompt
to the batch.
batch.append(prompt)
max_prompt_len = max(max_prompt_len, prompt_len)
max_
output_len = max(max_
output_len,
output_len)
if len(batch) < max_batch_size and i != len(
requests) - 1:
# Check if we can add more
requests to the batch.
next_prompt_len =
requests[i + 1].prompt_len
next_
output_len =
requests[i + 1].expected_
output_len
if (
max(max_prompt_len, next_prompt_len)
+ max(max_
output_len, next_
output_len)
) <= 2048:
# We can add more
requests to the batch.
cont
inue
#
inputs =
tokenizer(batch, return_tensors="pt", padd
ing=True)
inputs =
tokenizer(batch, return_tensors="pt", padd
ing=True)
input_ids =
inputs.
input_ids.npu()
attention_mask =
inputs.attention_mask.npu()
torch.npu.synchronize()
llm_
outputs = llm.generate(
input_ids=
input_ids,
attention_mask=attention_mask,
max_new_
tokens=32,
do_sample=False,
)
if not disable_de
tokenize:
#
Include the decod
ing time.
tokenizer.batch_decode(llm_
outputs, skip_special_
tokens=True)
pbar.update(len(batch))
# Clear the batch.
batch = []
max_prompt_len = 0
max_
output_len = 0
end = time.perf_counter()
return end - start
def save_
to_py
torch_benchmark_
format(
args: argparse.Namespace, results: dict[str, Any]
) -> None:
pt_records = convert_
to_py
torch_benchmark_
format(
args=args,
metrics={
"
requests_per_second": [results["
requests_per_second"]],
"
tokens_per_second": [results["
tokens_per_second"]],
},
extra_
info={
k: results[k] for k
in ["elapsed_time", "num_
requests", "
total_num_
tokens"]
},
)
if pt_records:
# Don't use json suffix here as we don't want CI
to pick it up
pt_
file = f"{os.path.splitext(args.
output_json)[0]}.py
torch.json"
write_
to_json(pt_
file, pt_records)
def get_
requests(args,
tokenizer):
# Common parameters for all dataset types.
common_kwargs = {
"dataset_path": args.dataset_path,
"random_seed": args.seed,
}
sample_kwargs = {
"
tokenizer":
tokenizer,
"lora_path": args.lora_path,
"max_loras": args.max_loras,
"num_
requests": args.num_prompts,
"
input_len": args.
input_len,
"
output_len": args.
output_len,
}
if args.dataset_path is None or args.dataset_name == "random":
sample_kwargs["range_ratio"] = args.random_range_ratio
sample_kwargs["prefix_len"] = args.prefix_len
dataset_cls = RandomDataset
elif args.dataset_name == "sharegpt":
dataset_cls = ShareGPTDataset
if args.backend == "vllm-chat":
sample_kwargs["enable_multimodal_chat"] = True
elif args.dataset_name == "sonnet":
assert
tokenizer.chat_template or
tokenizer.default_chat_template, (
"
Tokenizer/model must have chat template for sonnet dataset."
)
dataset_cls = SonnetDataset
sample_kwargs["prefix_len"] = args.prefix_len
sample_kwargs["return_prompt_
formatted"] = True
elif args.dataset_name == "burstgpt":
dataset_cls = BurstGPTDataset
elif args.dataset_name == "hf":
common_kwargs["no_stream"] = args.no_stream
if args.dataset_path
in VisionArenaDataset.SUPPORTED_DATASET_PATHS:
dataset_cls = VisionArenaDataset
common_kwargs["dataset_subset"] = None
common_kwargs["dataset_split"] = "tra
in"
sample_kwargs["enable_multimodal_chat"] = True
elif args.dataset_path
in InstructCoderDataset.SUPPORTED_DATASET_PATHS:
dataset_cls =
InstructCoderDataset
common_kwargs["dataset_split"] = "tra
in"
elif args.dataset_path
in ConversationDataset.SUPPORTED_DATASET_PATHS:
dataset_cls = ConversationDataset
common_kwargs["dataset_subset"] = args.hf_subset
common_kwargs["dataset_split"] = args.hf_split
sample_kwargs["enable_multimodal_chat"] = True
elif args.dataset_path
in AIMODataset.SUPPORTED_DATASET_PATHS:
dataset_cls = AIMODataset
common_kwargs["dataset_subset"] = None
common_kwargs["dataset_split"] = "tra
in"
else:
raise ValueError(f"Unknown dataset name: {args.dataset_name}")
# Remove None values
sample_kwargs = {k: v for k, v
in sample_kwargs.items() if v is not None}
return dataset_cls(**common_kwargs).sample(**sample_kwargs)
def ma
in(args: argparse.Namespace):
if args.seed is None:
args.seed = 0
print(args)
random.seed(args.seed)
# Sample the
requests.
print("*****args.
tokenizer:", args.
tokenizer)
tokenizer = Au
toTokenizer.from_pretra
ined(
args.
tokenizer,
trust_remote_code=True,
local_
files_only=True,
use_fast=True
)
tokenizer.padd
ing_side = "left" # 新增
requests = get_
requests(args,
tokenizer)
is_multi_modal = any(request.multi_modal_data is not None for request
in requests)
request_
outputs: Optional[list[Reques
tOutput]] = None
if args.backend == "hf":
assert args.tensor_parallel_size == 1
elapsed_time = run_hf(
requests,
args.model,
tokenizer,
args.n,
args.hf_max_batch_size,
args.trust_remote_code,
args.disable_de
tokenize,
)
else:
raise ValueError(f"Unknown backend: {args.backend}")
total_num_
tokens = sum(r.prompt_len + r.expected_
output_len for r
in requests)
total_
output_
tokens = sum(r.expected_
output_len for r
in requests)
total_prompt_
tokens =
total_num_
tokens -
total_
output_
tokens
print(
f"Throughput: {len(
requests) / elapsed_time:.2f}
requests/s, "
f"{
total_num_
tokens / elapsed_time:.2f}
total
tokens/s, "
f"{
total_
output_
tokens / elapsed_time:.2f}
output tokens/s"
)
print(f"
Total num prompt
tokens: {
total_prompt_
tokens}")
print(f"
Total num
output tokens: {
total_
output_
tokens}")
#
Output JSON results if specified
if args.
output_json:
results = {
"elapsed_time": elapsed_time,
"num_
requests": len(
requests),
"
total_num_
tokens":
total_num_
tokens,
"
requests_per_second": len(
requests) / elapsed_time,
"
tokens_per_second":
total_num_
tokens / elapsed_time,
}
with open(args.
output_json, "w") as f:
json.dump(results, f,
indent=4)
save_
to_py
torch_benchmark_
format(args, results)
def validate_args(args):
# === Deprecation and Default
ing ===
if args.dataset is not None:
warn
ings.warn(
"The '--dataset' argument will be deprecated
in the next release. "
"Please use '--dataset-name' and '--dataset-path'
instead.",
stacklevel=2,
)
args.dataset_path = args.dataset
if not getattr(args, "
tokenizer", None):
args.
tokenizer = args.model
# === Backend Validation ===
valid_backends = {"vllm", "hf", "mii", "vllm-chat"}
if args.backend not
in valid_backends:
raise ValueError(f"Unsupported backend: {args.backend}")
# === Dataset Configuration ===
if not args.dataset and not args.dataset_path:
print("When dataset path is not set, it will default
to random dataset")
args.dataset_name = "random"
if args.
input_len is None:
raise ValueError("
input_len must be provided for a random dataset")
# === Dataset Name Specific Checks ===
# --hf-subset and --hf-split: only used
# when dataset_name is 'hf'
if args.dataset_name != "hf" and (
getattr(args, "hf_subset", None) is not None
or getattr(args, "hf_split", None) is not None
):
warn
ings.warn(
"--hf-subset and --hf-split will be ignored \
s
ince --dataset-name is not 'hf'.",
stacklevel=2,
)
elif args.dataset_name == "hf":
if args.dataset_path
in (
VisionArenaDataset.SUPPORTED_DATASET_PATHS.keys()
| ConversationDataset.SUPPORTED_DATASET_PATHS
):
assert args.backend == "vllm-chat", (
f"{args.dataset_path} needs
to use vllm-chat as the backend."
) # noqa: E501
elif args.dataset_path
in (
InstructCoderDataset.SUPPORTED_DATASET_PATHS
| AIMODataset.SUPPORTED_DATASET_PATHS
):
assert args.backend == "vllm", (
f"{args.dataset_path} needs
to use vllm as the backend."
) # noqa: E501
else:
raise ValueError(f"{args.dataset_path} is not supported by hf dataset.")
# --random-range-ratio: only used when dataset_name is 'random'
if args.dataset_name != "random" and args.random_range_ratio is not None:
warn
ings.warn(
"--random-range-ratio will be ignored s
ince \
--dataset-name is not 'random'.",
stacklevel=2,
)
# --prefix-len: only used when dataset_name is 'random', 'sonnet', or not
# set.
if (
args.dataset_name not
in {"random", "sonnet", None}
and args.prefix_len is not None
):
warn
ings.warn(
"--prefix-len will be ignored s
ince --dataset-name\
is not 'random', 'sonnet', or not set.",
stacklevel=2,
)
# === LoRA Sett
ings ===
if getattr(args, "enable_lora", False) and args.backend != "vllm":
raise ValueError("LoRA benchmark
ing is only supported for vLLM backend")
if getattr(args, "enable_lora", False) and args.lora_path is None:
raise ValueError("LoRA path must be provided when enable_lora is True")
# === Backend-specific Validations ===
if args.backend == "hf" and args.hf_max_batch_size is None:
raise ValueError("HF max batch size is required for HF backend")
if args.backend != "hf" and args.hf_max_batch_size is not None:
raise ValueError("HF max batch size is only for HF backend.")
if (
args.backend
in {"hf", "mii"}
and getattr(args, "quantization", None) is not None
):
raise ValueError("Quantization is only for vLLM backend.")
if args.backend == "mii" and args.dtype != "au
to":
raise ValueError("dtype must be au
to for MII backend.")
if args.backend == "mii" and args.n != 1:
raise ValueError("n must be 1 for MII backend.")
if args.backend == "mii" and args.
tokenizer != args.model:
raise ValueError("
Tokenizer must be the same as the model for MII backend.")
# --data-parallel is not supported currently.
# https://github.com/vllm-project/vllm/issues/16222
if args.data_parallel_size > 1:
raise ValueError(
"Data parallel is not supported
in offl
ine benchmark, \
please use benchmark serv
ing
instead"
)
def create_argument_parser():
parser = FlexibleArgumentParser(description="Benchmark the throughput.")
parser.add_argument(
"--backend",
type=str,
choices=["vllm", "hf", "mii", "vllm-chat"],
default="vllm",
)
parser.add_argument(
"--dataset-name",
type=str,
choices=["sharegpt", "random", "sonnet", "burstgpt", "hf"],
help="Name of the dataset
to benchmark on.",
default="sharegpt",
)
parser.add_argument(
"--no-stream",
action="s
tore_true",
help="Do not load the dataset
in stream
ing mode.",
)
parser.add_argument(
"--dataset",
type=str,
default=None,
help="Path
to the ShareGPT dataset, will be deprecated
in\
the next release. The dataset is expected
to "
"be a json
in form of list[dict[..., conversations: "
"list[dict[..., value: <prompt_or_response>]]]]",
)
parser.add_argument(
"--dataset-path", type=str, default=None, help="Path
to the dataset"
)
parser.add_argument(
"--
input-len",
type=
int,
default=None,
help="
Input prompt length for each request",
)
parser.add_argument(
"--
output-len",
type=
int,
default=None,
help="
Output length for each request. Overrides the "
"
output length from the dataset.",
)
parser.add_argument(
"--n", type=
int, default=1, help="Number of generated sequences per prompt."
)
parser.add_argument(
"--num-prompts", type=
int, default=1000, help="Number of prompts
to process."
)
parser.add_argument(
"--hf-max-batch-size",
type=
int,
default=None,
help="Maximum batch size for HF backend.",
)
parser.add_argument(
"--
output-json",
type=str,
default=None,
help="Path
to save the throughput results
in JSON
format.",
)
parser.add_argument(
"--async-eng
ine",
action="s
tore_true",
default=False,
help="Use vLLM async eng
ine rather than LLM class.",
)
parser.add_argument(
"--disable-frontend-multiprocess
ing",
action="s
tore_true",
default=False,
help="Disable decoupled async eng
ine frontend.",
)
parser.add_argument(
"--disable-de
tokenize",
action="s
tore_true",
help=(
"Do not de
tokenize the response (i.e. do not
include "
"de
tokenization time
in the measurement)"
),
)
# LoRA
parser.add_argument(
"--lora-path",
type=str,
default=None,
help="Path
to the LoRA adapters
to use. This can be an absolute path, "
"a relative path, or a Hugg
ing Face model identifier.",
)
parser.add_argument(
"--prefix-len",
type=
int,
default=None,
help=f"Number of prefix
tokens
to be used
in RandomDataset "
"and SonnetDataset. For RandomDataset, the
total
input "
"length is the sum of prefix-len (default: "
f"{RandomDataset.DEFAULT_PREFIX_LEN}) and a random context length "
"sampled from [
input_len * (1 - range_ratio), "
"
input_len * (1 + range_ratio)]. For SonnetDataset, "
f"prefix_len (default: {SonnetDataset.DEFAULT_PREFIX_LEN}) "
"controls
how much of the
input is fixed l
ines versus "
"random l
ines, but the
total
input length rema
ins approximately "
"
input_len
tokens.",
)
# random dataset
parser.add_argument(
"--random-range-ratio",
type=float,
default=None,
help=f"Range ratio (default : {RandomDataset.DEFAULT_RANGE_RATIO}) "
"for sampl
ing
input/
output length, "
"used only for RandomDataset. Must be
in the range [0, 1)
to "
"def
ine a symmetric sampl
ing range "
"[length * (1 - range_ratio), length * (1 + range_ratio)].",
)
# hf dtaset
parser.add_argument(
"--hf-subset", type=str, default=None, help="Subset of the HF dataset."
)
parser.add_argument(
"--hf-split", type=str, default=None, help="Split of the HF dataset."
)
parser = AsyncEng
ineArgs.add_cli_args(parser)
return parser
if __name__ == "__ma
in__":
parser = create_argument_parser()
args = parser.parse_args()
if args.
tokenizer is None:
args.
tokenizer = args.model
validate_args(args)
ma
in(args)
'''
python nf4_bm.py \
--backend hf \
--dataset-name random \
--
input-len 512 \
--
output-len 128 \
--hf-max-batch-size 32 \
--model /models/z50051264/summary/Qwen2.5-7B-nf
python nf4_bm.py \
--backend hf \
--dataset-name random \
--
input-len 512 \
--
output-len 128 \
--hf-max-batch-size 32 \
--model /models/z50051264/vllm-0.10.0/benchmarks/pangu-ok-model-nf4
python nf4_bm.py \
--backend hf \
--dataset-name random \
--
input-len 512 \
--
output-len 128 \
--hf-max-batch-size 32 \
--model /models/z50051264/vllm-0.10.0/benchmarks/pangu
'''
修改上面的代码，支持并发测试；
类似于这种：
for parallel_num
in args.parallel_num:
for i, prompt_
tokens
in enumerate(args.prompt_
tokens):
output_
tokens = args.
output_
tokens[i]
input_
requests = get_dataset_
requests(args,
tokenizer, prompt_
tokens,
output_
tokens, parallel_num)
request_latency_record: List[Tuple[
int,
int, List]] = []
asyncio.run(benchmark(request_latency_record, args.backend, api_url,
input_
requests, args.best_of,
args.use_beam_search, args.request_rate, parallel_num,
args.epochs, args.app_code, args.
tokenizer,
args.served_model_name, args.num_scheduler_steps, args.use_spec_decode))
if args.use_spec_decode:
update_spec_
output_
tokens(request_latency_record,
tokenizer, args.backend)
statistics_and_
print_performance_data(args, prompt_
tokens,
output_
tokens, parallel_num,
request_latency_record, all_latency_record)
本文档解释了为何不能直接从Oracle应用程序的并发管理器中打印或重新打印PDF格式的并发请求报告，并提供了几种解决方法。
扫一扫
9090
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
