Chunked decoding in python

本文提供了一个简单的Python函数用于解析HTTP响应中的'chunked'编码数据。该函数避免了使用第三方库,通过查找特定字符串并进行十六进制转换来实现数据的逐步解析。

beezari: Chunked decoding in python

Chunked decoding in python

I haven't been able to find any short and quick method of decoding "chunked-encoding" encoded data in python without whacking 3rd party libraries, so here is my bet:
(chunked content encoding explained here: http://www.faqs.org/rfcs/rfc2616.html)

assuming that data contains raw http response. Code simplified for readability
def decode_chunked(data):
    offset = 0
    encdata = ''
    newdata = ''
    offset = string.index(data, "\r\n\r\n") + 4 # get the offset
    # of the data payload. you can also parse content-length header as well.
    encdata =data[offset:]
    try:
        while (encdata != ''):
            off = int(encdata[:string.index(encdata,"\r\n")],16)
            if off == 0:
                break
            encdata = encdata[string.index(encdata,"\r\n") + 2:]
            newdata = "%s%s" % (newdata, encdata[:off])
            encdata = encdata[off+2:]

    except:
       line = traceback.format_exc()
       print "Exception! %s" %line # probably indexes are wrong
    return newdata



for php version look here:

http://www.codingforums.com/showthread.php?t=147061

程序: import xarray as xr import numpy as np import cftime # 文件路径 input_file = r'E:\Except-Desktop\DC\CMIP6-download\historical-pp\pp_185001-201412.nc' # 输入文件名 output_file = r'E:\Except-Desktop\DC\CMIP6-download\historical-pp\pp_c1.nc' # 输出文件名 # 定义裁剪时间范围 start_time = cftime.DatetimeNoLeap(1961, 1, 1) end_time = cftime.DatetimeNoLeap(2014, 12, 31) # 打开NetCDF文件 ds = xr.open_dataset(input_file) # 裁剪时间范围 ds_time = ds.sel(time=slice(start_time, end_time)) # 保存裁剪后的数据到新文件 ds_time.to_netcdf(output_file) print(f"数据已裁剪并保存至 {output_file}") 输出: Warning (from warnings module): File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\conventions.py", line 204 var = coder.decode(var, name=name) SerializationWarning: variable 'pp' has multiple fill values {np.float32(1e+20), np.float64(1e+20)} defined, decoding all values to NaN. Traceback (most recent call last): File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\DC-run\8.27程序包\裁剪时间.py", line 23, in <module> ds_time.to_netcdf(output_file) File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\core\dataset.py", line 2029, in to_netcdf return to_netcdf( # type: ignore[return-value] # mypy cannot resolve the overloads:( File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\backends\api.py", line 1984, in to_netcdf dump_to_store( File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\backends\api.py", line 2031, in dump_to_store store.store(variables, attrs, check_encoding, writer, unlimited_dims=unlimited_dims) File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\backends\common.py", line 470, in store variables, attributes = self.encode(variables, attributes) File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\backends\common.py", line 653, in encode variables, attributes = cf_encoder(variables, attributes) File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\conventions.py", line 798, in cf_encoder new_vars[k] = encode_cf_variable(v, name=k) File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\conventions.py", line 102, in encode_cf_variable var = coder.encode(var, name=name) File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\coding\times.py", line 1362, in encode variable.data.dtype, np.datetime64 File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\core\variable.py", line 436, in data duck_array = self._data.get_duck_array() File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\core\indexing.py", line 843, in get_duck_array self._ensure_cached() File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\core\indexing.py", line 840, in _ensure_cached self.array = as_indexable(self.array.get_duck_array()) File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\core\indexing.py", line 797, in get_duck_array return self.array.get_duck_array() File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\coding\common.py", line 80, in get_duck_array return self.func(self.array.get_duck_array()) File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\core\indexing.py", line 652, in get_duck_array array = self.array[self.key] File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\backends\netCDF4_.py", line 108, in __getitem__ return indexing.explicit_indexing_adapter( File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\core\indexing.py", line 1021, in explicit_indexing_adapter result = raw_indexing_method(raw_key.tuple) File "C:\Users\ASUS\AppData\Local\Programs\Python\Python313\Lib\site-packages\xarray\backends\netCDF4_.py", line 121, in _getitem array = getitem(original_array, key) File "src\\netCDF4\\_netCDF4.pyx", line 5079, in netCDF4._netCDF4.Variable.__getitem__ File "src\\netCDF4\\_netCDF4.pyx", line 6028, in netCDF4._netCDF4.Variable._get numpy._core._exceptions._ArrayMemoryError: Unable to allocate 4.45 GiB for an array with shape (648, 15, 384, 320) and data type float32 Raised while encoding variable 'pp' with value <xarray.Variable (time: 648, lev: 15, nlat: 384, nlon: 320)> Size: 5GB [1194393600 values with dtype=float32] Attributes: (12/20) cell_measures: area: areacello volume: volcello cell_methods: area: mean where sea time: mean comment: photoC_TOT description: total primary (organic carbon) production by phytoplankton frequency: mon id: pp ... ... time_title: Temporal mean title: Primary Carbon Production by Phytoplankton type: real units: mol m-3 s-1 variable_id: pp coordinates: time lev lat lon
最新发布
09-01
运行CUDA_VISIBLE_DEVICES=0,2 dbgpt start webserver --config /app/configs/dbgpt-local-vllm.toml报错如下: =========================== VLLMDeployModelParameters =========================== name: DeepSeek-R1-Distill-Qwen-32B provider: vllm verbose: False concurrency: 100 backend: None prompt_template: None context_length: None reasoning_model: None path: models/DeepSeek-R1-Distill-Qwen-32B device: auto trust_remote_code: True download_dir: None load_format: auto config_format: auto dtype: auto kv_cache_dtype: auto seed: 0 max_model_len: None distributed_executor_backend: None pipeline_parallel_size: 1 tensor_parallel_size: 1 max_parallel_loading_workers: None block_size: None enable_prefix_caching: None swap_space: 4.0 cpu_offload_gb: 0.0 gpu_memory_utilization: 0.9 max_num_batched_tokens: None max_num_seqs: 2 max_logprobs: 20 revision: None code_revision: None tokenizer_revision: None tokenizer_mode: auto quantization: fp8 max_seq_len_to_capture: 8192 worker_cls: auto extras: None ====================================================================== 2025-08-05 07:43:32 1249bbe41ac7 dbgpt.util.code.server[4248] INFO Code server is ready INFO 08-05 07:43:36 config.py:520] This model supports multiple tasks: {'score', 'classify', 'generate', 'reward', 'embed'}. Defaulting to 'generate'. WARNING 08-05 07:43:36 arg_utils.py:1107] Chunked prefill is enabled by default for models with max_model_len > 32K. Currently, chunked prefill might not work with some features or models. If you encounter any issues, please disable chunked prefill by setting --enable-chunked-prefill=False. INFO 08-05 07:43:36 config.py:1483] Chunked prefill is enabled with max_num_batched_tokens=2048. INFO 08-05 07:43:37 llm_engine.py:232] Initializing an LLM engine (v0.7.0) with config: model='/app/models/DeepSeek-R1-Distill-Qwen-32B', speculative_config=None, tokenizer='/app/models/DeepSeek-R1-Distill-Qwen-32B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=131072, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=fp8, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='xgrammar'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=/app/models/DeepSeek-R1-Distill-Qwen-32B, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=False, chunked_prefill_enabled=True, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={"splitting_ops":[],"compile_sizes":[],"cudagraph_capture_sizes":[2,1],"max_capture_size":2}, use_cached_outputs=False, INFO 08-05 07:43:37 cuda.py:225] Using Flash Attention backend. INFO 08-05 07:43:37 model_runner.py:1110] Starting to load model /app/models/DeepSeek-R1-Distill-Qwen-32B... /app/packages/dbgpt-core/src/dbgpt/util/model_utils.py:27: UserWarning: 'has_mps' is deprecated, please use 'torch.backends.mps.is_built()' if (hasattr(backends, "mps") and backends.mps.is_built()) or torch.has_mps: 2025-08-05 07:43:38 1249bbe41ac7 dbgpt.util.model_utils[4248] INFO Clear torch cache of device: cuda:0 2025-08-05 07:43:38 1249bbe41ac7 dbgpt.util.model_utils[4248] INFO Clear torch cache of device: cuda:1 2025-08-05 07:43:38 1249bbe41ac7 dbgpt.model.cluster.worker.embedding_worker[4248] INFO Load embeddings model: bge-large-zh-v1.5 2025-08-05 07:43:38 1249bbe41ac7 datasets[4248] INFO PyTorch version 2.5.1+cu121 available. 2025-08-05 07:43:38 1249bbe41ac7 datasets[4248] INFO Duckdb version 1.2.0 available. 2025-08-05 07:43:38 1249bbe41ac7 sentence_transformers.SentenceTransformer[4248] INFO Use pytorch device_name: cuda 2025-08-05 07:43:38 1249bbe41ac7 sentence_transformers.SentenceTransformer[4248] INFO Load pretrained SentenceTransformer: /app/models/bge-large-zh-v1.5 INFO: 127.0.0.1:42180 - "POST /api/controller/models HTTP/1.1" 200 OK 2025-08-05 07:43:39 1249bbe41ac7 dbgpt.model.cluster.worker.manager[4248] ERROR Error starting worker manager: model DeepSeek-R1-Distill-Qwen-32B@vllm(172.17.0.2:8002) start failed, Traceback (most recent call last): File "/app/packages/dbgpt-core/src/dbgpt/model/cluster/worker/manager.py", line 631, in _start_worker await self.run_blocking_func( File "/app/packages/dbgpt-core/src/dbgpt/model/cluster/worker/manager.py", line 146, in run_blocking_func return await loop.run_in_executor(self.executor, func, *args) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.11/concurrent/futures/thread.py", line 58, in run result = self.fn(*self.args, **self.kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/packages/dbgpt-core/src/dbgpt/model/cluster/worker/default_worker.py", line 122, in start self.model, self.tokenizer = self.ml.loader_with_params( ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/packages/dbgpt-core/src/dbgpt/model/adapter/loader.py", line 70, in loader_with_params return llm_adapter.load_from_params(model_params) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/app/packages/dbgpt-core/src/dbgpt/model/adapter/vllm_adapter.py", line 488, in load_from_params engine = AsyncLLMEngine.from_engine_args(engine_args) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/engine/async_llm_engine.py", line 642, in from_engine_args engine = cls( ^^^^ File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/engine/async_llm_engine.py", line 592, in __init__ self.engine = self._engine_class(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/engine/async_llm_engine.py", line 265, in __init__ super().__init__(*args, **kwargs) File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/engine/llm_engine.py", line 271, in __init__ self.model_executor = executor_class(vllm_config=vllm_config, ) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/executor/executor_base.py", line 49, in __init__ self._init_executor() File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/executor/uniproc_executor.py", line 40, in _init_executor self.collective_rpc("load_model") File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/executor/uniproc_executor.py", line 49, in collective_rpc answer = run_method(self.driver_worker, method, args, kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/utils.py", line 2208, in run_method return func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/worker/worker.py", line 182, in load_model self.model_runner.load_model() File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/worker/model_runner.py", line 1112, in load_model self.model = get_model(vllm_config=self.vllm_config) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/model_executor/model_loader/__init__.py", line 12, in get_model return loader.load_model(vllm_config=vllm_config) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/model_executor/model_loader/loader.py", line 376, in load_model model = _initialize_model(vllm_config=vllm_config) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/model_executor/model_loader/loader.py", line 118, in _initialize_model return model_class(vllm_config=vllm_config, prefix=prefix) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/model_executor/models/qwen2.py", line 451, in __init__ self.model = Qwen2Model(vllm_config=vllm_config, ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/compilation/decorators.py", line 149, in __init__ old_init(self, vllm_config=vllm_config, prefix=prefix, **kwargs) File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/model_executor/models/qwen2.py", line 305, in __init__ self.start_layer, self.end_layer, self.layers = make_layers( ^^^^^^^^^^^^ File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/model_executor/models/utils.py", line 555, in make_layers [PPMissingLayer() for _ in range(start_layer)] + [ ^ File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/model_executor/models/utils.py", line 556, in <listcomp> maybe_offload_to_cpu(layer_fn(prefix=f"{prefix}.{idx}")) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/model_executor/models/qwen2.py", line 307, in <lambda> lambda prefix: Qwen2DecoderLayer(config=config, ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/model_executor/models/qwen2.py", line 206, in __init__ self.self_attn = Qwen2Attention( ^^^^^^^^^^^^^^^ File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/model_executor/models/qwen2.py", line 134, in __init__ self.qkv_proj = QKVParallelLinear( ^^^^^^^^^^^^^^^^^^ File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/model_executor/layers/linear.py", line 728, in __init__ super().__init__(input_size=input_size, File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/model_executor/layers/linear.py", line 311, in __init__ self.quant_method.create_weights( File "//opt/.uv.venv/lib/python3.11/site-packages/vllm/model_executor/layers/quantization/fp8.py", line 199, in create_weights weight = ModelWeightParameter(data=torch.empty( ^^^^^^^^^^^^ File "//opt/.uv.venv/lib/python3.11/site-packages/torch/utils/_device.py", line 106, in __torch_function__ return func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 70.00 MiB. GPU 0 has a total capacity of 44.53 GiB of which 15.94 MiB is free. Process 1334940 has 44.51 GiB memory in use. Of the allocated memory 44.17 GiB is allocated by PyTorch, and 14.30 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) ;model bge-large-zh-v1.5@hf(172.17.0.2:8002) start successfully INFO: Shutting down
08-06
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值