pytorch每日一学15(torch.as_strided())根据步长创建一个现有tensor的视图

""" Inference benchmarking tool. Requirements: transformers accelerate bitsandbytes optimum-benchmark Usage: python inference_benchmark.py model_id options: -h, --help show this help message and exit --configs {bf16,fp16,nf4,nf4-dq,int8,int8-decomp} [{bf16,fp16,nf4,nf4-dq,int8,int8-decomp} ...] --bf16 --fp16 --nf4 --nf4-dq --int8 --int8-decomp --batches BATCHES [BATCHES ...] --input-length INPUT_LENGTH --out-dir OUT_DIR """ import argparse from pathlib import Path from optimum_benchmark import Benchmark, BenchmarkConfig, InferenceConfig, ProcessConfig, PyTorchConfig from optimum_benchmark.logging_utils import setup_logging import torch,torch_npu from torch_npu.contrib import transfer_to_npu BFLOAT16_SUPPORT = torch.cuda.get_device_capability()[0] >= 8 WEIGHTS_CONFIGS = { "fp16": {"torch_dtype": "float16", "quantization_scheme": None, "quantization_config": {}}, "bf16": {"torch_dtype": "bfloat16", "quantization_scheme": None, "quantization_config": {}}, "nf4": { "torch_dtype": "bfloat16" if BFLOAT16_SUPPORT else "float16", "quantization_scheme": "bnb", "quantization_config": { "load_in_4bit": True, "bnb_4bit_quant_type": "nf4", "bnb_4bit_use_double_quant": False, "bnb_4bit_compute_dtype": torch.bfloat16 if BFLOAT16_SUPPORT else "float16", }, }, "nf4-dq": { "torch_dtype": "bfloat16" if BFLOAT16_SUPPORT else "float16", "quantization_scheme": "bnb", "quantization_config": { "load_in_4bit": True, "bnb_4bit_quant_type": "nf4", "bnb_4bit_use_double_quant": True, "bnb_4bit_compute_dtype": torch.bfloat16 if BFLOAT16_SUPPORT else "float16", }, }, "int8-decomp": { "torch_dtype": "float16", "quantization_scheme": "bnb", "quantization_config": { "load_in_8bit": True, "llm_int8_threshold": 6.0, }, }, "int8": { "torch_dtype": "float16", "quantization_scheme": "bnb", "quantization_config": { "load_in_8bit": True, "llm_int8_threshold": 0.0, }, }, } if __name__ == "__main__": setup_logging(level="INFO") parser = argparse.ArgumentParser(description="bitsandbytes inference benchmark tool") parser.add_argument("model_id", type=str, help="The model checkpoint to use.") parser.add_argument( "--configs", nargs="+", choices=["bf16", "fp16", "nf4", "nf4-dq", "int8", "int8-decomp"], default=["nf4", "int8", "int8-decomp"], ) parser.add_argument("--bf16", dest="configs", action="append_const", const="bf16") parser.add_argument("--fp16", dest="configs", action="append_const", const="fp16") parser.add_argument("--nf4", dest="configs", action="append_const", const="nf4") parser.add_argument("--nf4-dq", dest="configs", action="append_const", const="nf4-dq") parser.add_argument("--int8", dest="configs", action="append_const", const="int8") parser.add_argument("--int8-decomp", dest="configs", action="append_const", const="int8-decomp") parser.add_argument("--batches", nargs="+", type=int, default=[1, 8, 16, 32]) parser.add_argument("--input-length", type=int, default=64) parser.add_argument("--out-dir", type=str, default="reports") args = parser.parse_args() out_dir = Path(args.out_dir) out_dir.mkdir(parents=True, exist_ok=True) for batch_size in args.batches: print(f"Benchmarking batch size: {batch_size}") for config in args.configs: launcher_config = ProcessConfig(device_isolation=True, start_method="spawn") scenario_config = InferenceConfig( latency=True, memory=True, input_shapes={"batch_size": batch_size, "sequence_length": args.input_length}, ) backend_config = PyTorchConfig( device_map={"":0}, no_weights=False, model=args.model_id, trust_remote_code=True, **WEIGHTS_CONFIGS[config], ) benchmark_config = BenchmarkConfig( name=f"benchmark-{config}-bsz{batch_size}", scenario=scenario_config, launcher=launcher_config, backend=backend_config, ) out_path = out_dir / f"benchmark_{config}_bsz{batch_size}.json" benchmark_report = Benchmark.launch(benchmark_config) benchmark_report.log() benchmark_report.save_json(out_path) ''' python inference_benchmark.py "/models/z50051264/summary/Qwen2.5-7B-nf4" \ --configs bf16 nf4 \ --batches 1 4 8 \ --input-length 128 \ --out-dir ./qwen2.5_nf4_reports ''' 我使用的是npu [root@190f3c453709 benchmarking]# python inference_benchmark.py "/models/z50051264/summary/Qwen2.5-7B-nf4" --configs bf16 nf4 --batches 1 4 8 --input-length 128 --out-dir ./qwen2.5_nf4_reports /usr/local/python3.10.17/lib/python3.10/site-packages/torch_npu/contrib/transfer_to_npu.py:292: ImportWarning: ************************************************************************************************************* The torch.Tensor.cuda and torch.nn.Module.cuda are replaced with torch.Tensor.npu and torch.nn.Module.npu now.. The torch.cuda.DoubleTensor is replaced with torch.npu.FloatTensor cause the double type is not supported now.. The backend in torch.distributed.init_process_group set to hccl now.. The torch.cuda.* and torch.cuda.amp.* are replaced with torch.npu.* and torch.npu.amp.* now.. The device parameters have been replaced with npu in the function below: torch.logspace, torch.randint, torch.hann_window, torch.rand, torch.full_like, torch.ones_like, torch.rand_like, torch.randperm, torch.arange, torch.frombuffer, torch.normal, torch._empty_per_channel_affine_quantized, torch.empty_strided, torch.empty_like, torch.scalar_tensor, torch.tril_indices, torch.bartlett_window, torch.ones, torch.sparse_coo_tensor, torch.randn, torch.kaiser_window, torch.tensor, torch.triu_indices, torch.as_tensor, torch.zeros, torch.randint_like, torch.full, torch.eye, torch._sparse_csr_tensor_unsafe, torch.empty, torch._sparse_coo_tensor_unsafe, torch.blackman_window, torch.zeros_like, torch.range, torch.sparse_csr_tensor, torch.randn_like, torch.from_file, torch._cudnn_init_dropout_state, torch._empty_affine_quantized, torch.linspace, torch.hamming_window, torch.empty_quantized, torch._pin_memory, torch.autocast, torch.load, torch.Generator, torch.set_default_device, torch.Tensor.new_empty, torch.Tensor.new_empty_strided, torch.Tensor.new_full, torch.Tensor.new_ones, torch.Tensor.new_tensor, torch.Tensor.new_zeros, torch.Tensor.to, torch.Tensor.pin_memory, torch.nn.Module.to, torch.nn.Module.to_empty ************************************************************************************************************* warnings.warn(msg, ImportWarning) /usr/local/python3.10.17/lib/python3.10/site-packages/torch_npu/contrib/transfer_to_npu.py:247: RuntimeWarning: torch.jit.script and torch.jit.script_method will be disabled by transfer_to_npu, which currently does not support them, if you need to enable them, please do not use transfer_to_npu. warnings.warn(msg, RuntimeWarning) /usr/local/python3.10.17/lib/python3.10/site-packages/torch_npu/npu/utils.py:118: UserWarning: torch.npu.get_device_capability isn't implemented! warnings.warn("torch.npu.get_device_capability isn't implemented!") Traceback (most recent call last): File "/models/z50051264/bitsandbytes-main/benchmarking/inference_benchmark.py", line 35, in <module> BFLOAT16_SUPPORT = torch.cuda.get_device_capability()[0] >= 8 TypeError: 'NoneType' object is not subscriptable [ERROR] 2025-07-28-02:31:16 (PID:668, Device:-1, RankID:-1) ERR99999 UNKNOWN applicaiton exception /usr/local/python3.10.17/lib/python3.10/tempfile.py:869: ResourceWarning: Implicitly cleaning up <TemporaryDirectory '/tmp/tmpv19_pa_6'> _warnings.warn(warn_message, ResourceWarning)