{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# vllm-lens Benchmark: Overhead of Activation Capture & Steering\n", "\n", "Measures the performance overhead of vllm-lens features compared to plain vLLM generation.\n", "\n", "A single `AsyncLLMEngine` is used throughout — the vllm-lens plugin is always loaded\n", "(via entry point) but only activates when `extra_args` contains\n", "`output_residual_stream` or `apply_steering_vectors`. The baseline is generation\n", "with no `extra_args`, i.e. effectively plain vLLM.\n", "\n", "| # | Benchmark | Description |\n", "|---|-----------|-------------|\n", "| 1 | Baseline | Pure generation, no vllm-lens features |\n", "| 2 | One layer | Capture residual stream from 1 layer |\n", "| 3 | All layers | Capture residual stream from all 80 layers |\n", "| 4 | Random layer | Each request captures a random layer (0–10) |\n", "| 5 | Steering | Apply a steering vector at layer 5 |\n", "| 6 | LoRA alt | Alternate LoRA adapter on/off per request |" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import random\n", "import time\n", "from collections.abc import Callable\n", "from dataclasses import dataclass\n", "\n", "import torch\n", "from datasets import load_dataset\n", "from tqdm.asyncio import tqdm\n", "from vllm import AsyncEngineArgs, AsyncLLMEngine, SamplingParams\n", "from vllm.lora.request import LoRARequest\n", "\n", "MODEL = \"meta-llama/Llama-3.3-70B-Instruct\"\n", "GPU_MEM = 0.85\n", "NUM_PROMPTS = 50\n", "MAX_TOKENS = 1024\n", "TEMPERATURE = 0.0\n", "SEED = 42\n", "TP_SIZE = 4\n", "\n", "# Llama 3.3 70B: 80 decoder layers (0-79), 8192 hidden dim\n", "NUM_LAYERS = 80\n", "HIDDEN_DIM = 8192\n", "\n", "LORA_PATH = \"adamkarvonen/checkpoints_act_cls_latentqa_pretrain_mix_adding_Llama-3_3-70B-Instruct\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Load Dataset" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loaded 50 prompts\n", "Example: Give three tips for staying healthy....\n" ] } ], "source": [ "ds = load_dataset(\"tatsu-lab/alpaca\", split=\"train\")\n", "prompts = [ds[i][\"instruction\"] for i in range(NUM_PROMPTS)]\n", "print(f\"Loaded {len(prompts)} prompts\")\n", "print(f\"Example: {prompts[0][:100]}...\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Initialize Engine" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "INFO 03-14 10:30:41 [model.py:531] Resolved architecture: LlamaForCausalLM\n", "INFO 03-14 10:30:41 [model.py:1554] Using max model len 1024\n", "INFO 03-14 10:30:41 [scheduler.py:231] Chunked prefill is enabled with max_num_batched_tokens=2048.\n", "INFO 03-14 10:30:41 [vllm.py:747] Asynchronous scheduling is enabled.\n", "WARNING 03-14 10:30:41 [vllm.py:781] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none\n", "WARNING 03-14 10:30:41 [vllm.py:792] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.\n", "INFO 03-14 10:30:46 [vllm.py:957] Cudagraph is disabled under eager mode\n", "(EngineCore_DP0 pid=35711) INFO 03-14 10:30:48 [core.py:101] Initializing a V1 LLM engine (v0.17.0) with config: model='meta-llama/Llama-3.3-70B-Instruct', speculative_config=None, tokenizer='meta-llama/Llama-3.3-70B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=1024, download_dir=None, load_format=auto, tensor_parallel_size=4, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=meta-llama/Llama-3.3-70B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'splitting_ops': [], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': True, 'static_all_moe_layers': []}\n", "(EngineCore_DP0 pid=35711) WARNING 03-14 10:30:48 [multiproc_executor.py:945] Reducing Torch parallelism from 288 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.\n", "(EngineCore_DP0 pid=35711) INFO 03-14 10:30:48 [multiproc_executor.py:134] DP group leader: node_rank=0, node_rank_within_dp=0, master_addr=127.0.0.1, mq_connect_ip=10.242.3.185 (local), world_size=4, local_world_size=4\n", "(EngineCore_DP0 pid=35711) (EngineCore_DP0 pid=35711) (EngineCore_DP0 pid=35711) INFO 03-14 10:30:50 [worker_base.py:283] Injected into for extended collective_rpc calls ['clear_captured_states', 'clear_steering_data', 'get_captured_states', 'install_hooks', 'set_steering_data']\n", "INFO 03-14 10:30:50 [worker_base.py:283] Injected into for extended collective_rpc calls ['clear_captured_states', 'clear_steering_data', 'get_captured_states', 'install_hooks', 'set_steering_data']\n", "INFO 03-14 10:30:50 [worker_base.py:283] Injected into for extended collective_rpc calls ['clear_captured_states', 'clear_steering_data', 'get_captured_states', 'install_hooks', 'set_steering_data']\n", "(EngineCore_DP0 pid=35711) INFO 03-14 10:30:50 [worker_base.py:283] Injected into for extended collective_rpc calls ['clear_captured_states', 'clear_steering_data', 'get_captured_states', 'install_hooks', 'set_steering_data']\n", "(EngineCore_DP0 pid=35711) (Worker pid=35733) INFO 03-14 10:30:55 [parallel_state.py:1393] world_size=4 rank=2 local_rank=2 distributed_init_method=tcp://127.0.0.1:36823 backend=nccl\n", "(EngineCore_DP0 pid=35711) (Worker pid=35729) INFO 03-14 10:30:55 [parallel_state.py:1393] world_size=4 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:36823 backend=nccl\n", "(EngineCore_DP0 pid=35711) (Worker pid=35731) INFO 03-14 10:30:55 [parallel_state.py:1393] world_size=4 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:36823 backend=nccl\n", "(EngineCore_DP0 pid=35711) (Worker pid=35735) INFO 03-14 10:30:55 [parallel_state.py:1393] world_size=4 rank=3 local_rank=3 distributed_init_method=tcp://127.0.0.1:36823 backend=nccl\n", "[Gloo] Rank [Gloo] Rank 13 is connected to is connected to 33 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 33\n", "\n", "[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3\n", "[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3\n", "[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3\n", "[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3\n", "[Gloo] Rank [Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 13 is connected to \n", "3 peer ranks. Expected number of connected peer ranks is : 3\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "(EngineCore_DP0 pid=35711) (EngineCore_DP0 pid=35711) (Worker pid=35729) (Worker pid=35735) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.\n", ":1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.\n", "(EngineCore_DP0 pid=35711) (EngineCore_DP0 pid=35711) (Worker pid=35729) (Worker pid=35735) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.\n", ":1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.\n", "(EngineCore_DP0 pid=35711) (Worker pid=35731) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.\n", "(EngineCore_DP0 pid=35711) (Worker pid=35731) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "(EngineCore_DP0 pid=35711) (Worker pid=35729) INFO 03-14 10:30:56 [pynccl.py:111] vLLM is using nccl==2.27.5\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "(EngineCore_DP0 pid=35711) (Worker pid=35733) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.\n", "(EngineCore_DP0 pid=35711) (Worker pid=35733) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "(EngineCore_DP0 pid=35711) (EngineCore_DP0 pid=35711) (EngineCore_DP0 pid=35711) (Worker pid=35729) (Worker pid=35735) (Worker pid=35733) WARNING 03-14 10:30:59 [symm_mem.py:107] SymmMemCommunicator: symmetric memory multicast operations are not supported.\n", "WARNING 03-14 10:30:59 [symm_mem.py:107] SymmMemCommunicator: symmetric memory multicast operations are not supported.\n", "WARNING 03-14 10:30:59 [symm_mem.py:107] SymmMemCommunicator: symmetric memory multicast operations are not supported.\n", "(EngineCore_DP0 pid=35711) (Worker pid=35731) WARNING 03-14 10:30:59 [symm_mem.py:107] SymmMemCommunicator: symmetric memory multicast operations are not supported.\n", "(EngineCore_DP0 pid=35711) (EngineCore_DP0 pid=35711) (EngineCore_DP0 pid=35711) (EngineCore_DP0 pid=35711) (Worker pid=35731) (Worker pid=35735) (Worker pid=35733) (Worker pid=35729) INFO 03-14 10:30:59 [parallel_state.py:1715] rank 1 in world size 4 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank N/A, EPLB rank N/A\n", "INFO 03-14 10:30:59 [parallel_state.py:1715] rank 3 in world size 4 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 3, EP rank N/A, EPLB rank N/A\n", "INFO 03-14 10:30:59 [parallel_state.py:1715] rank 2 in world size 4 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 2, EP rank N/A, EPLB rank N/A\n", "INFO 03-14 10:30:59 [parallel_state.py:1715] rank 0 in world size 4 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A\n", "[Gloo] Rank 0 is connected to 0[Gloo] Rank peer ranks. 0Expected number of connected peer ranks is : is connected to 00\n", " peer ranks. Expected number of connected peer ranks is : 0\n", "[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0\n", "[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0\n", "[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0\n", "[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0[Gloo] Rank \n", "0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0\n", "[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0\n", "[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0\n", "[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0\n", "[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0\n", "[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0\n", "[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0\n", "[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0\n", "[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0[Gloo] Rank \n", "0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0\n", "(EngineCore_DP0 pid=35711) (Worker pid=35735) INFO 03-14 10:31:00 [base.py:106] Offloader set to NoopOffloader\n", "(EngineCore_DP0 pid=35711) (Worker pid=35731) INFO 03-14 10:31:01 [base.py:106] Offloader set to NoopOffloader\n", "(EngineCore_DP0 pid=35711) (Worker pid=35733) INFO 03-14 10:31:01 [base.py:106] Offloader set to NoopOffloader\n", "(EngineCore_DP0 pid=35711) (Worker pid=35729) INFO 03-14 10:31:01 [base.py:106] Offloader set to NoopOffloader\n", "(EngineCore_DP0 pid=35711) (Worker pid=35729) (Worker_TP0 pid=35729) INFO 03-14 10:31:01 [gpu_model_runner.py:4255] Starting to load model meta-llama/Llama-3.3-70B-Instruct...\n", "(EngineCore_DP0 pid=35711) (Worker pid=35729) (Worker_TP0 pid=35729) INFO 03-14 10:31:02 [cuda.py:405] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].\n", "(EngineCore_DP0 pid=35711) (Worker pid=35729) (Worker_TP0 pid=35729) INFO 03-14 10:31:02 [flash_attn.py:587] Using FlashAttention version 3\n", "(EngineCore_DP0 pid=35711) (Worker pid=35735) (Worker_TP3 pid=35735) INFO 03-14 10:31:02 [weight_utils.py:561] Time spent downloading weights for meta-llama/Llama-3.3-70B-Instruct: 1.186229 seconds\n", "(EngineCore_DP0 pid=35711) (Worker pid=35733) (Worker_TP2 pid=35733) INFO 03-14 10:31:03 [weight_utils.py:561] Time spent downloading weights for meta-llama/Llama-3.3-70B-Instruct: 1.010568 seconds\n", "(EngineCore_DP0 pid=35711) (Worker pid=35729) (Worker_TP0 pid=35729) INFO 03-14 10:31:04 [weight_utils.py:561] Time spent downloading weights for meta-llama/Llama-3.3-70B-Instruct: 0.856079 seconds\n", "(EngineCore_DP0 pid=35711) (Worker pid=35731) (Worker_TP1 pid=35731) INFO 03-14 10:31:05 [weight_utils.py:561] Time spent downloading weights for meta-llama/Llama-3.3-70B-Instruct: 0.978212 seconds\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "90d3c6b415fe49b396caab90ac48d78c", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Loading safetensors checkpoint shards: 0% Completed | 0/30 [00:00 int:\n", " \"\"\"Run a single request, return number of output tokens.\"\"\"\n", " final = None\n", " async for output in engine.generate(\n", " prompt, params, request_id, lora_request=lora_request\n", " ):\n", " final = output\n", " return len(final.outputs[0].token_ids)\n", "\n", "\n", "async def run_benchmark(\n", " engine: AsyncLLMEngine,\n", " prompts: list[str],\n", " make_params: Callable[[int], SamplingParams],\n", " name: str,\n", " make_lora: Callable[[int], LoRARequest | None] | None = None,\n", ") -> BenchmarkResult:\n", " \"\"\"Fire all requests concurrently with tqdm progress and return timing results.\"\"\"\n", "\n", " async def _task(idx: int) -> int:\n", " lora = make_lora(idx) if make_lora else None\n", " return await _generate_one(\n", " engine, prompts[idx], make_params(idx), f\"{name}-{idx}\", lora\n", " )\n", "\n", " start = time.perf_counter()\n", " token_counts = await tqdm.gather(\n", " *[_task(i) for i in range(len(prompts))],\n", " desc=name,\n", " )\n", " elapsed = time.perf_counter() - start\n", "\n", " total_tokens = sum(token_counts)\n", " return BenchmarkResult(\n", " name=name,\n", " wall_time=elapsed,\n", " num_requests=len(prompts),\n", " total_output_tokens=total_tokens,\n", " requests_per_second=len(prompts) / elapsed,\n", " tokens_per_second=total_tokens / elapsed,\n", " )\n", "\n", "\n", "def compare(baseline: BenchmarkResult, lens: BenchmarkResult) -> None:\n", " \"\"\"Print a comparison table with overhead percentages.\"\"\"\n", " oh_time = (lens.wall_time - baseline.wall_time) / baseline.wall_time * 100\n", " oh_tps = (\n", " (baseline.tokens_per_second - lens.tokens_per_second)\n", " / baseline.tokens_per_second\n", " * 100\n", " )\n", "\n", " print(f\"\\n{'Metric':<25} {'Baseline':>12} {'Lens':>12} {'Overhead':>10}\")\n", " print(\"-\" * 61)\n", " print(\n", " f\"{'Wall time (s)':<25} {baseline.wall_time:>12.2f} {lens.wall_time:>12.2f} {oh_time:>+9.1f}%\"\n", " )\n", " print(\n", " f\"{'Requests/s':<25} {baseline.requests_per_second:>12.1f} {lens.requests_per_second:>12.1f}\"\n", " )\n", " print(\n", " f\"{'Tokens/s':<25} {baseline.tokens_per_second:>12.1f} {lens.tokens_per_second:>12.1f} {oh_tps:>+9.1f}%\"\n", " )\n", " print(\n", " f\"{'Total output tokens':<25} {baseline.total_output_tokens:>12} {lens.total_output_tokens:>12}\"\n", " )" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Warmup" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "warmup: 0%| | 0/10 [00:00 SamplingParams:\n", " return SamplingParams(\n", " temperature=TEMPERATURE,\n", " max_tokens=MAX_TOKENS,\n", " extra_args={\"output_residual_stream\": [random_layers[idx]]},\n", " )\n", "\n", "\n", "random_layer = await run_benchmark(\n", " engine, prompts, make_random_layer_params, \"random-layer\"\n", ")\n", "compare(baseline, random_layer)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Benchmark 5 — Steering Vector\n", "\n", "Apply a random steering vector at layer 5 with `norm_match=True` and `scale=0.1`." ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "steering: 100%|██████████| 50/50 [01:36<00:00, 1.94s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Metric Baseline Lens Overhead\n", "-------------------------------------------------------------\n", "Wall time (s) 82.15 96.92 +18.0%\n", "Requests/s 0.6 0.5\n", "Tokens/s 413.6 374.0 +9.6%\n", "Total output tokens 33976 36251\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "from vllm_lens._helpers.types import SteeringVector\n", "\n", "torch.manual_seed(SEED)\n", "steering_config = [\n", " SteeringVector(\n", " activations=torch.randn(1, HIDDEN_DIM),\n", " layer_indices=[5],\n", " scale=0.1,\n", " norm_match=True,\n", " )\n", "]\n", "\n", "steering_params = SamplingParams(\n", " temperature=TEMPERATURE,\n", " max_tokens=MAX_TOKENS,\n", " extra_args={\"apply_steering_vectors\": steering_config},\n", ")\n", "steering = await run_benchmark(engine, prompts, lambda i: steering_params, \"steering\")\n", "compare(baseline, steering)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Benchmark 6 — LoRA Alternation\n", "\n", "Alternate a LoRA adapter on/off per request (even index = no LoRA, odd index = LoRA).\n", "Measures the overhead of mixed LoRA/base-model serving." ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "lora-alt: 0%| | 0/50 [00:00 LoRARequest | None:\n", " return lora_request if idx % 2 else None\n", "\n", "\n", "lora_params = SamplingParams(temperature=TEMPERATURE, max_tokens=MAX_TOKENS)\n", "lora_alt = await run_benchmark(\n", " engine, prompts, lambda i: lora_params, \"lora-alt\", make_lora=make_lora_alternating\n", ")\n", "compare(baseline, lora_alt)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Summary" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Benchmark Time (s) Tok/s Req/s Overhead\n", "==============================================================\n", "baseline 82.15 413.6 0.6 ---\n", "one-layer 92.02 351.4 0.5 +12.0%\n", "all-layers 342.59 93.9 0.1 +317.0%\n", "random-layer 91.78 366.1 0.5 +11.7%\n", "steering 96.92 374.0 0.5 +18.0%\n", "lora-alt 166.26 203.6 0.3 +102.4%\n" ] } ], "source": [ "results = [baseline, one_layer, all_layers, random_layer, steering, lora_alt]\n", "\n", "print(\n", " f\"{'Benchmark':<20} {'Time (s)':>10} {'Tok/s':>10} {'Req/s':>10} {'Overhead':>10}\"\n", ")\n", "print(\"=\" * 62)\n", "for r in results:\n", " if r is baseline:\n", " overhead = \"---\"\n", " else:\n", " pct = (r.wall_time - baseline.wall_time) / baseline.wall_time * 100\n", " overhead = f\"{pct:+.1f}%\"\n", " print(\n", " f\"{r.name:<20} {r.wall_time:>10.2f} {r.tokens_per_second:>10.1f}\"\n", " f\" {r.requests_per_second:>10.1f} {overhead:>10}\"\n", " )" ] } ], "metadata": { "kernelspec": { "display_name": "vllm-lens (3.12.13)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.13" } }, "nbformat": 4, "nbformat_minor": 4 }