{
 "schema": "strim-headline/1",
 "comment": "Generated by `strim site build` from the final head-to-head summary; every number the charts and scoreboards show is here. Do not hand-edit. Schema: site/data/headline.schema.md.",
 "run": {
  "id": "h2h-final3-20261028",
  "date": "2026-10-28",
  "source": "bench/results/h2h-final3-20261028/summary.md",
  "raw_dir": "bench/results/h2h-final3-20261028",
  "machine": "Apple M5 Max, 64 GB, macOS 27.0.1",
  "model": "Qwen3.8-27B 4-bit + MTP head",
  "strim_build": "ee9f9d8",
  "versions": {
   "rapid-mlx": "rapid-mlx 0.15.7",
   "mlx-vlm": "mlx-vlm 0.7.10",
   "mlx-lm": "mlx-lm 0.32.2",
   "omlx": "oMLX 0.4.5.dev1"
  },
  "passes": 2,
  "reps_per_pass": 1,
  "tie_band_pct": 2,
  "validity": {
   "clean": false,
   "sessions": 30,
   "polluted": 2,
   "load_max": 4.68,
   "swap_growth_max_mb": 0,
   "text": "28 of 30 primary sessions were clean; 2 were marked polluted. Reruns are retained in the data. Claims use only cells marked claim eligible (highest primary-session 1-minute load 4.7)."
  }
 },
 "engines": {
  "strim": {
   "label": "Strim",
   "name": "Strim",
   "mtp": "yes (qwen38-mtp-4bit head)",
   "prefix_cache": "yes, default (STRIM_PREFIX_CACHE_MB 12288)",
   "flags": "serve models/qwen38-4bit models/qwen38-mtp-4bit --port 8203 --no-auth"
  },
  "rapid-mlx": {
   "label": "rapid-mlx (MTP)",
   "name": "rapid-mlx",
   "mtp": "yes (--speculative-config mtp, qwen38-mtp-4bit)",
   "prefix_cache": "yes, default on",
   "flags": "--no-telemetry serve models/qwen38-4bit --port 8651 --max-num-seqs 4 --speculative-config {mtp,qwen38-mtp-4bit}"
  },
  "mlx-vlm": {
   "label": "mlx-vlm (no MTP)",
   "name": "mlx-vlm",
   "mtp": "no",
   "prefix_cache": "no cross-request reuse (0% cached in replay)",
   "flags": "--model models/qwen38-4bit --port 8651 (server defaults)"
  },
  "mlx-lm": {
   "label": "mlx-lm (no MTP)",
   "name": "mlx-lm",
   "mtp": "no (no speculative decoding)",
   "prefix_cache": "yes, default in-memory LRU prompt cache",
   "flags": "--model models/qwen38-4bit --port 8654 --decode-concurrency 4 --prompt-concurrency 4"
  },
  "omlx": {
   "label": "oMLX (no MTP)",
   "name": "oMLX",
   "mtp": "no (MTP needs oMLX's own checkpoint; stock weights have no MTP head)",
   "prefix_cache": "yes, default paged SSD prefix cache under the scratch HOME (fresh per session, so no cross-session hits)",
   "flags": "serve --model-dir <scratch>/models (symlink to models/qwen38-4bit) --port 8653, HOME=<scratch>/home, defaults otherwise (memory-guard balanced, max-concurrent-requests 8)"
  }
 },
 "hero": {
  "rows": [
   {
    "regime": "decode",
    "cells": [
     "d4k",
     "d16k",
     "d32k"
    ]
   },
   {
    "regime": "prefill",
    "cells": [
     "c4k_pf",
     "c16k_pf",
     "c32k_pf"
    ]
   },
   {
    "regime": "shared",
    "cells": [
     "aw5",
     "p50_w5",
     "p95_w5",
     "rp8"
    ]
   },
   {
    "regime": "concurrency",
    "cells": [
     "a512c4",
     "a512c8",
     "aagc4",
     "aagc8",
     "a4kc4",
     "a4kc8"
    ]
   },
   {
    "regime": "latency",
    "cells": [
     "p50_512c4",
     "p95_512c4",
     "p50_512c8",
     "p95_512c8"
    ]
   }
  ]
 },
 "cells": [
  {
   "id": "c4k_pf",
   "regime": "prefill",
   "label": "cold 4K prefill",
   "short": "cold prefill 4K",
   "unit": "tok/s",
   "better": "higher",
   "values": {
    "strim": 994.5,
    "rapid-mlx": 996.8,
    "mlx-vlm": 1017.0,
    "mlx-lm": 949.1,
    "omlx": 896.7
   },
   "speedup": {
    "rapid-mlx": 0.998,
    "mlx-vlm": 0.978,
    "mlx-lm": 1.048,
    "omlx": 1.109
   },
   "verdict": {
    "rapid-mlx": "TIE",
    "mlx-vlm": "LOSS",
    "mlx-lm": "WIN",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row cold 4K prefill"
  },
  {
   "id": "c4k_tt",
   "regime": "prefill_ttft",
   "label": "cold 4K TTFT",
   "short": "cold 4K TTFT",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 4.13,
    "rapid-mlx": 4.12,
    "mlx-vlm": 4.039,
    "mlx-lm": 4.327,
    "omlx": 4.58
   },
   "speedup": {
    "rapid-mlx": 0.998,
    "mlx-vlm": 0.978,
    "mlx-lm": 1.048,
    "omlx": 1.109
   },
   "verdict": {
    "rapid-mlx": "TIE",
    "mlx-vlm": "LOSS",
    "mlx-lm": "WIN",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row cold 4K TTFT"
  },
  {
   "id": "d4k",
   "regime": "decode",
   "label": "decode 256 tok @ 4K ctx",
   "short": "decode @ 4K",
   "unit": "tok/s",
   "better": "higher",
   "values": {
    "strim": 75.34,
    "rapid-mlx": 50.12,
    "mlx-vlm": 32.69,
    "mlx-lm": 30.11,
    "omlx": 33.53
   },
   "speedup": {
    "rapid-mlx": 1.503,
    "mlx-vlm": 2.305,
    "mlx-lm": 2.502,
    "omlx": 2.247
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "WIN",
    "mlx-lm": "WIN",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row decode 256 tok @ 4K ctx"
  },
  {
   "id": "c16k_pf",
   "regime": "prefill",
   "label": "cold 16K prefill",
   "short": "cold prefill 16K",
   "unit": "tok/s",
   "better": "higher",
   "values": {
    "strim": 963.1,
    "rapid-mlx": 972.4,
    "mlx-vlm": 988.0,
    "mlx-lm": 919.8,
    "omlx": 891.9
   },
   "speedup": {
    "rapid-mlx": 0.99,
    "mlx-vlm": 0.975,
    "mlx-lm": 1.047,
    "omlx": 1.08
   },
   "verdict": {
    "rapid-mlx": "TIE",
    "mlx-vlm": "LOSS",
    "mlx-lm": "WIN",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row cold 16K prefill"
  },
  {
   "id": "c16k_tt",
   "regime": "prefill_ttft",
   "label": "cold 16K TTFT",
   "short": "cold 16K TTFT",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 17.02,
    "rapid-mlx": 16.86,
    "mlx-vlm": 16.59,
    "mlx-lm": 17.82,
    "omlx": 18.38
   },
   "speedup": {
    "rapid-mlx": 0.991,
    "mlx-vlm": 0.975,
    "mlx-lm": 1.047,
    "omlx": 1.08
   },
   "verdict": {
    "rapid-mlx": "TIE",
    "mlx-vlm": "LOSS",
    "mlx-lm": "WIN",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row cold 16K TTFT"
  },
  {
   "id": "d16k",
   "regime": "decode",
   "label": "decode 256 tok @ 16K ctx",
   "short": "decode @ 16K",
   "unit": "tok/s",
   "better": "higher",
   "values": {
    "strim": 65.18,
    "rapid-mlx": 45.75,
    "mlx-vlm": 30.93,
    "mlx-lm": 28.68,
    "omlx": 31.42
   },
   "speedup": {
    "rapid-mlx": 1.425,
    "mlx-vlm": 2.107,
    "mlx-lm": 2.273,
    "omlx": 2.074
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "WIN",
    "mlx-lm": "WIN",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row decode 256 tok @ 16K ctx"
  },
  {
   "id": "c32k_pf",
   "regime": "prefill",
   "label": "cold 32K prefill",
   "short": "cold prefill 32K",
   "unit": "tok/s",
   "better": "higher",
   "values": {
    "strim": 884.6,
    "rapid-mlx": 910.0,
    "mlx-vlm": 923.2,
    "mlx-lm": 850.8,
    "omlx": 823.7
   },
   "speedup": {
    "rapid-mlx": 0.972,
    "mlx-vlm": 0.958,
    "mlx-lm": 1.04,
    "omlx": 1.074
   },
   "verdict": {
    "rapid-mlx": "LOSS",
    "mlx-vlm": "LOSS",
    "mlx-lm": "WIN",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row cold 32K prefill"
  },
  {
   "id": "c32k_tt",
   "regime": "prefill_ttft",
   "label": "cold 32K TTFT",
   "short": "cold 32K TTFT",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 37.06,
    "rapid-mlx": 36.02,
    "mlx-vlm": 35.51,
    "mlx-lm": 38.53,
    "omlx": 39.79
   },
   "speedup": {
    "rapid-mlx": 0.972,
    "mlx-vlm": 0.958,
    "mlx-lm": 1.04,
    "omlx": 1.074
   },
   "verdict": {
    "rapid-mlx": "LOSS",
    "mlx-vlm": "LOSS",
    "mlx-lm": "WIN",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row cold 32K TTFT"
  },
  {
   "id": "d32k",
   "regime": "decode",
   "label": "decode 128 tok @ 32K ctx",
   "short": "decode @ 32K",
   "unit": "tok/s",
   "better": "higher",
   "values": {
    "strim": 52.17,
    "rapid-mlx": 42.17,
    "mlx-vlm": 29.35,
    "mlx-lm": 27.62,
    "omlx": 29.9
   },
   "speedup": {
    "rapid-mlx": 1.237,
    "mlx-vlm": 1.778,
    "mlx-lm": 1.889,
    "omlx": 1.745
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "WIN",
    "mlx-lm": "WIN",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row decode 128 tok @ 32K ctx"
  },
  {
   "id": "a512c4",
   "regime": "concurrency",
   "label": "c=4 512-tok prompts aggregate",
   "short": "c=4 512-tok: tok/s",
   "unit": "tok/s",
   "better": "higher",
   "values": {
    "strim": 69.22,
    "rapid-mlx": 71.69,
    "mlx-vlm": 67.29,
    "mlx-lm": 66.31,
    "omlx": 54.61
   },
   "speedup": {
    "rapid-mlx": 0.966,
    "mlx-vlm": 1.029,
    "mlx-lm": 1.044,
    "omlx": 1.268
   },
   "verdict": {
    "rapid-mlx": "LOSS",
    "mlx-vlm": "WIN",
    "mlx-lm": "WIN",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=4 512-tok prompts aggregate"
  },
  {
   "id": "p50_512c4",
   "regime": "latency",
   "label": "c=4 512-tok prompts p50 TTFT",
   "short": "c=4 512-tok: p50 TTFT",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 1.493,
    "rapid-mlx": 2.339,
    "mlx-vlm": 2.305,
    "mlx-lm": 2.467,
    "omlx": 3.049
   },
   "speedup": {
    "rapid-mlx": 1.567,
    "mlx-vlm": 1.544,
    "mlx-lm": 1.652,
    "omlx": 2.042
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "WIN",
    "mlx-lm": "WIN",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=4 512-tok prompts p50 TTFT"
  },
  {
   "id": "p95_512c4",
   "regime": "latency",
   "label": "c=4 512-tok prompts p95 TTFT",
   "short": "c=4 512-tok: p95 TTFT",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 2.292,
    "rapid-mlx": 2.339,
    "mlx-vlm": 2.305,
    "mlx-lm": 2.469,
    "omlx": 3.198
   },
   "speedup": {
    "rapid-mlx": 1.021,
    "mlx-vlm": 1.006,
    "mlx-lm": 1.077,
    "omlx": 1.395
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "TIE",
    "mlx-lm": "WIN",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=4 512-tok prompts p95 TTFT"
  },
  {
   "id": "a512c8",
   "regime": "concurrency",
   "label": "c=8 512-tok prompts aggregate",
   "short": "c=8 512-tok: tok/s",
   "unit": "tok/s",
   "better": "higher",
   "values": {
    "strim": 76.37,
    "rapid-mlx": 71.76,
    "mlx-vlm": 74.65,
    "mlx-lm": 67.9,
    "omlx": 58.74
   },
   "speedup": {
    "rapid-mlx": 1.064,
    "mlx-vlm": 1.023,
    "mlx-lm": 1.125,
    "omlx": 1.3
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "WIN",
    "mlx-lm": "WIN",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=8 512-tok prompts aggregate"
  },
  {
   "id": "p50_512c8",
   "regime": "latency",
   "label": "c=8 512-tok prompts p50 TTFT",
   "short": "c=8 512-tok: p50 TTFT",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 2.683,
    "rapid-mlx": 5.889,
    "mlx-vlm": 4.34,
    "mlx-lm": 6.064,
    "omlx": 5.726
   },
   "speedup": {
    "rapid-mlx": 2.195,
    "mlx-vlm": 1.618,
    "mlx-lm": 2.26,
    "omlx": 2.134
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "WIN",
    "mlx-lm": "WIN",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=8 512-tok prompts p50 TTFT"
  },
  {
   "id": "p95_512c8",
   "regime": "latency",
   "label": "c=8 512-tok prompts p95 TTFT",
   "short": "c=8 512-tok: p95 TTFT",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 4.61,
    "rapid-mlx": 9.466,
    "mlx-vlm": 4.341,
    "mlx-lm": 9.837,
    "omlx": 6.304
   },
   "speedup": {
    "rapid-mlx": 2.053,
    "mlx-vlm": 0.942,
    "mlx-lm": 2.134,
    "omlx": 1.367
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "LOSS",
    "mlx-lm": "WIN",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=8 512-tok prompts p95 TTFT"
  },
  {
   "id": "aagc4",
   "regime": "concurrency",
   "label": "c=4 agent prompts (~2.8K tok) aggregate",
   "short": "c=4 agent: tok/s",
   "unit": "tok/s",
   "better": "higher",
   "values": {
    "strim": 28.73,
    "rapid-mlx": 27.73,
    "mlx-vlm": 28.57,
    "mlx-lm": 27.22,
    "omlx": 25.28
   },
   "speedup": {
    "rapid-mlx": 1.036,
    "mlx-vlm": 1.006,
    "mlx-lm": 1.055,
    "omlx": 1.136
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "TIE",
    "mlx-lm": "WIN",
    "omlx": "NC"
   },
   "not_claimable": {
    "omlx": "oMLX: pass 2 excluded (engine_guard_abort, memory_abort_watchdog)"
   },
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=4 agent prompts (~2.8K tok) aggregate"
  },
  {
   "id": "p50_agc4",
   "regime": "latency",
   "label": "c=4 agent prompts p50 TTFT",
   "short": "c=4 agent: p50 TTFT",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 7.678,
    "rapid-mlx": 13.44,
    "mlx-vlm": 12.43,
    "mlx-lm": 13.47,
    "omlx": 13.32
   },
   "speedup": {
    "rapid-mlx": 1.75,
    "mlx-vlm": 1.619,
    "mlx-lm": 1.754,
    "omlx": 1.735
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "WIN",
    "mlx-lm": "WIN",
    "omlx": "NC"
   },
   "not_claimable": {
    "omlx": "oMLX: pass 2 excluded (engine_guard_abort, memory_abort_watchdog)"
   },
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=4 agent prompts p50 TTFT"
  },
  {
   "id": "p95_agc4",
   "regime": "latency",
   "label": "c=4 agent prompts p95 TTFT",
   "short": "c=4 agent: p95 TTFT",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 12.07,
    "rapid-mlx": 13.44,
    "mlx-vlm": 12.43,
    "mlx-lm": 13.47,
    "omlx": 13.47
   },
   "speedup": {
    "rapid-mlx": 1.114,
    "mlx-vlm": 1.03,
    "mlx-lm": 1.116,
    "omlx": 1.116
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "WIN",
    "mlx-lm": "WIN",
    "omlx": "NC"
   },
   "not_claimable": {
    "omlx": "oMLX: pass 2 excluded (engine_guard_abort, memory_abort_watchdog)"
   },
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=4 agent prompts p95 TTFT"
  },
  {
   "id": "aagc8",
   "regime": "concurrency",
   "label": "c=8 agent prompts aggregate",
   "short": "c=8 agent: tok/s",
   "unit": "tok/s",
   "better": "higher",
   "values": {
    "strim": 31.19,
    "rapid-mlx": 28.47,
    "mlx-vlm": 31.18,
    "mlx-lm": 27.63,
    "omlx": 26.89
   },
   "speedup": {
    "rapid-mlx": 1.096,
    "mlx-vlm": 1.0,
    "mlx-lm": 1.129,
    "omlx": 1.16
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "TIE",
    "mlx-lm": "WIN",
    "omlx": "NC"
   },
   "not_claimable": {
    "omlx": "oMLX: pass 2 excluded (engine_guard_abort, memory_abort_watchdog)"
   },
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=8 agent prompts aggregate"
  },
  {
   "id": "p50_agc8",
   "regime": "latency",
   "label": "c=8 agent prompts p50 TTFT",
   "short": "c=8 agent: p50 TTFT",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 13.34,
    "rapid-mlx": 22.1,
    "mlx-vlm": 23.1,
    "mlx-lm": 22.61,
    "omlx": 25.08
   },
   "speedup": {
    "rapid-mlx": 1.657,
    "mlx-vlm": 1.732,
    "mlx-lm": 1.695,
    "omlx": 1.88
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "WIN",
    "mlx-lm": "WIN",
    "omlx": "NC"
   },
   "not_claimable": {
    "omlx": "oMLX: pass 2 excluded (engine_guard_abort, memory_abort_watchdog)"
   },
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=8 agent prompts p50 TTFT"
  },
  {
   "id": "p95_agc8",
   "regime": "latency",
   "label": "c=8 agent prompts p95 TTFT",
   "short": "c=8 agent: p95 TTFT",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 22.67,
    "rapid-mlx": 30.98,
    "mlx-vlm": 23.1,
    "mlx-lm": 31.68,
    "omlx": 25.69
   },
   "speedup": {
    "rapid-mlx": 1.367,
    "mlx-vlm": 1.019,
    "mlx-lm": 1.397,
    "omlx": 1.133
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "TIE",
    "mlx-lm": "WIN",
    "omlx": "NC"
   },
   "not_claimable": {
    "omlx": "oMLX: pass 2 excluded (engine_guard_abort, memory_abort_watchdog)"
   },
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=8 agent prompts p95 TTFT"
  },
  {
   "id": "a4kc4",
   "regime": "concurrency",
   "label": "c=4 4K prompts aggregate (prefill-bound)",
   "short": "c=4 4K: tok/s",
   "unit": "tok/s",
   "better": "higher",
   "values": {
    "strim": 13.24,
    "rapid-mlx": 13.16,
    "mlx-vlm": 13.52,
    "mlx-lm": 13.1
   },
   "speedup": {
    "rapid-mlx": 1.006,
    "mlx-vlm": 0.979,
    "mlx-lm": 1.011
   },
   "verdict": {
    "rapid-mlx": "TIE",
    "mlx-vlm": "LOSS",
    "mlx-lm": "TIE",
    "omlx": "n/a"
   },
   "not_claimable": {
    "omlx": "oMLX: no valid value, pass 1 excluded (engine_guard_abort, memory_abort_watchdog), pass 2 excluded (engine_guard_abort, memory_abort_watchdog)"
   },
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=4 4K prompts aggregate (prefill-bound)"
  },
  {
   "id": "p50_4kc4",
   "regime": "latency",
   "label": "c=4 4K prompts p50 TTFT",
   "short": "c=4 4K: p50 TTFT",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 10.36,
    "rapid-mlx": 16.86,
    "mlx-vlm": 16.17,
    "mlx-lm": 16.84
   },
   "speedup": {
    "rapid-mlx": 1.627,
    "mlx-vlm": 1.561,
    "mlx-lm": 1.625
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "WIN",
    "mlx-lm": "WIN",
    "omlx": "n/a"
   },
   "not_claimable": {
    "omlx": "oMLX: no valid value, pass 1 excluded (engine_guard_abort, memory_abort_watchdog), pass 2 excluded (engine_guard_abort, memory_abort_watchdog)"
   },
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=4 4K prompts p50 TTFT"
  },
  {
   "id": "p95_4kc4",
   "regime": "latency",
   "label": "c=4 4K prompts p95 TTFT",
   "short": "c=4 4K: p95 TTFT",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 16.0,
    "rapid-mlx": 16.86,
    "mlx-vlm": 16.17,
    "mlx-lm": 16.85
   },
   "speedup": {
    "rapid-mlx": 1.054,
    "mlx-vlm": 1.011,
    "mlx-lm": 1.053
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "TIE",
    "mlx-lm": "WIN",
    "omlx": "n/a"
   },
   "not_claimable": {
    "omlx": "oMLX: no valid value, pass 1 excluded (engine_guard_abort, memory_abort_watchdog), pass 2 excluded (engine_guard_abort, memory_abort_watchdog)"
   },
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=4 4K prompts p95 TTFT"
  },
  {
   "id": "a4kc8",
   "regime": "concurrency",
   "label": "c=8 4K prompts aggregate (prefill-bound)",
   "short": "c=8 4K: tok/s",
   "unit": "tok/s",
   "better": "higher",
   "values": {
    "strim": 13.29,
    "rapid-mlx": 13.2,
    "mlx-vlm": 13.84,
    "mlx-lm": 13.04
   },
   "speedup": {
    "rapid-mlx": 1.007,
    "mlx-vlm": 0.96,
    "mlx-lm": 1.019
   },
   "verdict": {
    "rapid-mlx": "TIE",
    "mlx-vlm": "LOSS",
    "mlx-lm": "TIE",
    "omlx": "n/a"
   },
   "not_claimable": {
    "omlx": "oMLX: no valid value, pass 1 excluded (engine_guard_abort, memory_abort_watchdog), pass 2 excluded (engine_guard_abort, memory_abort_watchdog)"
   },
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=8 4K prompts aggregate (prefill-bound)"
  },
  {
   "id": "p50_4kc8",
   "regime": "latency",
   "label": "c=8 4K prompts p50 TTFT",
   "short": "c=8 4K: p50 TTFT",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 18.75,
    "rapid-mlx": 26.56,
    "mlx-vlm": 32.07,
    "mlx-lm": 26.83
   },
   "speedup": {
    "rapid-mlx": 1.417,
    "mlx-vlm": 1.71,
    "mlx-lm": 1.431
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "WIN",
    "mlx-lm": "WIN",
    "omlx": "n/a"
   },
   "not_claimable": {
    "omlx": "oMLX: no valid value, pass 1 excluded (engine_guard_abort, memory_abort_watchdog), pass 2 excluded (engine_guard_abort, memory_abort_watchdog)"
   },
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=8 4K prompts p50 TTFT"
  },
  {
   "id": "p95_4kc8",
   "regime": "latency",
   "label": "c=8 4K prompts p95 TTFT",
   "short": "c=8 4K: p95 TTFT",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 32.0,
    "rapid-mlx": 36.24,
    "mlx-vlm": 32.07,
    "mlx-lm": 36.57
   },
   "speedup": {
    "rapid-mlx": 1.133,
    "mlx-vlm": 1.002,
    "mlx-lm": 1.143
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "TIE",
    "mlx-lm": "WIN",
    "omlx": "n/a"
   },
   "not_claimable": {
    "omlx": "oMLX: no valid value, pass 1 excluded (engine_guard_abort, memory_abort_watchdog), pass 2 excluded (engine_guard_abort, memory_abort_watchdog)"
   },
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row c=8 4K prompts p95 TTFT"
  },
  {
   "id": "aw5",
   "regime": "shared",
   "label": "W5 4 clients, 16K shared prefix aggregate",
   "short": "4 agents, 16K prefix: tok/s",
   "unit": "tok/s",
   "better": "higher",
   "values": {
    "strim": 9.773,
    "rapid-mlx": 3.344,
    "mlx-vlm": 3.473,
    "mlx-lm": 3.346
   },
   "speedup": {
    "rapid-mlx": 2.923,
    "mlx-vlm": 2.814,
    "mlx-lm": 2.921
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "WIN",
    "mlx-lm": "WIN",
    "omlx": "n/a"
   },
   "not_claimable": {
    "omlx": "oMLX: no valid value, pass 1 excluded (engine_guard_abort, memory_abort_watchdog), pass 2 excluded (engine_guard_abort, memory_abort_watchdog)"
   },
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row W5 4 clients, 16K shared prefix aggregate"
  },
  {
   "id": "p50_w5",
   "regime": "shared",
   "label": "W5 p50 TTFT",
   "short": "4 agents: p50 TTFT",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 20.29,
    "rapid-mlx": 73.53,
    "mlx-vlm": 70.48,
    "mlx-lm": 73.4
   },
   "speedup": {
    "rapid-mlx": 3.624,
    "mlx-vlm": 3.474,
    "mlx-lm": 3.618
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "WIN",
    "mlx-lm": "WIN",
    "omlx": "n/a"
   },
   "not_claimable": {
    "omlx": "oMLX: no valid value, pass 1 excluded (engine_guard_abort, memory_abort_watchdog), pass 2 excluded (engine_guard_abort, memory_abort_watchdog)"
   },
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row W5 p50 TTFT"
  },
  {
   "id": "p95_w5",
   "regime": "shared",
   "label": "W5 p95 TTFT",
   "short": "4 agents: p95 TTFT",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 22.83,
    "rapid-mlx": 73.53,
    "mlx-vlm": 70.48,
    "mlx-lm": 73.4
   },
   "speedup": {
    "rapid-mlx": 3.221,
    "mlx-vlm": 3.087,
    "mlx-lm": 3.215
   },
   "verdict": {
    "rapid-mlx": "WIN",
    "mlx-vlm": "WIN",
    "mlx-lm": "WIN",
    "omlx": "n/a"
   },
   "not_claimable": {
    "omlx": "oMLX: no valid value, pass 1 excluded (engine_guard_abort, memory_abort_watchdog), pass 2 excluded (engine_guard_abort, memory_abort_watchdog)"
   },
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row W5 p95 TTFT"
  },
  {
   "id": "rp8",
   "regime": "shared",
   "label": "agent replay prefill sum T1-T8",
   "short": "8-turn replay prefill",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 48.93,
    "rapid-mlx": 49.13,
    "mlx-vlm": 192.5,
    "mlx-lm": 55.02,
    "omlx": 64.84
   },
   "speedup": {
    "rapid-mlx": 1.004,
    "mlx-vlm": 3.934,
    "mlx-lm": 1.124,
    "omlx": 1.325
   },
   "verdict": {
    "rapid-mlx": "TIE",
    "mlx-vlm": "WIN",
    "mlx-lm": "WIN",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row agent replay prefill sum T1-T8"
  },
  {
   "id": "rp6",
   "regime": "other",
   "label": "agent replay prefill sum T1-T6",
   "short": "agent replay prefill sum T1-T6",
   "unit": "s",
   "better": "lower",
   "values": {
    "strim": 34.6,
    "rapid-mlx": 34.77,
    "mlx-vlm": 107.7,
    "mlx-lm": 39.31,
    "omlx": 44.9
   },
   "speedup": {
    "rapid-mlx": 1.005,
    "mlx-vlm": 3.113,
    "mlx-lm": 1.136,
    "omlx": 1.298
   },
   "verdict": {
    "rapid-mlx": "TIE",
    "mlx-vlm": "WIN",
    "mlx-lm": "WIN",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row agent replay prefill sum T1-T6"
  },
  {
   "id": "rpc8",
   "regime": "other",
   "label": "agent replay T8 cached fraction",
   "short": "agent replay T8 cached fraction",
   "unit": "%",
   "better": "higher",
   "values": {
    "strim": 87.25,
    "rapid-mlx": 87.23,
    "mlx-vlm": 0.0,
    "mlx-lm": 87.26,
    "omlx": 85.25
   },
   "speedup": {
    "rapid-mlx": 1.0,
    "mlx-lm": 1.0,
    "omlx": 1.023
   },
   "verdict": {
    "rapid-mlx": "TIE",
    "mlx-vlm": "n/a",
    "mlx-lm": "TIE",
    "omlx": "WIN"
   },
   "not_claimable": {},
   "source": "bench/results/h2h-final3-20261028/summary.md section Results, row agent replay T8 cached fraction"
  }
 ],
 "memory": {
  "unit": "GB",
  "source": "bench/results/h2h-final3-20261028/summary.md section Peak memory per cell (phys_footprint of the engine process tree, max over passes and sessions)",
  "abort_limit_gb": null,
  "whole_run": {
   "strim": 30.8,
   "rapid-mlx": 35.9,
   "mlx-vlm": 36.3,
   "mlx-lm": 37.5,
   "omlx": 55.2
  },
  "whole_run_verdict": {
   "rapid-mlx": "WIN",
   "mlx-vlm": "WIN",
   "mlx-lm": "WIN",
   "omlx": "WIN"
  },
  "strim_lowest_cells": 2,
  "cells_total": 9,
  "cells": [
   {
    "id": "warmup",
    "label": "warmup",
    "values": {
     "strim": 18.4,
     "rapid-mlx": 20.1,
     "mlx-vlm": 19.3,
     "mlx-lm": 17.6,
     "omlx": 18.8
    }
   },
   {
    "id": "agree",
    "label": "greedy agreement",
    "values": {
     "strim": 17.5,
     "rapid-mlx": 17.4,
     "mlx-vlm": 16.9,
     "mlx-lm": 17.8,
     "omlx": 19.7
    }
   },
   {
    "id": "w3",
    "label": "cold 4K and 16K",
    "values": {
     "strim": 23.3,
     "rapid-mlx": 22.1,
     "mlx-vlm": 20.5,
     "mlx-lm": 21.7,
     "omlx": 24.7
    }
   },
   {
    "id": "w3_32k",
    "label": "cold 32K",
    "values": {
     "strim": 28.7,
     "rapid-mlx": 24.8,
     "mlx-vlm": 23.5,
     "mlx-lm": 27.0,
     "omlx": 29.0
    }
   },
   {
    "id": "w4",
    "label": "c=4/8, 4K prompts",
    "values": {
     "strim": 30.0,
     "rapid-mlx": 31.3,
     "mlx-vlm": 36.3,
     "mlx-lm": 34.4,
     "omlx": 49.5
    }
   },
   {
    "id": "w4s",
    "label": "c=4/8, 512-token prompts",
    "values": {
     "strim": 26.0,
     "rapid-mlx": 20.4,
     "mlx-vlm": 24.4,
     "mlx-lm": 23.9,
     "omlx": 28.1
    }
   },
   {
    "id": "w6",
    "label": "c=4/8, agent prompts",
    "values": {
     "strim": 29.5,
     "rapid-mlx": 26.6,
     "mlx-vlm": 33.8,
     "mlx-lm": 31.5,
     "omlx": 43.7
    }
   },
   {
    "id": "w5",
    "label": "4 agents, 16K shared prefix",
    "values": {
     "strim": 30.8,
     "rapid-mlx": 33.5,
     "mlx-vlm": 30.2,
     "mlx-lm": 31.4,
     "omlx": 40.5
    }
   },
   {
    "id": "replay",
    "label": "8-turn replay",
    "values": {
     "strim": 27.5,
     "rapid-mlx": 35.9,
     "mlx-vlm": 28.6,
     "mlx-lm": 37.5,
     "omlx": 36.9
    }
   }
  ]
 },
 "scoreboard": {
  "regimes": [
   {
    "id": "decode",
    "label": "Decode",
    "metric": "tok/s at 4K, 16K, 32K context",
    "cells": [
     "d4k",
     "d16k",
     "d32k"
    ],
    "vs": {
     "rapid-mlx": {
      "WIN": 3,
      "TIE": 0,
      "LOSS": 0,
      "NC": 0
     },
     "mlx-vlm": {
      "WIN": 3,
      "TIE": 0,
      "LOSS": 0,
      "NC": 0
     },
     "mlx-lm": {
      "WIN": 3,
      "TIE": 0,
      "LOSS": 0,
      "NC": 0
     },
     "omlx": {
      "WIN": 3,
      "TIE": 0,
      "LOSS": 0,
      "NC": 0
     }
    }
   },
   {
    "id": "prefill",
    "label": "Cold prefill",
    "metric": "tok/s on a cold 4K, 16K, 32K prompt",
    "cells": [
     "c4k_pf",
     "c16k_pf",
     "c32k_pf"
    ],
    "vs": {
     "rapid-mlx": {
      "WIN": 0,
      "TIE": 2,
      "LOSS": 1,
      "NC": 0
     },
     "mlx-vlm": {
      "WIN": 0,
      "TIE": 0,
      "LOSS": 3,
      "NC": 0
     },
     "mlx-lm": {
      "WIN": 3,
      "TIE": 0,
      "LOSS": 0,
      "NC": 0
     },
     "omlx": {
      "WIN": 3,
      "TIE": 0,
      "LOSS": 0,
      "NC": 0
     }
    }
   },
   {
    "id": "shared",
    "label": "Shared prefix",
    "metric": "4 agents on a 16K prefix (aggregate, p50, p95) and 8-turn replay",
    "cells": [
     "aw5",
     "p50_w5",
     "p95_w5",
     "rp8"
    ],
    "vs": {
     "rapid-mlx": {
      "WIN": 3,
      "TIE": 1,
      "LOSS": 0,
      "NC": 0
     },
     "mlx-vlm": {
      "WIN": 4,
      "TIE": 0,
      "LOSS": 0,
      "NC": 0
     },
     "mlx-lm": {
      "WIN": 4,
      "TIE": 0,
      "LOSS": 0,
      "NC": 0
     },
     "omlx": {
      "WIN": 1,
      "TIE": 0,
      "LOSS": 0,
      "NC": 0
     }
    }
   },
   {
    "id": "concurrency",
    "label": "Concurrency",
    "metric": "aggregate tok/s at c=4 and c=8, three prompt sizes",
    "cells": [
     "a512c4",
     "a512c8",
     "aagc4",
     "aagc8",
     "a4kc4",
     "a4kc8"
    ],
    "vs": {
     "rapid-mlx": {
      "WIN": 3,
      "TIE": 2,
      "LOSS": 1,
      "NC": 0
     },
     "mlx-vlm": {
      "WIN": 2,
      "TIE": 2,
      "LOSS": 2,
      "NC": 0
     },
     "mlx-lm": {
      "WIN": 4,
      "TIE": 2,
      "LOSS": 0,
      "NC": 0
     },
     "omlx": {
      "WIN": 2,
      "TIE": 0,
      "LOSS": 0,
      "NC": 2
     }
    }
   },
   {
    "id": "latency",
    "label": "Latency",
    "metric": "TTFT p50 and p95 at c=4 and c=8, three prompt sizes",
    "cells": [
     "p50_512c4",
     "p95_512c4",
     "p50_512c8",
     "p95_512c8",
     "p50_agc4",
     "p95_agc4",
     "p50_agc8",
     "p95_agc8",
     "p50_4kc4",
     "p95_4kc4",
     "p50_4kc8",
     "p95_4kc8"
    ],
    "vs": {
     "rapid-mlx": {
      "WIN": 12,
      "TIE": 0,
      "LOSS": 0,
      "NC": 0
     },
     "mlx-vlm": {
      "WIN": 7,
      "TIE": 4,
      "LOSS": 1,
      "NC": 0
     },
     "mlx-lm": {
      "WIN": 12,
      "TIE": 0,
      "LOSS": 0,
      "NC": 0
     },
     "omlx": {
      "WIN": 4,
      "TIE": 0,
      "LOSS": 0,
      "NC": 4
     }
    }
   },
   {
    "id": "memory",
    "label": "Memory",
    "metric": "peak footprint over the whole run",
    "cells": [
     "whole_run"
    ],
    "vs": {
     "rapid-mlx": {
      "WIN": 1,
      "TIE": 0,
      "LOSS": 0
     },
     "mlx-vlm": {
      "WIN": 1,
      "TIE": 0,
      "LOSS": 0
     },
     "mlx-lm": {
      "WIN": 1,
      "TIE": 0,
      "LOSS": 0
     },
     "omlx": {
      "WIN": 1,
      "TIE": 0,
      "LOSS": 0
     }
    }
   }
  ],
  "total": {
   "rapid-mlx": {
    "WIN": 22,
    "TIE": 5,
    "LOSS": 2,
    "NC": 0
   },
   "mlx-vlm": {
    "WIN": 17,
    "TIE": 6,
    "LOSS": 6,
    "NC": 0
   },
   "mlx-lm": {
    "WIN": 27,
    "TIE": 2,
    "LOSS": 0,
    "NC": 0
   },
   "omlx": {
    "WIN": 14,
    "TIE": 0,
    "LOSS": 0,
    "NC": 6
   }
  }
 }
}