{
 "schema": "local-ai-bench/result/v1",
 "run_id": "20261011T194914Z-sglang-qwen3.8-27b-registry-208k-rtx-3090-24gb-gpu2-quality_eval",
 "ticket": "T48",
 "kind": "quality_eval",
 "model": "qwen3.8-27b",
 "config_id": "sglang-qwen3.8-27b-registry-208k-rtx-3090-24gb-gpu2",
 "outcome": "pass",
 "failed_stage": null,
 "exit_status": 0,
 "started_at": "2026-10-11T19:49:14.858942+00:00",
 "finished_at": "2026-10-12T01:47:39.736310+00:00",
 "launch": {
  "file": "launches/sglang-qwen3.8-27b-registry-208k-rtx-3090-24gb.json",
  "sha256": "11948b11dab392d837fae6ef37fb0fcd96c03ec96372fc16afa89ce2505a35cf",
  "image": "ghcr.io/0xsero/sglang-exl3@sha256:c8922bd7256caf1b273af7ac49221e652be9e6072c1cdec4b959f506fd415d79",
  "argv": [
   "/opt/entrypoint.sh",
   "python3",
   "-m",
   "sglang.launch_server",
   "--model-path",
   "/models/turboderp-Qwen3.8-27B-exl3-3.00bpw",
   "--quantization",
   "exl3",
   "--trust-remote-code",
   "--host",
   "0.0.0.0",
   "--port",
   "30000",
   "--served-model-name",
   "turboderp-Qwen3.8-27B-exl3-3.00bpw",
   "--context-length",
   "212992",
   "--mem-fraction-static",
   "0.82",
   "--kv-cache-dtype",
   "fp8_e4m3",
   "--mamba-ssm-dtype",
   "bfloat16",
   "--max-running-requests",
   "1",
   "--max-mamba-cache-size",
   "3",
   "--cuda-graph-max-bs-decode",
   "1",
   "--cuda-graph-max-bs-prefill",
   "240",
   "--chunked-prefill-size",
   "4096",
   "--max-prefill-tokens",
   "4096",
   "--prefill-decode-interval",
   "16",
   "--reasoning-parser",
   "qwen3",
   "--tool-call-parser",
   "qwen3_coder",
   "--mamba-radix-cache-strategy",
   "no_buffer",
   "--disable-overlap-schedule",
   "--image-processor-backend",
   "pil",
   "--speculative-algorithm",
   "NEXTN",
   "--speculative-num-steps",
   "3",
   "--speculative-eagle-topk",
   "1",
   "--speculative-num-draft-tokens",
   "4",
   "--speculative-token-map",
   "/opt/sglang-exl3/tokenmaps/qwen38_hot32k_v2.pt"
  ],
  "env": {
   "CUDA_DEVICE_ORDER": "PCI_BUS_ID",
   "HF_HUB_OFFLINE": "1",
   "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True",
   "SGLANG_EXL3_EMBED_HOST": "1",
   "SGLANG_EXL3_PREFILL_FP16_ACC": "1",
   "SGLANG_EXL3_VIT_MLP_CHUNK": "8192",
   "SGLANG_EXL3_VIT_SDPA": "1",
   "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1",
   "SGLANG_VLM_CACHE_SIZE_MB": "256"
  },
  "engine": "sglang-exl3",
  "engine_source_commit": null,
  "mtp": {
   "algorithm": "NEXTN",
   "num_steps": 3,
   "eagle_topk": 1,
   "draft_tokens": 4,
   "token_map": "qwen38_hot32k_v2.pt"
  },
  "entrypoint": "/opt/entrypoint.sh",
  "port": 30000,
  "shm": "16g",
  "flags": [],
  "weights_at": [
   "/models"
  ]
 },
 "weights": [
  {
   "repo": "turboderp/Qwen3.8-27B-exl3",
   "revision": "6fe61ad620abfe97c5b49f9722c2bceeea4ccc28",
   "manifest": "models/turboderp-Qwen3.8-27B-exl3-3.00bpw-src.json"
  }
 ],
 "host": {
  "hostname": "omarchy-gpu",
  "cpu": "AMD Ryzen 9 5950X 16-Core Processor",
  "ram_total_gib": 125.7,
  "ram_layout": "DIMM_A1 32 GiB DDR4 3200 MT/s, DIMM_A2 32 GiB DDR4 3200 MT/s, DIMM_B1 32 GiB DDR4 3200 MT/s, DIMM_B2 32 GiB DDR4 3200 MT/s (AM4: 2 channels)",
  "kernel": "7.2.5-3-omarchy",
  "driver": "610.57.04",
  "cuda": "13.3",
  "cpus_online": "0-1,3-17,19-31",
  "cpus_offline": "2,18",
  "cpu_boost": false,
  "cpu_max_mhz": 3401,
  "machine_checks_this_boot": 6
 },
 "gpus": [
  {
   "index": 0,
   "uuid": "GPU-816b43e4-8d65-dfd7-2234-8517b3dfbf2d",
   "name": "NVIDIA GeForce RTX 3090",
   "bus_id": "00000000:04:00.0",
   "pcie_gen_max": 4,
   "pcie_width": 8,
   "display_active": false,
   "used_by_run": false
  },
  {
   "index": 1,
   "uuid": "GPU-c67ac872-3371-a88c-aa92-d969daf6405e",
   "name": "NVIDIA GeForce RTX 3090",
   "bus_id": "00000000:0B:00.0",
   "pcie_gen_max": 4,
   "pcie_width": 8,
   "display_active": true,
   "used_by_run": false
  },
  {
   "index": 2,
   "uuid": "GPU-76d3c6af-7f18-69d2-3145-899103de1722",
   "name": "NVIDIA GeForce RTX 3090",
   "bus_id": "00000000:0C:00.0",
   "pcie_gen_max": 4,
   "pcie_width": 8,
   "display_active": false,
   "used_by_run": true
  }
 ],
 "layout": {
  "name": "current",
  "cards": 1,
  "display": "on"
 },
 "context": {
  "configured": 212992,
  "occupied_max": {
   "value": 99327,
   "unit": "tokens"
  },
  "headroom_min": {
   "value": 109482,
   "unit": "tokens",
   "detail": {
    "configured": 212992,
    "configured_source": "env CONFIGURED_CTX"
   }
  }
 },
 "cache_state": {
  "cold": false,
  "reset": []
 },
 "provenance": {
  "bench_commit": "7c4216a051bd1d711db4765e3f24c8e0b48da6d7",
  "bench_dirty": true,
  "pins_sha256": "858564b6e07125ac48c2ef6c5178d0a53596f21eb5f0ebda43906472b01b4d78",
  "registry_commit": "d21258dd744e7c78be28177c90060c6af8e10b7e",
  "kit_commit": "ef883d269e50ecbea290f919f095e2f3ca633b42",
  "harness_commit": "04d809ceab9df28f9adaed044884180159172930"
 },
 "metrics": {
  "tasks_attempted": {
   "value": 50,
   "unit": "tasks",
   "detail": {
    "task_ids": [
     "bramstroker__homeassistant-powercalc-4795",
     "pandas-dev__pandas-68003",
     "UKGovernmentBEIS__inspect_ai-5264",
     "apache__airflow-72774",
     "UKGovernmentBEIS__inspect_ai-5190",
     "zauberzeug__nicegui-6380",
     "WeblateOrg__weblate-21643",
     "nilearn__nilearn-6608",
     "marimo-team__marimo-10788",
     "mesa__mesa-3892",
     "pytorch__rl-4210",
     "more-itertools__more-itertools-1248",
     "datalayer__jupyter-mcp-server-427",
     "kyegomez__swarms-2374",
     "domokane__FinancePy-317",
     "arc53__DocsGPT-2772",
     "optiland__optiland-796",
     "locustio__locust-3523",
     "docling-project__docling-4318",
     "pallets__werkzeug-3298",
     "pipeshub-ai__pipeshub-ai-3244",
     "holoviz__holoviews-7058",
     "supabase__supabase-py-1619",
     "agno-agi__agno-9887",
     "openai__openai-agents-python-5016",
     "marin-community__marin-9450",
     "google-deepmind__mujoco_warp-1660",
     "LearningCircuit__local-deep-research-6226",
     "ansible-collections__community.general-12804",
     "embeddings-benchmark__mteb-5443",
     "nautobot__nautobot-9531",
     "github__spec-kit-4559",
     "Genesis-Embodied-AI__genesis-world-3399",
     "fossasia__eventyay-5507",
     "dstackai__dstack-4256",
     "ogx-ai__ogx-6725",
     "marin-community__marin-9419",
     "pymc-labs__pymc-marketing-3099",
     "fsspec__filesystem_spec-2131",
     "stickerdaniel__linkedin-mcp-server-973",
     "ansible__ansible-87624",
     "scipy__scipy-26156",
     "roboflow__rf-detr-1502",
     "scrapy__scrapy-8138",
     "WeblateOrg__weblate-21746",
     "wkentaro__labelme-2667",
     "gdsfactory__gdsfactory-4840",
     "cognizant-ai-lab__neuro-san-studio-1452",
     "OpenHands__software-agent-sdk-4871",
     "roboflow__supervision-2600"
    ],
    "task_set_sha256": "b28ea453590fb21d8d9375c95519cae00599595b3d5ca85a011d872db16b560b",
    "sampling_sha256": "fdcbc89037b763c925766cce107e85ae1097c3456888ffa25108f2380b940b22",
    "swebench": "5.0.2"
   }
  },
  "tasks_solved": {
   "value": 39,
   "unit": "tasks",
   "detail": {
    "eval_errors": 1
   }
  },
  "solve_rate": {
   "value": 0.78,
   "unit": "fraction"
  },
  "token_budget_per_task": {
   "value": 60000,
   "unit": "completion tokens"
  },
  "per_task": {
   "value": [
    {
     "instance_id": "bramstroker__homeassistant-powercalc-4795",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 29667,
     "budget_exhausted": false,
     "seconds": 508.4,
     "requests": 61,
     "tool_calls": 83,
     "occupied_max": 68863
    },
    {
     "instance_id": "pandas-dev__pandas-68003",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": false,
     "completion_tokens": 12864,
     "budget_exhausted": false,
     "seconds": 260.1,
     "requests": 28,
     "tool_calls": 37,
     "occupied_max": 32478
    },
    {
     "instance_id": "UKGovernmentBEIS__inspect_ai-5264",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 22223,
     "budget_exhausted": false,
     "seconds": 507.7,
     "requests": 76,
     "tool_calls": 83,
     "occupied_max": 54810
    },
    {
     "instance_id": "apache__airflow-72774",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 7718,
     "budget_exhausted": false,
     "seconds": 74.2,
     "requests": 16,
     "tool_calls": 23,
     "occupied_max": 17777
    },
    {
     "instance_id": "UKGovernmentBEIS__inspect_ai-5190",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 9397,
     "budget_exhausted": false,
     "seconds": 193.1,
     "requests": 32,
     "tool_calls": 40,
     "occupied_max": 28500
    },
    {
     "instance_id": "zauberzeug__nicegui-6380",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 49962,
     "budget_exhausted": false,
     "seconds": 641.9,
     "requests": 56,
     "tool_calls": 61,
     "occupied_max": 71289
    },
    {
     "instance_id": "WeblateOrg__weblate-21643",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 17446,
     "budget_exhausted": false,
     "seconds": 438.2,
     "requests": 49,
     "tool_calls": 56,
     "occupied_max": 36321
    },
    {
     "instance_id": "nilearn__nilearn-6608",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 32018,
     "budget_exhausted": false,
     "seconds": 358.4,
     "requests": 39,
     "tool_calls": 49,
     "occupied_max": 49496
    },
    {
     "instance_id": "marimo-team__marimo-10788",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 19250,
     "budget_exhausted": false,
     "seconds": 220.9,
     "requests": 34,
     "tool_calls": 48,
     "occupied_max": 43629
    },
    {
     "instance_id": "mesa__mesa-3892",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 15718,
     "budget_exhausted": false,
     "seconds": 177.1,
     "requests": 26,
     "tool_calls": 34,
     "occupied_max": 39389
    },
    {
     "instance_id": "pytorch__rl-4210",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 12790,
     "budget_exhausted": false,
     "seconds": 169.4,
     "requests": 21,
     "tool_calls": 22,
     "occupied_max": 22905
    },
    {
     "instance_id": "more-itertools__more-itertools-1248",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 44893,
     "budget_exhausted": false,
     "seconds": 553.5,
     "requests": 53,
     "tool_calls": 60,
     "occupied_max": 68495
    },
    {
     "instance_id": "datalayer__jupyter-mcp-server-427",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 16488,
     "budget_exhausted": false,
     "seconds": 214.2,
     "requests": 22,
     "tool_calls": 32,
     "occupied_max": 30899
    },
    {
     "instance_id": "kyegomez__swarms-2374",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 13996,
     "budget_exhausted": false,
     "seconds": 226.0,
     "requests": 36,
     "tool_calls": 43,
     "occupied_max": 29618
    },
    {
     "instance_id": "domokane__FinancePy-317",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": null,
     "completion_tokens": 12596,
     "budget_exhausted": false,
     "seconds": 169.1,
     "requests": 28,
     "tool_calls": 33,
     "occupied_max": 31333
    },
    {
     "instance_id": "arc53__DocsGPT-2772",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 18143,
     "budget_exhausted": false,
     "seconds": 258.2,
     "requests": 17,
     "tool_calls": 21,
     "occupied_max": 19196
    },
    {
     "instance_id": "optiland__optiland-796",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 14547,
     "budget_exhausted": false,
     "seconds": 331.5,
     "requests": 36,
     "tool_calls": 48,
     "occupied_max": 36517
    },
    {
     "instance_id": "locustio__locust-3523",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 6212,
     "budget_exhausted": false,
     "seconds": 105.4,
     "requests": 19,
     "tool_calls": 25,
     "occupied_max": 19479
    },
    {
     "instance_id": "docling-project__docling-4318",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": false,
     "completion_tokens": 42365,
     "budget_exhausted": false,
     "seconds": 461.9,
     "requests": 39,
     "tool_calls": 53,
     "occupied_max": 62092
    },
    {
     "instance_id": "pallets__werkzeug-3298",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 12636,
     "budget_exhausted": false,
     "seconds": 125.4,
     "requests": 18,
     "tool_calls": 28,
     "occupied_max": 25241
    },
    {
     "instance_id": "pipeshub-ai__pipeshub-ai-3244",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 55326,
     "budget_exhausted": false,
     "seconds": 849.2,
     "requests": 62,
     "tool_calls": 68,
     "occupied_max": 90422
    },
    {
     "instance_id": "holoviz__holoviews-7058",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 37313,
     "budget_exhausted": false,
     "seconds": 425.6,
     "requests": 42,
     "tool_calls": 56,
     "occupied_max": 39298
    },
    {
     "instance_id": "supabase__supabase-py-1619",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 25453,
     "budget_exhausted": false,
     "seconds": 230.8,
     "requests": 36,
     "tool_calls": 47,
     "occupied_max": 31915
    },
    {
     "instance_id": "agno-agi__agno-9887",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 33850,
     "budget_exhausted": false,
     "seconds": 343.0,
     "requests": 34,
     "tool_calls": 43,
     "occupied_max": 54816
    },
    {
     "instance_id": "openai__openai-agents-python-5016",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 10138,
     "budget_exhausted": false,
     "seconds": 149.1,
     "requests": 29,
     "tool_calls": 37,
     "occupied_max": 24960
    },
    {
     "instance_id": "marin-community__marin-9450",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 22085,
     "budget_exhausted": false,
     "seconds": 312.5,
     "requests": 39,
     "tool_calls": 59,
     "occupied_max": 50513
    },
    {
     "instance_id": "google-deepmind__mujoco_warp-1660",
     "exit_status": "TokenBudgetExhausted",
     "submitted": false,
     "resolved": false,
     "completion_tokens": 60000,
     "budget_exhausted": true,
     "seconds": 641.4,
     "requests": 34,
     "tool_calls": 42,
     "occupied_max": 54175
    },
    {
     "instance_id": "LearningCircuit__local-deep-research-6226",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 38476,
     "budget_exhausted": false,
     "seconds": 452.1,
     "requests": 50,
     "tool_calls": 72,
     "occupied_max": 64416
    },
    {
     "instance_id": "ansible-collections__community.general-12804",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 52535,
     "budget_exhausted": false,
     "seconds": 612.0,
     "requests": 87,
     "tool_calls": 109,
     "occupied_max": 83204
    },
    {
     "instance_id": "embeddings-benchmark__mteb-5443",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 4988,
     "budget_exhausted": false,
     "seconds": 190.4,
     "requests": 21,
     "tool_calls": 27,
     "occupied_max": 16013
    },
    {
     "instance_id": "nautobot__nautobot-9531",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 23809,
     "budget_exhausted": false,
     "seconds": 1257.9,
     "requests": 109,
     "tool_calls": 127,
     "occupied_max": 62642
    },
    {
     "instance_id": "github__spec-kit-4559",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 28165,
     "budget_exhausted": false,
     "seconds": 540.4,
     "requests": 60,
     "tool_calls": 83,
     "occupied_max": 67025
    },
    {
     "instance_id": "Genesis-Embodied-AI__genesis-world-3399",
     "exit_status": "RepeatedFormatError",
     "submitted": false,
     "resolved": false,
     "completion_tokens": 57625,
     "budget_exhausted": false,
     "seconds": 644.5,
     "requests": 42,
     "tool_calls": 38,
     "occupied_max": 54696
    },
    {
     "instance_id": "fossasia__eventyay-5507",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 8598,
     "budget_exhausted": false,
     "seconds": 194.9,
     "requests": 40,
     "tool_calls": 50,
     "occupied_max": 25888
    },
    {
     "instance_id": "dstackai__dstack-4256",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 13442,
     "budget_exhausted": false,
     "seconds": 197.9,
     "requests": 41,
     "tool_calls": 53,
     "occupied_max": 36955
    },
    {
     "instance_id": "ogx-ai__ogx-6725",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 25221,
     "budget_exhausted": false,
     "seconds": 322.6,
     "requests": 42,
     "tool_calls": 53,
     "occupied_max": 55458
    },
    {
     "instance_id": "marin-community__marin-9419",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 45787,
     "budget_exhausted": false,
     "seconds": 510.6,
     "requests": 36,
     "tool_calls": 44,
     "occupied_max": 72796
    },
    {
     "instance_id": "pymc-labs__pymc-marketing-3099",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 19860,
     "budget_exhausted": false,
     "seconds": 1117.5,
     "requests": 74,
     "tool_calls": 85,
     "occupied_max": 55676
    },
    {
     "instance_id": "fsspec__filesystem_spec-2131",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 8397,
     "budget_exhausted": false,
     "seconds": 153.6,
     "requests": 15,
     "tool_calls": 21,
     "occupied_max": 17233
    },
    {
     "instance_id": "stickerdaniel__linkedin-mcp-server-973",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 16622,
     "budget_exhausted": false,
     "seconds": 230.5,
     "requests": 24,
     "tool_calls": 30,
     "occupied_max": 39013
    },
    {
     "instance_id": "ansible__ansible-87624",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": false,
     "completion_tokens": 41882,
     "budget_exhausted": false,
     "seconds": 491.0,
     "requests": 83,
     "tool_calls": 114,
     "occupied_max": 77462
    },
    {
     "instance_id": "scipy__scipy-26156",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 26034,
     "budget_exhausted": false,
     "seconds": 465.9,
     "requests": 47,
     "tool_calls": 60,
     "occupied_max": 41930
    },
    {
     "instance_id": "roboflow__rf-detr-1502",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 9951,
     "budget_exhausted": false,
     "seconds": 252.9,
     "requests": 32,
     "tool_calls": 50,
     "occupied_max": 26069
    },
    {
     "instance_id": "scrapy__scrapy-8138",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 16914,
     "budget_exhausted": false,
     "seconds": 164.0,
     "requests": 22,
     "tool_calls": 27,
     "occupied_max": 28474
    },
    {
     "instance_id": "WeblateOrg__weblate-21746",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": true,
     "completion_tokens": 34673,
     "budget_exhausted": false,
     "seconds": 790.4,
     "requests": 61,
     "tool_calls": 68,
     "occupied_max": 63418
    },
    {
     "instance_id": "wkentaro__labelme-2667",
     "exit_status": "TokenBudgetExhausted",
     "submitted": false,
     "resolved": false,
     "completion_tokens": 60000,
     "budget_exhausted": true,
     "seconds": 805.0,
     "requests": 54,
     "tool_calls": 60,
     "occupied_max": 63073
    },
    {
     "instance_id": "gdsfactory__gdsfactory-4840",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": false,
     "completion_tokens": 36398,
     "budget_exhausted": false,
     "seconds": 464.3,
     "requests": 49,
     "tool_calls": 65,
     "occupied_max": 59467
    },
    {
     "instance_id": "cognizant-ai-lab__neuro-san-studio-1452",
     "exit_status": "Submitted",
     "submitted": true,
     "resolved": false,
     "completion_tokens": 11499,
     "budget_exhausted": false,
     "seconds": 135.0,
     "requests": 21,
     "tool_calls": 31,
     "occupied_max": 27207
    },
    {
     "instance_id": "OpenHands__software-agent-sdk-4871",
     "exit_status": "TokenBudgetExhausted",
     "submitted": false,
     "resolved": false,
     "completion_tokens": 60000,
     "budget_exhausted": true,
     "seconds": 811.1,
     "requests": 61,
     "tool_calls": 82,
     "occupied_max": 99327
    },
    {
     "instance_id": "roboflow__supervision-2600",
     "exit_status": "TokenBudgetExhausted",
     "submitted": false,
     "resolved": false,
     "completion_tokens": 60000,
     "budget_exhausted": true,
     "seconds": 538.5,
     "requests": 20,
     "tool_calls": 18,
     "occupied_max": 33907
    }
   ],
   "unit": "list"
  },
  "tool_calls": {
   "value": 2598,
   "unit": "calls",
   "detail": {
    "counted_toward_100_minimum": false,
    "why": "tool calls in quality_eval runs do not count toward the \u00a75 100-call minimum"
   }
  },
  "parse_failures": {
   "value": 1,
   "unit": "responses",
   "detail": {
    "wire": {
     "invalid_tool_call": 1
    },
    "harness_format_errors": {
     "truncated_no_tool_call": 27
    },
    "union_rule": "a response counts once even if both the wire check and the harness flag it",
    "counted": "malformed tool calls only (decision 0003); truncations and other format errors are separate metrics"
   }
  },
  "output_truncations": {
   "value": 27,
   "unit": "responses",
   "detail": {
    "rule": "cut off at max_tokens with no tool call"
   }
  },
  "agent_format_errors": {
   "value": 0,
   "unit": "responses",
   "detail": {
    "rule": "agent protocol errors other than malformed tool calls"
   }
  },
  "repetition_hits": {
   "value": 43,
   "unit": "hits",
   "detail": {
    "ngram64x3": 43,
    "same_call_x4": 0,
    "ngram64x3_cross_field_not_counted": 0,
    "tokenizer": "server /v1/tokenize"
   }
  },
  "decode_whole_run_tps": {
   "value": 112.198,
   "unit": "tok/s"
  },
  "kld": {
   "state": "unavailable",
   "reason": "not produced by the agentic evaluation; see quality_panel / KLD records"
  },
  "top1_agreement": {
   "state": "unavailable",
   "reason": "not produced by the agentic evaluation; see quality_panel records"
  },
  "mtp_acceptance_rate": {
   "value": 0.6948,
   "unit": "fraction",
   "detail": {
    "mean_accept_len": 3.085,
    "batches": 10995,
    "source": "SGLang decode log"
   }
  },
  "mtp_accepted_tps": {
   "value": 112.198,
   "unit": "tok/s",
   "detail": "output tok/s including accepted draft tokens"
  }
 },
 "eligibility": "none",
 "artifacts": [
  "artifacts/runs/20261011T194914Z-lab-sglang-qwen3.8-27b-registry-208k-rtx-3090-24gb-gpu2-quality_eval"
 ],
 "notes": "quality evaluation, uncapped, 60000 completion tokens per task; mini-swe-agent 04d809ceab9d; sampling {\"temperature\": 0.7, \"top_p\": 0.95, \"seed\": 1234, \"max_tokens\": 8192} | tool calls in quality_eval runs do not count toward the \u00a75 100-call minimum"
}
