{
  "run_id": "all_systems_plus_decision_gate_20261007_184150",
  "mode": "live",
  "models_used": [
    "gpt-6-luna",
    "gpt-6.1-sol",
    "text-embedding-3-small"
  ],
  "unknown_priced_models": [],
  "pricing_as_of": "2026-10-01",
  "run_wall_time_ms": 559507.1694159415,
  "max_workers": 4,
  "execution": {
    "max_workers": 4,
    "system_workers": 1,
    "ingest_workers": 4,
    "latency_probe_questions": 5,
    "latency_source": "probe",
    "probe_calls": 120,
    "probe_cost_usd": 0.062643
  },
  "num_systems": 24,
  "budget": {
    "max_cost_usd": 6.0,
    "spent_usd": 3.258221080000001,
    "stopped": false,
    "completed_systems": [
      "bm25_default",
      "vector_default",
      "hybrid_default",
      "rerank_default",
      "hybrid_rerank_default",
      "hyde_default",
      "parent_doc_default",
      "llm_heavy_default",
      "no_retrieval_floor",
      "full_context_ceiling",
      "sentence_window_default",
      "vector_mmr",
      "contextual_default",
      "hierarchical_default",
      "rag_fusion_default",
      "decompose_default",
      "corrective_default",
      "iterative_default",
      "agent_search_plain",
      "agent_search_tools",
      "grep_agent_default",
      "adaptive_default",
      "decision_gate_default",
      "decision_gate_strict"
    ],
    "incomplete_systems": {}
  },
  "num_question_rows": 3912,
  "num_errors": 0,
  "errors_by_system": {},
  "k_values": [
    1,
    3,
    5,
    10
  ],
  "retrieval_depth": 10,
  "context_k": 5,
  "primary_k": 5,
  "embedding_cache": {
    "hits": 1812,
    "misses": 1745,
    "saved_cost_usd": 0.009359759999999984
  },
  "cache": {
    "enabled": true,
    "path": ".ragbench_cache/cache.sqlite3",
    "hits": 12662,
    "misses": 9,
    "hit_rate": 0.9992897166758741,
    "saved_cost_usd": 3.2420809999999816,
    "by_namespace": {
      "decision": {
        "hits": 615,
        "misses": 0,
        "saved_cost_usd": 0.21653430000000018
      },
      "embeddings": {
        "hits": 1632,
        "misses": 0,
        "saved_cost_usd": 0.002342300000000001
      },
      "llm": {
        "hits": 10415,
        "misses": 9,
        "saved_cost_usd": 3.0232043999999814
      }
    },
    "size_bytes": 21562264,
    "charged_cost_usd": 3.25822108,
    "real_spend_usd": 0.06942332000001868
  },
  "dataset": {
    "questions": 163,
    "labeled_questions": 138,
    "label_free": false,
    "synthetic_questions": 0,
    "needs_review_questions": 0,
    "mock_questions": 0
  },
  "notices": [
    "Self-preference risk: the judge (gpt-6-luna) is also the generator of bm25_default, vector_default, hybrid_default, rerank_default, hybrid_rerank_default, hyde_default, parent_doc_default, no_retrieval_floor, full_context_ceiling, sentence_window_default, vector_mmr, contextual_default, hierarchical_default, rag_fusion_default, decompose_default, corrective_default, iterative_default, agent_search_plain, agent_search_tools, grep_agent_default, adaptive_default, decision_gate_default, decision_gate_strict. Judges tend to rate answers from their own model higher, so those scores may be inflated. Use a different model under `evaluation.judge.model`, or set `evaluation.judge.independent: false` to accept this."
  ],
  "dataset_warnings": [],
  "document_warnings": [],
  "outputs": {
    "leaderboard": "leaderboard.md",
    "report": "report.html",
    "qrels_audit": "qrels_audit.md",
    "stats": "stats.json",
    "significance": "significance.csv",
    "pareto": "pareto.json",
    "recommendation": "recommendation.md"
  }
}