{
  "method": "percentile bootstrap over questions; paired bootstrap against the baseline with Holm-Bonferroni adjustment per metric",
  "n_boot": 2000,
  "seed": 0,
  "alpha": 0.05,
  "baseline": "no_retrieval_floor",
  "baseline_source": "cheapest system",
  "correct_threshold": 4.0,
  "metrics": [
    "retrieval_recall@5",
    "retrieval_mrr@10",
    "retrieval_ndcg@10",
    "answer_score",
    "faithfulness"
  ],
  "systems": {
    "bm25_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.745398773006135,
          "lo": 0.6809560327198364,
          "hi": 0.8077709611451942
        },
        "retrieval_mrr@10": {
          "mean": 0.7203476482617587,
          "lo": 0.653872699386503,
          "hi": 0.7822085889570553
        },
        "retrieval_ndcg@10": {
          "mean": 0.7077904096249197,
          "lo": 0.6465777159751233,
          "hi": 0.7666610942308638
        },
        "answer_score": {
          "mean": 4.6871165644171775,
          "lo": 4.54601226993865,
          "hi": 4.808650306748465
        },
        "faithfulness": {
          "mean": 4.987730061349693,
          "lo": 4.969325153374233,
          "hi": 5.0
        }
      },
      "cost_per_correct": 0.0005230611842105262
    },
    "vector_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.7903885480572596,
          "lo": 0.7326048057259713,
          "hi": 0.8471370143149283
        },
        "retrieval_mrr@10": {
          "mean": 0.7376278118609408,
          "lo": 0.6759611451942741,
          "hi": 0.7958154396728016
        },
        "retrieval_ndcg@10": {
          "mean": 0.7180551866936772,
          "lo": 0.6626131848099256,
          "hi": 0.7737370255857489
        },
        "answer_score": {
          "mean": 4.673619631901841,
          "lo": 4.54478527607362,
          "hi": 4.791411042944785
        },
        "faithfulness": {
          "mean": 4.9447852760736195,
          "lo": 4.883435582822086,
          "hi": 4.987730061349693
        }
      },
      "cost_per_correct": 0.0004730938666666668
    },
    "hybrid_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.7745398773006135,
          "lo": 0.7146855828220858,
          "hi": 0.8328220858895705
        },
        "retrieval_mrr@10": {
          "mean": 0.74546693933197,
          "lo": 0.6786971710974778,
          "hi": 0.8064459781867757
        },
        "retrieval_ndcg@10": {
          "mean": 0.7321147167967854,
          "lo": 0.6712097342440698,
          "hi": 0.7897778131828561
        },
        "answer_score": {
          "mean": 4.606134969325153,
          "lo": 4.458865030674847,
          "hi": 4.742331288343558
        },
        "faithfulness": {
          "mean": 4.920245398773006,
          "lo": 4.822085889570552,
          "hi": 4.993865030674846
        }
      },
      "cost_per_correct": 0.0005239797297297298
    },
    "rerank_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.7888548057259712,
          "lo": 0.729026073619632,
          "hi": 0.8456032719836399
        },
        "retrieval_mrr@10": {
          "mean": 0.7465746421267895,
          "lo": 0.687377300613497,
          "hi": 0.8049079754601226
        },
        "retrieval_ndcg@10": {
          "mean": 0.7321952229422962,
          "lo": 0.6742219172717648,
          "hi": 0.7863967396115126
        },
        "answer_score": {
          "mean": 4.743558282208589,
          "lo": 4.626993865030675,
          "hi": 4.844171779141105
        },
        "faithfulness": {
          "mean": 4.950920245398773,
          "lo": 4.901840490797546,
          "hi": 4.987883435582821
        }
      },
      "cost_per_correct": 0.0005459324675324671
    },
    "hybrid_rerank_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.7735173824130879,
          "lo": 0.712678936605317,
          "hi": 0.8323108384458079
        },
        "retrieval_mrr@10": {
          "mean": 0.7365541922290388,
          "lo": 0.6754984662576687,
          "hi": 0.7969401840490797
        },
        "retrieval_ndcg@10": {
          "mean": 0.7272660360158002,
          "lo": 0.6692058147030353,
          "hi": 0.7837471709173263
        },
        "answer_score": {
          "mean": 4.723926380368098,
          "lo": 4.590184049079755,
          "hi": 4.839263803680981
        },
        "faithfulness": {
          "mean": 4.9815950920245395,
          "lo": 4.950920245398773,
          "hi": 5.0
        }
      },
      "cost_per_correct": 0.0005650129870129871
    },
    "hyde_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.7873210633946829,
          "lo": 0.7290260736196319,
          "hi": 0.8450920245398773
        },
        "retrieval_mrr@10": {
          "mean": 0.7246932515337423,
          "lo": 0.6638356339468302,
          "hi": 0.7840299079754598
        },
        "retrieval_ndcg@10": {
          "mean": 0.7097791408818571,
          "lo": 0.6557107073480065,
          "hi": 0.7649071751612844
        },
        "answer_score": {
          "mean": 4.679754601226994,
          "lo": 4.544754601226994,
          "hi": 4.796319018404907
        },
        "faithfulness": {
          "mean": 4.969325153374233,
          "lo": 4.920245398773006,
          "hi": 5.0
        }
      },
      "cost_per_correct": 0.0004955582894736844
    },
    "parent_doc_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.7893660531697341,
          "lo": 0.7326048057259714,
          "hi": 0.8450920245398773
        },
        "retrieval_mrr@10": {
          "mean": 0.7376278118609408,
          "lo": 0.6757617586912066,
          "hi": 0.7962218813905931
        },
        "retrieval_ndcg@10": {
          "mean": 0.7171408896935809,
          "lo": 0.662516265046344,
          "hi": 0.7725213870436626
        },
        "answer_score": {
          "mean": 4.712883435582822,
          "lo": 4.6,
          "hi": 4.812300613496933
        },
        "faithfulness": {
          "mean": 4.932515337423313,
          "lo": 4.846625766871166,
          "hi": 4.987730061349693
        }
      },
      "cost_per_correct": 0.0005257872847682124
    },
    "llm_heavy_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.7898773006134969,
          "lo": 0.728489263803681,
          "hi": 0.848159509202454
        },
        "retrieval_mrr@10": {
          "mean": 0.7696319018404909,
          "lo": 0.7095040899795502,
          "hi": 0.8271983640081798
        },
        "retrieval_ndcg@10": {
          "mean": 0.7472330514989168,
          "lo": 0.6915494686629413,
          "hi": 0.8014880134535127
        },
        "answer_score": {
          "mean": 4.7165644171779135,
          "lo": 4.60361963190184,
          "hi": 4.8196319018404905
        },
        "faithfulness": {
          "mean": 4.92638036809816,
          "lo": 4.858895705521473,
          "hi": 4.975460122699387
        }
      },
      "cost_per_correct": 0.0053866329032258055
    },
    "no_retrieval_floor": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": null,
          "lo": null,
          "hi": null
        },
        "retrieval_mrr@10": {
          "mean": null,
          "lo": null,
          "hi": null
        },
        "retrieval_ndcg@10": {
          "mean": null,
          "lo": null,
          "hi": null
        },
        "answer_score": {
          "mean": 1.536196319018405,
          "lo": 1.3975460122699384,
          "hi": 1.6797546012269935
        },
        "faithfulness": {
          "mean": 4.693251533742331,
          "lo": 4.50920245398773,
          "hi": 4.877300613496932
        }
      },
      "cost_per_correct": 0.004059140000000001
    },
    "full_context_ceiling": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.74079754601227,
          "lo": 0.6758563394683026,
          "hi": 0.803680981595092
        },
        "retrieval_mrr@10": {
          "mean": 0.7284083162917518,
          "lo": 0.6661822597137015,
          "hi": 0.7912499999999998
        },
        "retrieval_ndcg@10": {
          "mean": 0.7150316676290754,
          "lo": 0.6565984790762031,
          "hi": 0.771949393914326
        },
        "answer_score": {
          "mean": 4.775460122699387,
          "lo": 4.701840490797546,
          "hi": 4.8429754601227
        },
        "faithfulness": {
          "mean": 4.840490797546012,
          "lo": 4.711656441717792,
          "hi": 4.9447852760736195
        }
      },
      "cost_per_correct": 0.002966699350649352
    },
    "sentence_window_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.7801635991820041,
          "lo": 0.7203476482617587,
          "hi": 0.8384458077709611
        },
        "retrieval_mrr@10": {
          "mean": 0.6920318434122115,
          "lo": 0.6305740578439964,
          "hi": 0.7549227651183171
        },
        "retrieval_ndcg@10": {
          "mean": 0.6929110720206265,
          "lo": 0.6374797991584469,
          "hi": 0.7487271623633085
        },
        "answer_score": {
          "mean": 4.415950920245399,
          "lo": 4.2319018404907975,
          "hi": 4.587730061349693
        },
        "faithfulness": {
          "mean": 4.815950920245399,
          "lo": 4.674846625766871,
          "hi": 4.938650306748467
        }
      },
      "cost_per_correct": 0.00028342042553191504
    },
    "vector_mmr": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.6973415132924334,
          "lo": 0.6354677914110429,
          "hi": 0.757170245398773
        },
        "retrieval_mrr@10": {
          "mean": 0.7309475119291071,
          "lo": 0.6680725971370144,
          "hi": 0.7917910702113157
        },
        "retrieval_ndcg@10": {
          "mean": 0.6792493498704848,
          "lo": 0.619635772241118,
          "hi": 0.7389173227174158
        },
        "answer_score": {
          "mean": 4.543558282208589,
          "lo": 4.397546012269938,
          "hi": 4.6773312883435585
        },
        "faithfulness": {
          "mean": 4.9447852760736195,
          "lo": 4.889570552147239,
          "hi": 4.987730061349693
        }
      },
      "cost_per_correct": 0.00045751191489361694
    },
    "contextual_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.7842535787321063,
          "lo": 0.7249488752556238,
          "hi": 0.8399923312883436
        },
        "retrieval_mrr@10": {
          "mean": 0.7528702892199824,
          "lo": 0.6913776292725679,
          "hi": 0.8118917981302949
        },
        "retrieval_ndcg@10": {
          "mean": 0.7410028954358462,
          "lo": 0.6835374367717509,
          "hi": 0.7976359888298549
        },
        "answer_score": {
          "mean": 4.694478527607362,
          "lo": 4.569325153374233,
          "hi": 4.808588957055214
        },
        "faithfulness": {
          "mean": 4.932515337423313,
          "lo": 4.858895705521473,
          "hi": 4.987883435582821
        }
      },
      "cost_per_correct": 0.0004970478666666665
    },
    "hierarchical_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.7326175869120655,
          "lo": 0.6692101226993866,
          "hi": 0.7950025562372187
        },
        "retrieval_mrr@10": {
          "mean": 0.7065439672801637,
          "lo": 0.6416155419222904,
          "hi": 0.7699642126789364
        },
        "retrieval_ndcg@10": {
          "mean": 0.6661379608340154,
          "lo": 0.6067646472839487,
          "hi": 0.7251575277476701
        },
        "answer_score": {
          "mean": 4.558282208588957,
          "lo": 4.399938650306748,
          "hi": 4.69819018404908
        },
        "faithfulness": {
          "mean": 4.96319018404908,
          "lo": 4.914110429447852,
          "hi": 5.0
        }
      },
      "cost_per_correct": 0.0003712350344827588
    },
    "rag_fusion_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.808282208588957,
          "lo": 0.7530674846625767,
          "hi": 0.8634969325153374
        },
        "retrieval_mrr@10": {
          "mean": 0.7442740286298568,
          "lo": 0.6844401840490797,
          "hi": 0.8023594069529651
        },
        "retrieval_ndcg@10": {
          "mean": 0.743601274863904,
          "lo": 0.6877768365089492,
          "hi": 0.8004132052697103
        },
        "answer_score": {
          "mean": 4.754601226993865,
          "lo": 4.644171779141105,
          "hi": 4.856441717791411
        },
        "faithfulness": {
          "mean": 4.96319018404908,
          "lo": 4.914110429447852,
          "hi": 5.0
        }
      },
      "cost_per_correct": 0.00044679831168831187
    },
    "decompose_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.7714723926380368,
          "lo": 0.7131901840490797,
          "hi": 0.8312883435582822
        },
        "retrieval_mrr@10": {
          "mean": 0.7279141104294479,
          "lo": 0.6661554192229039,
          "hi": 0.7867075664621678
        },
        "retrieval_ndcg@10": {
          "mean": 0.7234731074300248,
          "lo": 0.6669323626180148,
          "hi": 0.7807141069522252
        },
        "answer_score": {
          "mean": 4.721472392638038,
          "lo": 4.596319018404908,
          "hi": 4.833128834355828
        },
        "faithfulness": {
          "mean": 4.92638036809816,
          "lo": 4.828220858895706,
          "hi": 4.993865030674846
        }
      },
      "cost_per_correct": 0.0004356605228758169
    },
    "corrective_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.7929447852760736,
          "lo": 0.7331288343558282,
          "hi": 0.8496932515337423
        },
        "retrieval_mrr@10": {
          "mean": 0.8302658486707565,
          "lo": 0.7740286298568506,
          "hi": 0.8834355828220859
        },
        "retrieval_ndcg@10": {
          "mean": 0.7960838396810264,
          "lo": 0.7416808040001748,
          "hi": 0.8499440377243017
        },
        "answer_score": {
          "mean": 4.7680981595092025,
          "lo": 4.6711042944785275,
          "hi": 4.858895705521473
        },
        "faithfulness": {
          "mean": 4.987730061349693,
          "lo": 4.969325153374233,
          "hi": 5.0
        }
      },
      "cost_per_correct": 0.0006306085897435896
    },
    "iterative_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.799079754601227,
          "lo": 0.7423312883435583,
          "hi": 0.8558282208588958
        },
        "retrieval_mrr@10": {
          "mean": 0.7798568507157465,
          "lo": 0.7207336400817996,
          "hi": 0.8333333333333333
        },
        "retrieval_ndcg@10": {
          "mean": 0.7689854029609017,
          "lo": 0.7132490727806039,
          "hi": 0.8228956616808633
        },
        "answer_score": {
          "mean": 4.811042944785276,
          "lo": 4.7092024539877295,
          "hi": 4.9006134969325155
        },
        "faithfulness": {
          "mean": 4.914110429447852,
          "lo": 4.828220858895706,
          "hi": 4.975460122699387
        }
      },
      "cost_per_correct": 0.0005927037179487179
    },
    "agent_search_plain": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.8144171779141104,
          "lo": 0.7561349693251533,
          "hi": 0.8711656441717791
        },
        "retrieval_mrr@10": {
          "mean": 0.7658486707566463,
          "lo": 0.7075664621676893,
          "hi": 0.8261758691206543
        },
        "retrieval_ndcg@10": {
          "mean": 0.7579604017275197,
          "lo": 0.7025495893089295,
          "hi": 0.813271418193743
        },
        "answer_score": {
          "mean": 4.580368098159509,
          "lo": 4.434355828220859,
          "hi": 4.720245398773006
        },
        "faithfulness": {
          "mean": 4.785276073619632,
          "lo": 4.644171779141105,
          "hi": 4.901840490797546
        }
      },
      "cost_per_correct": 0.0008021356164383561
    },
    "agent_search_tools": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.8205521472392638,
          "lo": 0.7668711656441718,
          "hi": 0.8758052147239263
        },
        "retrieval_mrr@10": {
          "mean": 0.7591002044989775,
          "lo": 0.6991743353783231,
          "hi": 0.8176891615541922
        },
        "retrieval_ndcg@10": {
          "mean": 0.7574605557307335,
          "lo": 0.7029725944583403,
          "hi": 0.8119377931216348
        },
        "answer_score": {
          "mean": 4.752147239263804,
          "lo": 4.628220858895706,
          "hi": 4.85398773006135
        },
        "faithfulness": {
          "mean": 4.846625766871166,
          "lo": 4.723926380368098,
          "hi": 4.9447852760736195
        }
      },
      "cost_per_correct": 0.000859329358974359
    },
    "grep_agent_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.7914110429447853,
          "lo": 0.7361196319018405,
          "hi": 0.8496932515337423
        },
        "retrieval_mrr@10": {
          "mean": 0.6939672801635991,
          "lo": 0.6336221881390592,
          "hi": 0.7542944785276073
        },
        "retrieval_ndcg@10": {
          "mean": 0.6961107436027345,
          "lo": 0.6402234334868605,
          "hi": 0.7498845644170841
        },
        "answer_score": {
          "mean": 4.78282208588957,
          "lo": 4.685889570552147,
          "hi": 4.863803680981595
        },
        "faithfulness": {
          "mean": 4.730061349693251,
          "lo": 4.564417177914111,
          "hi": 4.865184049079754
        }
      },
      "cost_per_correct": 0.001155446753246754
    },
    "adaptive_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.7714723926380368,
          "lo": 0.7116564417177914,
          "hi": 0.8282208588957055
        },
        "retrieval_mrr@10": {
          "mean": 0.7236196319018404,
          "lo": 0.6607310838445808,
          "hi": 0.7844631901840491
        },
        "retrieval_ndcg@10": {
          "mean": 0.7187513547947045,
          "lo": 0.6607323685715867,
          "hi": 0.7742965465350318
        },
        "answer_score": {
          "mean": 4.704294478527607,
          "lo": 4.571779141104295,
          "hi": 4.822085889570552
        },
        "faithfulness": {
          "mean": 4.993865030674846,
          "lo": 4.9815950920245395,
          "hi": 5.0
        }
      },
      "cost_per_correct": 0.0004726684210526317
    },
    "decision_gate_default": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.7735173824130879,
          "lo": 0.712678936605317,
          "hi": 0.8323108384458079
        },
        "retrieval_mrr@10": {
          "mean": 0.7365541922290388,
          "lo": 0.6754984662576687,
          "hi": 0.7969401840490797
        },
        "retrieval_ndcg@10": {
          "mean": 0.7272660360158002,
          "lo": 0.6692058147030353,
          "hi": 0.7837471709173263
        },
        "answer_score": {
          "mean": 4.804907975460123,
          "lo": 4.704294478527608,
          "hi": 4.888343558282208
        },
        "faithfulness": {
          "mean": 4.975460122699387,
          "lo": 4.9447852760736195,
          "hi": 5.0
        }
      },
      "cost_per_correct": 0.0009934258064516132
    },
    "decision_gate_strict": {
      "n": 163,
      "metrics": {
        "retrieval_recall@5": {
          "mean": 0.7735173824130879,
          "lo": 0.712678936605317,
          "hi": 0.8323108384458079
        },
        "retrieval_mrr@10": {
          "mean": 0.7365541922290388,
          "lo": 0.6754984662576687,
          "hi": 0.7969401840490797
        },
        "retrieval_ndcg@10": {
          "mean": 0.7272660360158002,
          "lo": 0.6692058147030353,
          "hi": 0.7837471709173263
        },
        "answer_score": {
          "mean": 4.654601226993865,
          "lo": 4.488941717791412,
          "hi": 4.796932515337424
        },
        "faithfulness": {
          "mean": 4.969325153374233,
          "lo": 4.9079754601226995,
          "hi": 5.0
        }
      },
      "cost_per_correct": 0.0009884746666666666
    }
  }
}