Official benchmark setup required. Current development checks and adapted results are not official benchmark runs. Official comparisons must use the upstream code, datasets, scoring and prescribed setup, with versions and deviations disclosed. Protocol status →
Trace d3ab962e
Stored context/evidence
["ultrachat_564044","166d4800","013b320f","97e4dd21_6","sharegpt_CDacKfJ_0","e0e09660","ultrachat_511452","sharegpt_MxT7oAc_0","answer_5237bb0b_2","24e44a64","ultrachat_546581","ultrachat_248741","ultrachat_48872","ultrachat_91354","ultrachat_370323","ultrachat_43823","ultrachat_810","sharegpt_4ubTesZ_0","sharegpt_6fTnCnw_49","92f1ea4d_1","sharegpt_FgbjQgy_0","sharegpt_6TSnbGC_0","ae15e8b6_3","7da6f798_1","sharegpt_JF2KwQt_15","ultrachat_164834","45b155ec_1","ultrachat_328896","ultrachat_507563","ultrachat_126384","sharegpt_yMwEvl7_19","ultrachat_155769","3329e5e8_1","218141fb_2","be679e09","b66e4f25_2","ultrachat_536037","answer_5237bb0b_1","ultrachat_225746","aa235649_1","007e7d81_4","a1d698f1_3","sharegpt_rxFVsTB_0","52403c17_1","ultrachat_219635","1980bbfa_2","ultrachat_125084","sharegpt_flHO7QM_17","ultrachat_503039"]
Retrieved context/evidence
["ultrachat_546581","ultrachat_810","sharegpt_6TSnbGC_0","sharegpt_FgbjQgy_0"]
Retrieval output (not a generated answer)
ultrachat_546581
Complete stored trace (JSON)
{
"answer": "ultrachat_546581",
"answer_session_ids": [
"answer_5237bb0b_2",
"answer_5237bb0b_1"
],
"gold_references": [
"answer_5237bb0b_2",
"answer_5237bb0b_1"
],
"oracle_has_answer_turns": {
"007e7d81_4": [],
"013b320f": [],
"166d4800": [],
"1980bbfa_2": [],
"218141fb_2": [],
"24e44a64": [],
"3329e5e8_1": [],
"45b155ec_1": [],
"52403c17_1": [],
"7da6f798_1": [],
"92f1ea4d_1": [],
"97e4dd21_6": [],
"a1d698f1_3": [],
"aa235649_1": [],
"ae15e8b6_3": [],
"answer_5237bb0b_1": [
0
],
"answer_5237bb0b_2": [
0
],
"b66e4f25_2": [],
"be679e09": [],
"e0e09660": [],
"sharegpt_4ubTesZ_0": [],
"sharegpt_6TSnbGC_0": [],
"sharegpt_6fTnCnw_49": [],
"sharegpt_CDacKfJ_0": [],
"sharegpt_FgbjQgy_0": [],
"sharegpt_JF2KwQt_15": [],
"sharegpt_MxT7oAc_0": [],
"sharegpt_flHO7QM_17": [],
"sharegpt_rxFVsTB_0": [],
"sharegpt_yMwEvl7_19": [],
"ultrachat_125084": [],
"ultrachat_126384": [],
"ultrachat_155769": [],
"ultrachat_164834": [],
"ultrachat_219635": [],
"ultrachat_225746": [],
"ultrachat_248741": [],
"ultrachat_328896": [],
"ultrachat_370323": [],
"ultrachat_43823": [],
"ultrachat_48872": [],
"ultrachat_503039": [],
"ultrachat_507563": [],
"ultrachat_511452": [],
"ultrachat_536037": [],
"ultrachat_546581": [],
"ultrachat_564044": [],
"ultrachat_810": [],
"ultrachat_91354": []
},
"question_id": "d3ab962e",
"ranked_retrieved_hits": [
"ultrachat_546581",
"ultrachat_810",
"sharegpt_6TSnbGC_0",
"sharegpt_FgbjQgy_0"
],
"scoring_family": "deterministic-retrieval",
"stored_records": [
"ultrachat_564044",
"166d4800",
"013b320f",
"97e4dd21_6",
"sharegpt_CDacKfJ_0",
"e0e09660",
"ultrachat_511452",
"sharegpt_MxT7oAc_0",
"answer_5237bb0b_2",
"24e44a64",
"ultrachat_546581",
"ultrachat_248741",
"ultrachat_48872",
"ultrachat_91354",
"ultrachat_370323",
"ultrachat_43823",
"ultrachat_810",
"sharegpt_4ubTesZ_0",
"sharegpt_6fTnCnw_49",
"92f1ea4d_1",
"sharegpt_FgbjQgy_0",
"sharegpt_6TSnbGC_0",
"ae15e8b6_3",
"7da6f798_1",
"sharegpt_JF2KwQt_15",
"ultrachat_164834",
"45b155ec_1",
"ultrachat_328896",
"ultrachat_507563",
"ultrachat_126384",
"sharegpt_yMwEvl7_19",
"ultrachat_155769",
"3329e5e8_1",
"218141fb_2",
"be679e09",
"b66e4f25_2",
"ultrachat_536037",
"answer_5237bb0b_1",
"ultrachat_225746",
"aa235649_1",
"007e7d81_4",
"a1d698f1_3",
"sharegpt_rxFVsTB_0",
"52403c17_1",
"ultrachat_219635",
"1980bbfa_2",
"ultrachat_125084",
"sharegpt_flHO7QM_17",
"ultrachat_503039"
]
}