Official benchmark setup required. Current development checks and adapted results are not official benchmark runs. Official comparisons must use the upstream code, datasets, scoring and prescribed setup, with versions and deviations disclosed. Protocol status →
Trace 5c40ec5b
Stored context/evidence
["156988fe","94e2773e","ea67b29e_1","ff2fd48c","ultrachat_68780","sharegpt_dDOb7on_0","fcae5c39_1","ultrachat_117952","sharegpt_vrQ82G5_0","d9868305_1","aaea9770_4","sharegpt_hGXFJYQ_5","sharegpt_5ZmWZjz_0","sharegpt_Ah0tRO5_0","9eb7bb35","692075ef","answer_1cb52d0a_1","ultrachat_545683","be050ab3_2","5cc9e1ed_3","sharegpt_Fv7hQ6N_0","sharegpt_AZELQqX_0","sharegpt_CWiYdQ8_152","ce7fedf8_2","ultrachat_216243","5560bdcc","sharegpt_e6daAMu_0","sharegpt_AVnVlaF_0","0e8c2a62","ultrachat_342093","sharegpt_gGV9ADI_13","ultrachat_333871","sharegpt_7y4qwfF_11","bf4fdf8f_2","ultrachat_91084","ultrachat_39117","sharegpt_XkVyVGq_0","25a2002c","ultrachat_122152","ultrachat_467192","ultrachat_271436","ultrachat_327236","sharegpt_xfPJOnE_0","sharegpt_dBQwSJX_0","sharegpt_RVk16qK_0","20cf06b8","4c8eec95","ec00a866_1","a200b713_3","6446f6e6","answer_1cb52d0a_2"]
Retrieved context/evidence
["answer_1cb52d0a_2","ultrachat_122152"]
Retrieval output (not a generated answer)
answer_1cb52d0a_2
Complete stored trace (JSON)
{
"answer": "answer_1cb52d0a_2",
"answer_session_ids": [
"answer_1cb52d0a_1",
"answer_1cb52d0a_2"
],
"gold_references": [
"answer_1cb52d0a_1",
"answer_1cb52d0a_2"
],
"oracle_has_answer_turns": {
"0e8c2a62": [],
"156988fe": [],
"20cf06b8": [],
"25a2002c": [],
"4c8eec95": [],
"5560bdcc": [],
"5cc9e1ed_3": [],
"6446f6e6": [],
"692075ef": [],
"94e2773e": [],
"9eb7bb35": [],
"a200b713_3": [],
"aaea9770_4": [],
"answer_1cb52d0a_1": [
6
],
"answer_1cb52d0a_2": [
0
],
"be050ab3_2": [],
"bf4fdf8f_2": [],
"ce7fedf8_2": [],
"d9868305_1": [],
"ea67b29e_1": [],
"ec00a866_1": [],
"fcae5c39_1": [],
"ff2fd48c": [],
"sharegpt_5ZmWZjz_0": [],
"sharegpt_7y4qwfF_11": [],
"sharegpt_AVnVlaF_0": [],
"sharegpt_AZELQqX_0": [],
"sharegpt_Ah0tRO5_0": [],
"sharegpt_CWiYdQ8_152": [],
"sharegpt_Fv7hQ6N_0": [],
"sharegpt_RVk16qK_0": [],
"sharegpt_XkVyVGq_0": [],
"sharegpt_dBQwSJX_0": [],
"sharegpt_dDOb7on_0": [],
"sharegpt_e6daAMu_0": [],
"sharegpt_gGV9ADI_13": [],
"sharegpt_hGXFJYQ_5": [],
"sharegpt_vrQ82G5_0": [],
"sharegpt_xfPJOnE_0": [],
"ultrachat_117952": [],
"ultrachat_122152": [],
"ultrachat_216243": [],
"ultrachat_271436": [],
"ultrachat_327236": [],
"ultrachat_333871": [],
"ultrachat_342093": [],
"ultrachat_39117": [],
"ultrachat_467192": [],
"ultrachat_545683": [],
"ultrachat_68780": [],
"ultrachat_91084": []
},
"question_id": "5c40ec5b",
"ranked_retrieved_hits": [
"answer_1cb52d0a_2",
"ultrachat_122152"
],
"scoring_family": "deterministic-retrieval",
"stored_records": [
"156988fe",
"94e2773e",
"ea67b29e_1",
"ff2fd48c",
"ultrachat_68780",
"sharegpt_dDOb7on_0",
"fcae5c39_1",
"ultrachat_117952",
"sharegpt_vrQ82G5_0",
"d9868305_1",
"aaea9770_4",
"sharegpt_hGXFJYQ_5",
"sharegpt_5ZmWZjz_0",
"sharegpt_Ah0tRO5_0",
"9eb7bb35",
"692075ef",
"answer_1cb52d0a_1",
"ultrachat_545683",
"be050ab3_2",
"5cc9e1ed_3",
"sharegpt_Fv7hQ6N_0",
"sharegpt_AZELQqX_0",
"sharegpt_CWiYdQ8_152",
"ce7fedf8_2",
"ultrachat_216243",
"5560bdcc",
"sharegpt_e6daAMu_0",
"sharegpt_AVnVlaF_0",
"0e8c2a62",
"ultrachat_342093",
"sharegpt_gGV9ADI_13",
"ultrachat_333871",
"sharegpt_7y4qwfF_11",
"bf4fdf8f_2",
"ultrachat_91084",
"ultrachat_39117",
"sharegpt_XkVyVGq_0",
"25a2002c",
"ultrachat_122152",
"ultrachat_467192",
"ultrachat_271436",
"ultrachat_327236",
"sharegpt_xfPJOnE_0",
"sharegpt_dBQwSJX_0",
"sharegpt_RVk16qK_0",
"20cf06b8",
"4c8eec95",
"ec00a866_1",
"a200b713_3",
"6446f6e6",
"answer_1cb52d0a_2"
]
}