Official benchmark setup required. Current development checks and adapted results are not official benchmark runs. Official comparisons must use the upstream code, datasets, scoring and prescribed setup, with versions and deviations disclosed. Protocol status →
Trace 0e4e4c46
Stored context/evidence
["4842b73e","ultrachat_446413","e82ce7cd_3","ultrachat_274273","ultrachat_180729","6c0ba52d_1","ultrachat_467820","0773943b","answer_f2f998c7_1","ultrachat_97515","607b478d","ultrachat_199519","sharegpt_XpiQ03D_0","5d1c7bf7","b7702ee4_1","70cae729_3","9f1c9048","sharegpt_1YtwybC_15","sharegpt_SpgAMBx_0","55e0c6db_1","sharegpt_FQi5DlJ_48","a0cc90de","ultrachat_449001","b42cbaf0_1","sharegpt_HBnEfom_0","dfe8e126_1","ultrachat_59250","71d1e644","ultrachat_198964","998616b4_2","sharegpt_K8mXaIT_0","sharegpt_puvUs0e_0","debc34d7_1","cbd08e3c_abs_2","ultrachat_564750","17822471","8409f0d2_3","60876def_2","ultrachat_525872","sharegpt_xhPNTAJ_0","sharegpt_XOT5vdc_6","sharegpt_iUc80a8_0","sharegpt_Wi7Op1u_39","6702277b_1","ultrachat_280826","sharegpt_XYnZHIX_10","11e9d591_1","93f2334f_1","answer_f2f998c7_2"]
Retrieved context/evidence
["sharegpt_SpgAMBx_0","sharegpt_XYnZHIX_10","sharegpt_XpiQ03D_0","ultrachat_59250","sharegpt_Wi7Op1u_39"]
Retrieval output (not a generated answer)
sharegpt_SpgAMBx_0
Complete stored trace (JSON)
{
"answer": "sharegpt_SpgAMBx_0",
"answer_session_ids": [
"answer_f2f998c7_1",
"answer_f2f998c7_2"
],
"gold_references": [
"answer_f2f998c7_1",
"answer_f2f998c7_2"
],
"oracle_has_answer_turns": {
"0773943b": [],
"11e9d591_1": [],
"17822471": [],
"4842b73e": [],
"55e0c6db_1": [],
"5d1c7bf7": [],
"607b478d": [],
"60876def_2": [],
"6702277b_1": [],
"6c0ba52d_1": [],
"70cae729_3": [],
"71d1e644": [],
"8409f0d2_3": [],
"93f2334f_1": [],
"998616b4_2": [],
"9f1c9048": [],
"a0cc90de": [],
"answer_f2f998c7_1": [
2
],
"answer_f2f998c7_2": [
8
],
"b42cbaf0_1": [],
"b7702ee4_1": [],
"cbd08e3c_abs_2": [],
"debc34d7_1": [],
"dfe8e126_1": [],
"e82ce7cd_3": [],
"sharegpt_1YtwybC_15": [],
"sharegpt_FQi5DlJ_48": [],
"sharegpt_HBnEfom_0": [],
"sharegpt_K8mXaIT_0": [],
"sharegpt_SpgAMBx_0": [],
"sharegpt_Wi7Op1u_39": [],
"sharegpt_XOT5vdc_6": [],
"sharegpt_XYnZHIX_10": [],
"sharegpt_XpiQ03D_0": [],
"sharegpt_iUc80a8_0": [],
"sharegpt_puvUs0e_0": [],
"sharegpt_xhPNTAJ_0": [],
"ultrachat_180729": [],
"ultrachat_198964": [],
"ultrachat_199519": [],
"ultrachat_274273": [],
"ultrachat_280826": [],
"ultrachat_446413": [],
"ultrachat_449001": [],
"ultrachat_467820": [],
"ultrachat_525872": [],
"ultrachat_564750": [],
"ultrachat_59250": [],
"ultrachat_97515": []
},
"question_id": "0e4e4c46",
"ranked_retrieved_hits": [
"sharegpt_SpgAMBx_0",
"sharegpt_XYnZHIX_10",
"sharegpt_XpiQ03D_0",
"ultrachat_59250",
"sharegpt_Wi7Op1u_39"
],
"scoring_family": "deterministic-retrieval",
"stored_records": [
"4842b73e",
"ultrachat_446413",
"e82ce7cd_3",
"ultrachat_274273",
"ultrachat_180729",
"6c0ba52d_1",
"ultrachat_467820",
"0773943b",
"answer_f2f998c7_1",
"ultrachat_97515",
"607b478d",
"ultrachat_199519",
"sharegpt_XpiQ03D_0",
"5d1c7bf7",
"b7702ee4_1",
"70cae729_3",
"9f1c9048",
"sharegpt_1YtwybC_15",
"sharegpt_SpgAMBx_0",
"55e0c6db_1",
"sharegpt_FQi5DlJ_48",
"a0cc90de",
"ultrachat_449001",
"b42cbaf0_1",
"sharegpt_HBnEfom_0",
"dfe8e126_1",
"ultrachat_59250",
"71d1e644",
"ultrachat_198964",
"998616b4_2",
"sharegpt_K8mXaIT_0",
"sharegpt_puvUs0e_0",
"debc34d7_1",
"cbd08e3c_abs_2",
"ultrachat_564750",
"17822471",
"8409f0d2_3",
"60876def_2",
"ultrachat_525872",
"sharegpt_xhPNTAJ_0",
"sharegpt_XOT5vdc_6",
"sharegpt_iUc80a8_0",
"sharegpt_Wi7Op1u_39",
"6702277b_1",
"ultrachat_280826",
"sharegpt_XYnZHIX_10",
"11e9d591_1",
"93f2334f_1",
"answer_f2f998c7_2"
]
}