Official benchmark setup required. Current development checks and adapted results are not official benchmark runs. Official comparisons must use the upstream code, datasets, scoring and prescribed setup, with versions and deviations disclosed. Protocol status →
Trace 71315a70
Stored context/evidence
["37d634c0_1","b2243528_1","e7999c4b_2","answer_c44b9df4_1","4ad82121_2","ultrachat_265676","sharegpt_ZUxBndS_26","sharegpt_GeNKkqW_0","6028dbf9","sharegpt_GqWEWQB_11","sharegpt_QFmKhh1_17","a08fbe88_1","ultrachat_42515","bcd5c30f_1","d8a74286","8cfa91c3_1","ultrachat_208238","ea27c6f5_1","sharegpt_1jEHGhM_0","1a43e6cf","773aebbd_1","sharegpt_OzK1xD9_0","sharegpt_VLn3Y3v_0","ultrachat_47868","sharegpt_rhjZmCm_0","464c3bb4","ultrachat_119182","sharegpt_djkhklV_0","44202c74_2","answer_c44b9df4_2","ultrachat_123587","sharegpt_SD7DDv2_0","sharegpt_I6xL7Th_0","ultrachat_433452","sharegpt_jyHBcl1_0","b5e94516_2","ultrachat_529773","6c0ba52d_2","sharegpt_F3hyS3i_0","b3953571","sharegpt_M6a9zYJ_0","ultrachat_543021","3d82ba51_3","ultrachat_511299","sharegpt_CjoQMV6_0","ultrachat_95799","c86a22a2_2","b89b0876","sharegpt_woQzv65_0","d1d3fd12_1","41743aae_1","ultrachat_399278","ultrachat_351984"]
Retrieved context/evidence
["sharegpt_I6xL7Th_0","sharegpt_VLn3Y3v_0","sharegpt_jyHBcl1_0","ultrachat_42515","sharegpt_OzK1xD9_0"]
Retrieval output (not a generated answer)
sharegpt_I6xL7Th_0
Complete stored trace (JSON)
{
"answer": "sharegpt_I6xL7Th_0",
"answer_session_ids": [
"answer_c44b9df4_1",
"answer_c44b9df4_2"
],
"gold_references": [
"answer_c44b9df4_1",
"answer_c44b9df4_2"
],
"oracle_has_answer_turns": {
"1a43e6cf": [],
"37d634c0_1": [],
"3d82ba51_3": [],
"41743aae_1": [],
"44202c74_2": [],
"464c3bb4": [],
"4ad82121_2": [],
"6028dbf9": [],
"6c0ba52d_2": [],
"773aebbd_1": [],
"8cfa91c3_1": [],
"a08fbe88_1": [],
"answer_c44b9df4_1": [
4
],
"answer_c44b9df4_2": [
0
],
"b2243528_1": [],
"b3953571": [],
"b5e94516_2": [],
"b89b0876": [],
"bcd5c30f_1": [],
"c86a22a2_2": [],
"d1d3fd12_1": [],
"d8a74286": [],
"e7999c4b_2": [],
"ea27c6f5_1": [],
"sharegpt_1jEHGhM_0": [],
"sharegpt_CjoQMV6_0": [],
"sharegpt_F3hyS3i_0": [],
"sharegpt_GeNKkqW_0": [],
"sharegpt_GqWEWQB_11": [],
"sharegpt_I6xL7Th_0": [],
"sharegpt_M6a9zYJ_0": [],
"sharegpt_OzK1xD9_0": [],
"sharegpt_QFmKhh1_17": [],
"sharegpt_SD7DDv2_0": [],
"sharegpt_VLn3Y3v_0": [],
"sharegpt_ZUxBndS_26": [],
"sharegpt_djkhklV_0": [],
"sharegpt_jyHBcl1_0": [],
"sharegpt_rhjZmCm_0": [],
"sharegpt_woQzv65_0": [],
"ultrachat_119182": [],
"ultrachat_123587": [],
"ultrachat_208238": [],
"ultrachat_265676": [],
"ultrachat_351984": [],
"ultrachat_399278": [],
"ultrachat_42515": [],
"ultrachat_433452": [],
"ultrachat_47868": [],
"ultrachat_511299": [],
"ultrachat_529773": [],
"ultrachat_543021": [],
"ultrachat_95799": []
},
"question_id": "71315a70",
"ranked_retrieved_hits": [
"sharegpt_I6xL7Th_0",
"sharegpt_VLn3Y3v_0",
"sharegpt_jyHBcl1_0",
"ultrachat_42515",
"sharegpt_OzK1xD9_0"
],
"scoring_family": "deterministic-retrieval",
"stored_records": [
"37d634c0_1",
"b2243528_1",
"e7999c4b_2",
"answer_c44b9df4_1",
"4ad82121_2",
"ultrachat_265676",
"sharegpt_ZUxBndS_26",
"sharegpt_GeNKkqW_0",
"6028dbf9",
"sharegpt_GqWEWQB_11",
"sharegpt_QFmKhh1_17",
"a08fbe88_1",
"ultrachat_42515",
"bcd5c30f_1",
"d8a74286",
"8cfa91c3_1",
"ultrachat_208238",
"ea27c6f5_1",
"sharegpt_1jEHGhM_0",
"1a43e6cf",
"773aebbd_1",
"sharegpt_OzK1xD9_0",
"sharegpt_VLn3Y3v_0",
"ultrachat_47868",
"sharegpt_rhjZmCm_0",
"464c3bb4",
"ultrachat_119182",
"sharegpt_djkhklV_0",
"44202c74_2",
"answer_c44b9df4_2",
"ultrachat_123587",
"sharegpt_SD7DDv2_0",
"sharegpt_I6xL7Th_0",
"ultrachat_433452",
"sharegpt_jyHBcl1_0",
"b5e94516_2",
"ultrachat_529773",
"6c0ba52d_2",
"sharegpt_F3hyS3i_0",
"b3953571",
"sharegpt_M6a9zYJ_0",
"ultrachat_543021",
"3d82ba51_3",
"ultrachat_511299",
"sharegpt_CjoQMV6_0",
"ultrachat_95799",
"c86a22a2_2",
"b89b0876",
"sharegpt_woQzv65_0",
"d1d3fd12_1",
"41743aae_1",
"ultrachat_399278",
"ultrachat_351984"
]
}