Official benchmark setup required. Current development checks and adapted results are not official benchmark runs. Official comparisons must use the upstream code, datasets, scoring and prescribed setup, with versions and deviations disclosed. Protocol status →
Trace 0bc8ad92
Stored context/evidence
["cc36cd3e","d3cc5bdc_1","ultrachat_538119","717d1aea_1","eca613fc","0868982f_1","sharegpt_qHeD2Bc_0","sharegpt_xxk9JDz_21","c8e74cf7_1","answer_f4ea84fb_3","sharegpt_FqUHzEk_0","answer_f4ea84fb_2","ultrachat_276712","ultrachat_465468","fccc9400","sharegpt_glG44yq_11","f58c36b8_1","ultrachat_397480","ultrachat_525935","38310d87_2","675a3da1_1","sharegpt_KC1VaRe_0","bcc37468_2","89c1cdfa_2","f87fb137_1","ultrachat_478305","ultrachat_300585","answer_f4ea84fb_1","sharegpt_42b1pxm_0","86e218e3_2","sharegpt_scMKXPh_9","sharegpt_ItFPuha_0","baee81da","sharegpt_ErOTMZ3_223","ultrachat_282889","sharegpt_znI61kP_0","ultrachat_349938","sharegpt_ntu1Cds_0","sharegpt_3xB7vJ2_6","756a4a02_2","sharegpt_U8CljCM_291","be6f9404_2","sharegpt_9oSrWUg_5","sharegpt_Le7CqkS_0","881acf41_1","f66ca54c","01f1e64a_5","a268827b_2","sharegpt_vClQxDr_21"]
Retrieved context/evidence
["answer_f4ea84fb_3","sharegpt_qHeD2Bc_0","sharegpt_Le7CqkS_0"]
Retrieval output (not a generated answer)
answer_f4ea84fb_3
Complete stored trace (JSON)
{
"answer": "answer_f4ea84fb_3",
"answer_session_ids": [
"answer_f4ea84fb_3",
"answer_f4ea84fb_2",
"answer_f4ea84fb_1"
],
"gold_references": [
"answer_f4ea84fb_3",
"answer_f4ea84fb_2",
"answer_f4ea84fb_1"
],
"oracle_has_answer_turns": {
"01f1e64a_5": [],
"0868982f_1": [],
"38310d87_2": [],
"675a3da1_1": [],
"717d1aea_1": [],
"756a4a02_2": [],
"86e218e3_2": [],
"881acf41_1": [],
"89c1cdfa_2": [],
"a268827b_2": [],
"answer_f4ea84fb_1": [
0
],
"answer_f4ea84fb_2": [
0
],
"answer_f4ea84fb_3": [
6
],
"baee81da": [],
"bcc37468_2": [],
"be6f9404_2": [],
"c8e74cf7_1": [],
"cc36cd3e": [],
"d3cc5bdc_1": [],
"eca613fc": [],
"f58c36b8_1": [],
"f66ca54c": [],
"f87fb137_1": [],
"fccc9400": [],
"sharegpt_3xB7vJ2_6": [],
"sharegpt_42b1pxm_0": [],
"sharegpt_9oSrWUg_5": [],
"sharegpt_ErOTMZ3_223": [],
"sharegpt_FqUHzEk_0": [],
"sharegpt_ItFPuha_0": [],
"sharegpt_KC1VaRe_0": [],
"sharegpt_Le7CqkS_0": [],
"sharegpt_U8CljCM_291": [],
"sharegpt_glG44yq_11": [],
"sharegpt_ntu1Cds_0": [],
"sharegpt_qHeD2Bc_0": [],
"sharegpt_scMKXPh_9": [],
"sharegpt_vClQxDr_21": [],
"sharegpt_xxk9JDz_21": [],
"sharegpt_znI61kP_0": [],
"ultrachat_276712": [],
"ultrachat_282889": [],
"ultrachat_300585": [],
"ultrachat_349938": [],
"ultrachat_397480": [],
"ultrachat_465468": [],
"ultrachat_478305": [],
"ultrachat_525935": [],
"ultrachat_538119": []
},
"question_id": "0bc8ad92",
"ranked_retrieved_hits": [
"answer_f4ea84fb_3",
"sharegpt_qHeD2Bc_0",
"sharegpt_Le7CqkS_0"
],
"scoring_family": "deterministic-retrieval",
"stored_records": [
"cc36cd3e",
"d3cc5bdc_1",
"ultrachat_538119",
"717d1aea_1",
"eca613fc",
"0868982f_1",
"sharegpt_qHeD2Bc_0",
"sharegpt_xxk9JDz_21",
"c8e74cf7_1",
"answer_f4ea84fb_3",
"sharegpt_FqUHzEk_0",
"answer_f4ea84fb_2",
"ultrachat_276712",
"ultrachat_465468",
"fccc9400",
"sharegpt_glG44yq_11",
"f58c36b8_1",
"ultrachat_397480",
"ultrachat_525935",
"38310d87_2",
"675a3da1_1",
"sharegpt_KC1VaRe_0",
"bcc37468_2",
"89c1cdfa_2",
"f87fb137_1",
"ultrachat_478305",
"ultrachat_300585",
"answer_f4ea84fb_1",
"sharegpt_42b1pxm_0",
"86e218e3_2",
"sharegpt_scMKXPh_9",
"sharegpt_ItFPuha_0",
"baee81da",
"sharegpt_ErOTMZ3_223",
"ultrachat_282889",
"sharegpt_znI61kP_0",
"ultrachat_349938",
"sharegpt_ntu1Cds_0",
"sharegpt_3xB7vJ2_6",
"756a4a02_2",
"sharegpt_U8CljCM_291",
"be6f9404_2",
"sharegpt_9oSrWUg_5",
"sharegpt_Le7CqkS_0",
"881acf41_1",
"f66ca54c",
"01f1e64a_5",
"a268827b_2",
"sharegpt_vClQxDr_21"
]
}