{
 "pool": {
  "n_skills": 139,
  "total_tokens_cl100k": 325094
 },
 "cells": {
  "deepseek-v4.1-flash:none": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 2,
   "ungraded": 0,
   "judge": [
    "gemini-3.8-flash"
   ],
   "correct_secondary_judge": 2,
   "n_secondary_graded": 30,
   "judges_disagree": 2,
   "mean_prompt_tokens": 2181.1,
   "mean_completion_tokens": 829.1,
   "mean_latency_s": 7.919532306989034,
   "total_cost_usd": 0.05473875000000001,
   "mean_cost_usd": 0.0018246250000000003,
   "n_web_flagged": 0,
   "accuracy": 0.06666666666666667,
   "correct_excl_web": 2,
   "n_excl_web": 30
  },
  "deepseek-v4.1-flash:full_1": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 30,
   "ungraded": 0,
   "judge": [
    "gemini-3.8-flash"
   ],
   "correct_secondary_judge": 30,
   "n_secondary_graded": 30,
   "judges_disagree": 0,
   "mean_prompt_tokens": 4873.833333333333,
   "mean_completion_tokens": 495.03333333333336,
   "mean_latency_s": 6.838502589861552,
   "total_cost_usd": 0.060842850000000004,
   "mean_cost_usd": 0.002028095,
   "n_web_flagged": 0,
   "accuracy": 1.0,
   "correct_excl_web": 30,
   "n_excl_web": 30
  },
  "deepseek-v4.1-flash:full_all": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 30,
   "ungraded": 0,
   "judge": [
    "gemini-3.8-flash"
   ],
   "correct_secondary_judge": 29,
   "n_secondary_graded": 30,
   "judges_disagree": 1,
   "mean_prompt_tokens": 371712.5,
   "mean_completion_tokens": 526.3666666666667,
   "mean_latency_s": 7.856806993484497,
   "total_cost_usd": 1.7121808499999998,
   "mean_cost_usd": 0.05707269499999999,
   "n_web_flagged": 0,
   "accuracy": 1.0,
   "correct_excl_web": 30,
   "n_excl_web": 30
  },
  "deepseek-v4.1-flash:index": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 24,
   "ungraded": 0,
   "judge": [
    "gemini-3.8-flash"
   ],
   "correct_secondary_judge": 24,
   "n_secondary_graded": 30,
   "judges_disagree": 0,
   "mean_prompt_tokens": 32748.233333333334,
   "mean_completion_tokens": 1049.6333333333334,
   "mean_latency_s": 19.54762444496155,
   "total_cost_usd": 0.44626045000000003,
   "mean_cost_usd": 0.014875348333333335,
   "n_web_flagged": 1,
   "accuracy": 0.8,
   "correct_excl_web": 23,
   "n_excl_web": 29,
   "opened_target": 24,
   "mean_opens": 1.0666666666666667,
   "opened_nothing": 5,
   "wrong_breakdown": {
    "no_skill_opened": 5,
    "only_other_skills_opened": 1,
    "target_opened_but_wrong": 0,
    "error_no_answer": 0
   },
   "correct_given_target_opened": "24/24",
   "right_without_target": 0,
   "other_opened": [
    [
     "download_and_transcribe",
     1
    ],
    [
     "source_search",
     1
    ],
    [
     "skill_agentic_patrol",
     1
    ],
    [
     "bestpractice_ai_programming_mindset",
     1
    ],
    [
     "ai_session_export",
     1
    ],
    [
     "apple_calendar",
     1
    ],
    [
     "skill_session_keeper",
     1
    ],
    [
     "playwright_ajax_capture",
     1
    ]
   ]
  },
  "deepseek-v4.1-flash:index_rep2": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 23,
   "ungraded": 0,
   "judge": [
    "gemini-3.8-flash"
   ],
   "correct_secondary_judge": 23,
   "n_secondary_graded": 30,
   "judges_disagree": 0,
   "mean_prompt_tokens": 31744.833333333332,
   "mean_completion_tokens": 1040.7,
   "mean_latency_s": 11.769102017084757,
   "total_cost_usd": 0.43158435000000006,
   "mean_cost_usd": 0.014386145000000001,
   "n_web_flagged": 0,
   "accuracy": 0.7666666666666667,
   "correct_excl_web": 23,
   "n_excl_web": 30,
   "opened_target": 23,
   "mean_opens": 1.0,
   "opened_nothing": 5,
   "wrong_breakdown": {
    "no_skill_opened": 5,
    "only_other_skills_opened": 2,
    "target_opened_but_wrong": 0,
    "error_no_answer": 0
   },
   "correct_given_target_opened": "23/23",
   "right_without_target": 0,
   "other_opened": [
    [
     "skill_agentic_patrol",
     1
    ],
    [
     "bestpractice_temporal_info_verification",
     1
    ],
    [
     "ai_session_export",
     1
    ],
    [
     "workflow_parallel_subagents",
     1
    ],
    [
     "apple_calendar",
     1
    ],
    [
     "skill_session_keeper",
     1
    ],
    [
     "playwright_ajax_capture",
     1
    ]
   ]
  },
  "deepseek-v4.1-flash:index_fixed": {
   "n": 16,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 9,
   "ungraded": 0,
   "judge": [
    "gemini-3.8-flash"
   ],
   "correct_secondary_judge": 10,
   "n_secondary_graded": 16,
   "judges_disagree": 1,
   "mean_prompt_tokens": 29775.5625,
   "mean_completion_tokens": 1196.4375,
   "mean_latency_s": 13.423677071928978,
   "total_cost_usd": 0.23294714999999996,
   "mean_cost_usd": 0.014559196874999998,
   "n_web_flagged": 0,
   "accuracy": 0.5625,
   "correct_excl_web": 9,
   "n_excl_web": 16,
   "opened_target": 9,
   "mean_opens": 0.75,
   "opened_nothing": 4,
   "wrong_breakdown": {
    "no_skill_opened": 4,
    "only_other_skills_opened": 3,
    "target_opened_but_wrong": 0,
    "error_no_answer": 0
   },
   "correct_given_target_opened": "9/9",
   "right_without_target": 0,
   "other_opened": [
    [
     "bestpractice_temporal_info_verification",
     1
    ],
    [
     "ai_session_export",
     1
    ],
    [
     "skill_tavily",
     1
    ]
   ]
  },
  "gemini-3.8-flash:none": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 1,
   "ungraded": 0,
   "judge": [
    "deepseek-v4.1-flash"
   ],
   "correct_secondary_judge": 1,
   "n_secondary_graded": 30,
   "judges_disagree": 0,
   "mean_prompt_tokens": 97.7,
   "mean_completion_tokens": 1877.5333333333333,
   "mean_latency_s": 20.230100663503013,
   "total_cost_usd": 0.21342074999999996,
   "mean_cost_usd": 0.007114024999999999,
   "n_web_flagged": 0,
   "accuracy": 0.03333333333333333,
   "correct_excl_web": 1,
   "n_excl_web": 30
  },
  "gemini-3.8-flash:full_1": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 30,
   "ungraded": 0,
   "judge": [
    "deepseek-v4.1-flash"
   ],
   "correct_secondary_judge": 30,
   "n_secondary_graded": 30,
   "judges_disagree": 0,
   "mean_prompt_tokens": 2630.4666666666667,
   "mean_completion_tokens": 531.1333333333333,
   "mean_latency_s": 7.510306731859843,
   "total_cost_usd": 0.11893800000000002,
   "mean_cost_usd": 0.0039646,
   "n_web_flagged": 0,
   "accuracy": 1.0,
   "correct_excl_web": 30,
   "n_excl_web": 30
  },
  "gemini-3.8-flash:full_all": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 30,
   "ungraded": 0,
   "judge": [
    "deepseek-v4.1-flash"
   ],
   "correct_secondary_judge": 30,
   "n_secondary_graded": 30,
   "judges_disagree": 0,
   "mean_prompt_tokens": 349906.7,
   "mean_completion_tokens": 634.0666666666667,
   "mean_latency_s": 9.700375111897786,
   "total_cost_usd": 7.944233250000002,
   "mean_cost_usd": 0.2648077750000001,
   "n_web_flagged": 0,
   "accuracy": 1.0,
   "correct_excl_web": 30,
   "n_excl_web": 30
  },
  "gemini-3.8-flash:index": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 30,
   "ungraded": 0,
   "judge": [
    "deepseek-v4.1-flash"
   ],
   "correct_secondary_judge": 30,
   "n_secondary_graded": 30,
   "judges_disagree": 0,
   "mean_prompt_tokens": 15928.033333333333,
   "mean_completion_tokens": 719.0,
   "mean_latency_s": 11.433129930496216,
   "total_cost_usd": 0.43926824999999997,
   "mean_cost_usd": 0.014642275,
   "n_web_flagged": 0,
   "accuracy": 1.0,
   "correct_excl_web": 30,
   "n_excl_web": 30,
   "opened_target": 30,
   "mean_opens": 1.4666666666666666,
   "opened_nothing": 0,
   "wrong_breakdown": {
    "no_skill_opened": 0,
    "only_other_skills_opened": 0,
    "target_opened_but_wrong": 0,
    "error_no_answer": 0
   },
   "correct_given_target_opened": "30/30",
   "right_without_target": 0,
   "other_opened": [
    [
     "download_and_transcribe",
     2
    ],
    [
     "skill_agentic_patrol",
     1
    ],
    [
     "delayed_execution",
     1
    ],
    [
     "bestpractice_ai_programming_mindset",
     1
    ],
    [
     "opencode_cli",
     1
    ],
    [
     "skill_ai_agent_cli",
     1
    ],
    [
     "real_device",
     1
    ],
    [
     "ios_development",
     1
    ],
    [
     "ai_session_export",
     1
    ],
    [
     "workflow_bilibili_whisper_transcription",
     1
    ],
    [
     "skill_google_docs",
     1
    ],
    [
     "skill_tavily",
     1
    ],
    [
     "skill_session_keeper",
     1
    ]
   ]
  },
  "gemini-3.8-flash:index_rep2": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 29,
   "ungraded": 0,
   "judge": [
    "deepseek-v4.1-flash"
   ],
   "correct_secondary_judge": 30,
   "n_secondary_graded": 30,
   "judges_disagree": 1,
   "mean_prompt_tokens": 15818.2,
   "mean_completion_tokens": 717.4666666666667,
   "mean_latency_s": 12.24199005762736,
   "total_cost_usd": 0.43662449999999997,
   "mean_cost_usd": 0.014554149999999998,
   "n_web_flagged": 0,
   "accuracy": 0.9666666666666667,
   "correct_excl_web": 29,
   "n_excl_web": 30,
   "opened_target": 30,
   "mean_opens": 1.4,
   "opened_nothing": 0,
   "wrong_breakdown": {
    "no_skill_opened": 0,
    "only_other_skills_opened": 0,
    "target_opened_but_wrong": 1,
    "error_no_answer": 0
   },
   "correct_given_target_opened": "29/30",
   "right_without_target": 0,
   "other_opened": [
    [
     "download_and_transcribe",
     1
    ],
    [
     "skill_agentic_patrol",
     1
    ],
    [
     "workflow_parallel_subagents",
     1
    ],
    [
     "opencode_cli",
     1
    ],
    [
     "skill_ai_agent_cli",
     1
    ],
    [
     "ai_session_export",
     1
    ],
    [
     "copy_workflow",
     1
    ],
    [
     "skill_google_docs",
     1
    ],
    [
     "skill_tavily",
     1
    ],
    [
     "skill_session_keeper",
     1
    ],
    [
     "playwright_ajax_capture",
     1
    ],
    [
     "bestpractice_ai_debugging_diagnosis",
     1
    ]
   ]
  },
  "gemini-3.8-flash:index_fixed": {
   "n": 10,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 10,
   "ungraded": 0,
   "judge": [
    "deepseek-v4.1-flash"
   ],
   "correct_secondary_judge": 10,
   "n_secondary_graded": 10,
   "judges_disagree": 0,
   "mean_prompt_tokens": 14621.3,
   "mean_completion_tokens": 734.2,
   "mean_latency_s": 14.65102252960205,
   "total_cost_usd": 0.13719225000000002,
   "mean_cost_usd": 0.013719225000000002,
   "n_web_flagged": 0,
   "accuracy": 1.0,
   "correct_excl_web": 10,
   "n_excl_web": 10,
   "opened_target": 10,
   "mean_opens": 1.3,
   "opened_nothing": 0,
   "wrong_breakdown": {
    "no_skill_opened": 0,
    "only_other_skills_opened": 0,
    "target_opened_but_wrong": 0,
    "error_no_answer": 0
   },
   "correct_given_target_opened": "10/10",
   "right_without_target": 0,
   "other_opened": [
    [
     "skill_agentic_patrol",
     1
    ],
    [
     "ai_session_export",
     1
    ],
    [
     "skill_google_docs",
     1
    ]
   ]
  },
  "grok-4-fast:none": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 0,
   "ungraded": 0,
   "judge": [
    "gemini-3.8-flash"
   ],
   "correct_secondary_judge": 0,
   "n_secondary_graded": 30,
   "judges_disagree": 0,
   "mean_prompt_tokens": 277.6,
   "mean_completion_tokens": 164.9,
   "mean_latency_s": 5.151150186856587,
   "total_cost_usd": 0.0227775,
   "mean_cost_usd": 0.0007592499999999999,
   "n_web_flagged": 0,
   "accuracy": 0.0,
   "correct_excl_web": 0,
   "n_excl_web": 30
  },
  "grok-4-fast:full_1": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 30,
   "ungraded": 0,
   "judge": [
    "gemini-3.8-flash"
   ],
   "correct_secondary_judge": 30,
   "n_secondary_graded": 30,
   "judges_disagree": 0,
   "mean_prompt_tokens": 2571.4333333333334,
   "mean_completion_tokens": 85.33333333333333,
   "mean_latency_s": 4.272424570719401,
   "total_cost_usd": 0.10282875000000001,
   "mean_cost_usd": 0.0034276250000000005,
   "n_web_flagged": 0,
   "accuracy": 1.0,
   "correct_excl_web": 30,
   "n_excl_web": 30
  },
  "grok-4-fast:full_all": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 29,
   "ungraded": 0,
   "judge": [
    "gemini-3.8-flash"
   ],
   "correct_secondary_judge": 29,
   "n_secondary_graded": 30,
   "judges_disagree": 0,
   "mean_prompt_tokens": 318140.6,
   "mean_completion_tokens": 194.16666666666666,
   "mean_latency_s": 15.31236294110616,
   "total_cost_usd": 11.944835,
   "mean_cost_usd": 0.3981611666666666,
   "n_web_flagged": 0,
   "accuracy": 0.9666666666666667,
   "correct_excl_web": 29,
   "n_excl_web": 30
  },
  "grok-4-fast:index": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 9,
   "ungraded": 0,
   "judge": [
    "gemini-3.8-flash"
   ],
   "correct_secondary_judge": 9,
   "n_secondary_graded": 30,
   "judges_disagree": 0,
   "mean_prompt_tokens": 7925.266666666666,
   "mean_completion_tokens": 189.13333333333333,
   "mean_latency_s": 7.454881318410238,
   "total_cost_usd": 0.3113825,
   "mean_cost_usd": 0.010379416666666667,
   "n_web_flagged": 0,
   "accuracy": 0.3,
   "correct_excl_web": 9,
   "n_excl_web": 30,
   "opened_target": 9,
   "mean_opens": 0.3,
   "opened_nothing": 21,
   "wrong_breakdown": {
    "no_skill_opened": 21,
    "only_other_skills_opened": 0,
    "target_opened_but_wrong": 0,
    "error_no_answer": 0
   },
   "correct_given_target_opened": "9/9",
   "right_without_target": 0,
   "other_opened": []
  },
  "grok-4-fast:index_rep2": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 8,
   "ungraded": 0,
   "judge": [
    "gemini-3.8-flash"
   ],
   "correct_secondary_judge": 8,
   "n_secondary_graded": 30,
   "judges_disagree": 0,
   "mean_prompt_tokens": 7716.866666666667,
   "mean_completion_tokens": 182.2,
   "mean_latency_s": 6.8427833636601765,
   "total_cost_usd": 0.3030475000000001,
   "mean_cost_usd": 0.010101583333333336,
   "n_web_flagged": 0,
   "accuracy": 0.26666666666666666,
   "correct_excl_web": 8,
   "n_excl_web": 30,
   "opened_target": 8,
   "mean_opens": 0.26666666666666666,
   "opened_nothing": 22,
   "wrong_breakdown": {
    "no_skill_opened": 22,
    "only_other_skills_opened": 0,
    "target_opened_but_wrong": 0,
    "error_no_answer": 0
   },
   "correct_given_target_opened": "8/8",
   "right_without_target": 0,
   "other_opened": []
  },
  "grok-4-fast:index_fixed": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 10,
   "ungraded": 0,
   "judge": [
    "gemini-3.8-flash"
   ],
   "correct_secondary_judge": 10,
   "n_secondary_graded": 30,
   "judges_disagree": 0,
   "mean_prompt_tokens": 8591.933333333332,
   "mean_completion_tokens": 192.76666666666668,
   "mean_latency_s": 7.409258222579956,
   "total_cost_usd": 0.33665500000000004,
   "mean_cost_usd": 0.011221833333333335,
   "n_web_flagged": 0,
   "accuracy": 0.3333333333333333,
   "correct_excl_web": 10,
   "n_excl_web": 30,
   "opened_target": 10,
   "mean_opens": 0.36666666666666664,
   "opened_nothing": 19,
   "wrong_breakdown": {
    "no_skill_opened": 19,
    "only_other_skills_opened": 1,
    "target_opened_but_wrong": 0,
    "error_no_answer": 0
   },
   "correct_given_target_opened": "10/10",
   "right_without_target": 0,
   "other_opened": [
    [
     "ai_session_export",
     1
    ]
   ]
  }
 },
 "analysis": {
  "deepseek-v4.1-flash": {
   "index_fixed": {
    "failed_not_opened": 6,
    "fixed_now_correct": [
     "T15"
    ],
    "fixed_now_opened_target": [
     "T15"
    ],
    "fixed_still_opened_nothing": [
     "T05",
     "T26",
     "T29"
    ],
    "control_rep2_now_correct": [],
    "control_rep2_now_opened_target": [],
    "passing_sample": 10,
    "passing_sample_still_correct_fixed": 8,
    "passing_sample_still_correct_rep2": 9,
    "passing_sample_broke_fixed": [
     "T07",
     "T20"
    ]
   },
   "index_cue": {
    "n": 23,
    "correct": 20,
    "opened_target": 20,
    "opened_nothing": 3
   },
   "index_no_cue": {
    "n": 7,
    "correct": 4,
    "opened_target": 4,
    "opened_nothing": 2
   },
   "index_rep2_cue": {
    "n": 23,
    "correct": 19,
    "opened_target": 19,
    "opened_nothing": 3
   },
   "index_rep2_no_cue": {
    "n": 7,
    "correct": 4,
    "opened_target": 4,
    "opened_nothing": 2
   },
   "index_vs_rep2_flips": [
    "T07"
   ],
   "full_1_correct": 30,
   "dilution": {
    "full_all": {
     "lost_vs_full_1": [],
     "gained_vs_full_1": [],
     "lost_target_pos": {}
    }
   },
   "index_right_without_target": [],
   "index_opened_counts": {
    "1": 20,
    "3": 2,
    "0": 5,
    "2": 3
   }
  },
  "gemini-3.8-flash": {
   "index_fixed": {
    "failed_not_opened": 0,
    "fixed_now_correct": [],
    "fixed_now_opened_target": [],
    "fixed_still_opened_nothing": [],
    "control_rep2_now_correct": [],
    "control_rep2_now_opened_target": [],
    "passing_sample": 10,
    "passing_sample_still_correct_fixed": 10,
    "passing_sample_still_correct_rep2": 10,
    "passing_sample_broke_fixed": []
   },
   "index_cue": {
    "n": 23,
    "correct": 23,
    "opened_target": 23,
    "opened_nothing": 0
   },
   "index_no_cue": {
    "n": 7,
    "correct": 7,
    "opened_target": 7,
    "opened_nothing": 0
   },
   "index_rep2_cue": {
    "n": 23,
    "correct": 23,
    "opened_target": 23,
    "opened_nothing": 0
   },
   "index_rep2_no_cue": {
    "n": 7,
    "correct": 6,
    "opened_target": 7,
    "opened_nothing": 0
   },
   "index_vs_rep2_flips": [
    "T28"
   ],
   "full_1_correct": 30,
   "dilution": {
    "full_all": {
     "lost_vs_full_1": [],
     "gained_vs_full_1": [],
     "lost_target_pos": {}
    }
   },
   "index_right_without_target": [],
   "index_opened_counts": {
    "1": 20,
    "2": 6,
    "3": 4
   }
  },
  "grok-4-fast": {
   "index_fixed": {
    "failed_not_opened": 21,
    "fixed_now_correct": [
     "T01",
     "T05"
    ],
    "fixed_now_opened_target": [
     "T01",
     "T05"
    ],
    "fixed_still_opened_nothing": [
     "T02",
     "T03",
     "T04",
     "T06",
     "T07",
     "T09",
     "T14",
     "T15",
     "T17",
     "T19",
     "T21",
     "T23",
     "T25",
     "T26",
     "T27",
     "T28",
     "T29",
     "T30"
    ],
    "control_rep2_now_correct": [
     "T01"
    ],
    "control_rep2_now_opened_target": [
     "T01"
    ],
    "passing_sample": 9,
    "passing_sample_still_correct_fixed": 8,
    "passing_sample_still_correct_rep2": 7,
    "passing_sample_broke_fixed": [
     "T18"
    ]
   },
   "index_cue": {
    "n": 23,
    "correct": 9,
    "opened_target": 9,
    "opened_nothing": 14
   },
   "index_no_cue": {
    "n": 7,
    "correct": 0,
    "opened_target": 0,
    "opened_nothing": 7
   },
   "index_rep2_cue": {
    "n": 23,
    "correct": 8,
    "opened_target": 8,
    "opened_nothing": 15
   },
   "index_rep2_no_cue": {
    "n": 7,
    "correct": 0,
    "opened_target": 0,
    "opened_nothing": 7
   },
   "index_vs_rep2_flips": [
    "T01",
    "T10",
    "T12"
   ],
   "full_1_correct": 30,
   "dilution": {
    "full_all": {
     "lost_vs_full_1": [
      "T28"
     ],
     "gained_vs_full_1": [],
     "lost_target_pos": {
      "T28": 106
     }
    }
   },
   "index_right_without_target": [],
   "index_opened_counts": {
    "0": 21,
    "1": 9
   }
  }
 },
 "spotcheck": {
  "n": 20,
  "agree": 20,
  "disagreements": []
 },
 "meta": {
  "search_mentions": {
   "note": "Answers whose text says the gateway ran a web search (deepseek orchestrator), or whose trace flagged a web tool. The trace flag caught only 1 of these; the gateway seems to run keyword-triggered searches that its debug trace does not list. In every case the answer says the results were irrelevant.",
   "cells": {
    "deepseek-v4.1-flash:full_1": [
     {
      "id": "T05",
      "correct": true,
      "trace_flag": false
     }
    ],
    "deepseek-v4.1-flash:full_all": [
     {
      "id": "T05",
      "correct": true,
      "trace_flag": false
     },
     {
      "id": "T23",
      "correct": true,
      "trace_flag": false
     }
    ],
    "deepseek-v4.1-flash:index": [
     {
      "id": "T05",
      "correct": false,
      "trace_flag": false
     },
     {
      "id": "T07",
      "correct": true,
      "trace_flag": true
     },
     {
      "id": "T23",
      "correct": true,
      "trace_flag": false
     },
     {
      "id": "T26",
      "correct": false,
      "trace_flag": false
     }
    ],
    "deepseek-v4.1-flash:index_fixed": [
     {
      "id": "T05",
      "correct": false,
      "trace_flag": false
     },
     {
      "id": "T26",
      "correct": false,
      "trace_flag": false
     }
    ],
    "deepseek-v4.1-flash:index_rep2": [
     {
      "id": "T05",
      "correct": false,
      "trace_flag": false
     },
     {
      "id": "T19",
      "correct": true,
      "trace_flag": false
     },
     {
      "id": "T23",
      "correct": true,
      "trace_flag": false
     },
     {
      "id": "T26",
      "correct": false,
      "trace_flag": false
     }
    ],
    "deepseek-v4.1-flash:none": [
     {
      "id": "T05",
      "correct": false,
      "trace_flag": false
     },
     {
      "id": "T23",
      "correct": false,
      "trace_flag": false
     },
     {
      "id": "T26",
      "correct": false,
      "trace_flag": false
     }
    ],
    "gemini-3.8-flash:index": [
     {
      "id": "T28",
      "correct": true,
      "trace_flag": false
     }
    ],
    "gemini-3.8-flash:index_rep2": [
     {
      "id": "T28",
      "correct": false,
      "trace_flag": false
     }
    ],
    "gemini-3.8-flash:none": [
     {
      "id": "T28",
      "correct": false,
      "trace_flag": false
     }
    ],
    "grok-4-fast:index": [
     {
      "id": "T09",
      "correct": false,
      "trace_flag": false
     }
    ],
    "grok-4-fast:index_fixed": [
     {
      "id": "T09",
      "correct": false,
      "trace_flag": false
     },
     {
      "id": "T28",
      "correct": false,
      "trace_flag": false
     }
    ]
   }
  },
  "date": "2026-10-14",
  "gateway": "AI Builder Space (OpenAI-compatible)",
  "models": {
   "deepseek-v4.1-flash": "gateway id deepseek-v4-flash, temperature 0, index via READ_SKILL text protocol",
   "gemini-3.8-flash": "reasoning model, temperature 0 requested, index via READ_SKILL text protocol (client tools return an empty message)",
   "grok-4-fast": "served as grok-4.3 (non-reasoning), temperature 0, index via native tool calls"
  },
  "judges": {
   "primary": "gemini-3.8-flash grades deepseek and grok; deepseek-v4.1-flash grades gemini",
   "secondary": "deepseek grades deepseek and grok; gemini grades gemini"
  },
  "task_generator": "gemini-3.8-flash",
  "description_writer": "gemini-3.8-flash",
  "pool": {
   "n_skills": 139,
   "total_tokens_cl100k": 325094,
   "total_chars": 1445885,
   "index_tokens_cl100k": 5316,
   "pool_tokens_by_model_full_all_prompt": {
    "deepseek-v4.1-flash": 336479,
    "gemini-3.8-flash": 349907,
    "grok-4-fast": 318141
   },
   "actions": {
    "unchanged (en mirror identical)": 2,
    "unchanged (same text as zh pool)": 84,
    "swapped to en mirror": 30,
    "en mirror (zh copy was already mostly English)": 1,
    "swapped to skills_en": 6,
    "skills_en (zh copy was already mostly English)": 1,
    "replacement (English public skill)": 15
   }
  },
  "context_limit": "No model was blocked: all three accepted the full pool (318K-350K prompt tokens) on all 30 tasks.",
  "prices_per_1M_implied_by_gateway_trace": {
   "gemini-3.8-flash": [
    0.75,
    3.75
   ],
   "grok-4-fast": [
    1.25,
    2.5
   ],
   "deepseek-v4.1-flash": "trace value used as is (~0.15 input)"
  },
  "gemini_nudge": "Gateway bug: for gemini, a final user message containing trigger words (now, today, latest, current, price, weather, ...) returns HTTP 400 \"Requests ending with a model turn are not supported\". Workaround: append one user turn \"Please respond to the message above.\" Applied on 3/30 questions in none/full_1/full_all and on 29/30 tasks in index (skill text returned as a user message).",
  "spend_usd_2_8_total": 34.46
 }
}