{
 "pool": {
  "n_skills": 139,
  "total_tokens_cl100k": 369217
 },
 "cells": {
  "deepseek-v4.1-flash:none": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 2,
   "accuracy": 0.06666666666666667,
   "mean_prompt_tokens": 495.26666666666665,
   "mean_completion_tokens": 616.1333333333333,
   "mean_latency_s": 6.859343806902568,
   "total_cost_usd": 0.0133191,
   "mean_cost_usd": 0.00044397,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.06666666666666667,
   "n_excl_web": 30
  },
  "deepseek-v4.1-flash:full_1": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 28,
   "accuracy": 0.9333333333333333,
   "mean_prompt_tokens": 2977.4333333333334,
   "mean_completion_tokens": 428.1333333333333,
   "mean_latency_s": 3.923152804374695,
   "total_cost_usd": 0.021104849999999998,
   "mean_cost_usd": 0.0007034949999999999,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.9333333333333333,
   "n_excl_web": 30
  },
  "deepseek-v4.1-flash:full_10": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 29,
   "accuracy": 0.9666666666666667,
   "mean_prompt_tokens": 16610.266666666666,
   "mean_completion_tokens": 450.3333333333333,
   "mean_latency_s": 4.509436098734538,
   "total_cost_usd": 0.08285219999999999,
   "mean_cost_usd": 0.0027617399999999995,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.9666666666666667,
   "n_excl_web": 30
  },
  "deepseek-v4.1-flash:full_30": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 30,
   "accuracy": 1.0,
   "mean_prompt_tokens": 64455.2,
   "mean_completion_tokens": 451.8333333333333,
   "mean_latency_s": 4.738598370552063,
   "total_cost_usd": 0.29818140000000004,
   "mean_cost_usd": 0.009939380000000001,
   "n_web_flagged": 0,
   "accuracy_excl_web": 1.0,
   "n_excl_web": 30
  },
  "deepseek-v4.1-flash:full_all": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 30,
   "accuracy": 1.0,
   "mean_prompt_tokens": 336205.26666666666,
   "mean_completion_tokens": 454.46666666666664,
   "mean_latency_s": 5.68160126209259,
   "total_cost_usd": 1.5211041,
   "mean_cost_usd": 0.05070347,
   "n_web_flagged": 0,
   "accuracy_excl_web": 1.0,
   "n_excl_web": 30
  },
  "deepseek-v4.1-flash:index": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 24,
   "accuracy": 0.8,
   "mean_prompt_tokens": 30690.6,
   "mean_completion_tokens": 972.1333333333333,
   "mean_latency_s": 18.51943678855896,
   "total_cost_usd": 0.39560609999999996,
   "mean_cost_usd": 0.013186869999999998,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.8,
   "n_excl_web": 30,
   "opened_target": 23,
   "mean_opens": 1.2,
   "wrong_breakdown": {
    "no_skill_opened": 2,
    "only_other_skills_opened": 4,
    "target_opened_but_wrong": 0,
    "error_no_answer": 0
   },
   "right_without_target": 1,
   "other_opened": [
    [
     "ai_session_search_archive",
     2
    ],
    [
     "skill_semantic_search",
     1
    ],
    [
     "koyeb",
     1
    ],
    [
     "download_and_transcribe",
     1
    ],
    [
     "online_media",
     1
    ],
    [
     "source_search",
     1
    ],
    [
     "skill_agentic_patrol",
     1
    ],
    [
     "skill_opencode_data",
     1
    ],
    [
     "skill_ai_usage_dashboard",
     1
    ],
    [
     "music_library_dedup",
     1
    ],
    [
     "procedural-video-frames",
     1
    ],
    [
     "skill_outlook",
     1
    ]
   ]
  },
  "deepseek-v4.1-flash:index_rep2": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 24,
   "accuracy": 0.8,
   "mean_prompt_tokens": 30114.0,
   "mean_completion_tokens": 898.0333333333333,
   "mean_latency_s": 10.964469480514527,
   "total_cost_usd": 0.39167759999999996,
   "mean_cost_usd": 0.013055919999999999,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.8,
   "n_excl_web": 30,
   "opened_target": 23,
   "mean_opens": 1.1,
   "wrong_breakdown": {
    "no_skill_opened": 2,
    "only_other_skills_opened": 4,
    "target_opened_but_wrong": 0,
    "error_no_answer": 0
   },
   "right_without_target": 1,
   "other_opened": [
    [
     "ai_session_search_archive",
     2
    ],
    [
     "skill_semantic_search",
     1
    ],
    [
     "download_and_transcribe",
     1
    ],
    [
     "online_media",
     1
    ],
    [
     "skill_agentic_patrol",
     1
    ],
    [
     "skill_opencode_data",
     1
    ],
    [
     "skill_ai_usage_dashboard",
     1
    ],
    [
     "music_library_dedup",
     1
    ],
    [
     "skill_outlook",
     1
    ]
   ]
  },
  "deepseek-v4.1-flash:index_fixed": {
   "n": 16,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 14,
   "accuracy": 0.875,
   "mean_prompt_tokens": 34319.9375,
   "mean_completion_tokens": 1083.6875,
   "mean_latency_s": 15.290099278092384,
   "total_cost_usd": 0.22277125,
   "mean_cost_usd": 0.013923203125,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.875,
   "n_excl_web": 16,
   "opened_target": 13,
   "mean_opens": 1.0,
   "wrong_breakdown": {
    "no_skill_opened": 0,
    "only_other_skills_opened": 2,
    "target_opened_but_wrong": 0,
    "error_no_answer": 0
   },
   "right_without_target": 1,
   "other_opened": [
    [
     "skill_semantic_search",
     1
    ],
    [
     "skill_session_keeper",
     1
    ],
    [
     "music_library_dedup",
     1
    ]
   ]
  },
  "gpt-5:none": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 0,
   "accuracy": 0.0,
   "mean_prompt_tokens": 107.4,
   "mean_completion_tokens": 3081.733333333333,
   "mean_latency_s": 31.987677637736002,
   "total_cost_usd": 0.9285474999999997,
   "mean_cost_usd": 0.030951583333333324,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.0,
   "n_excl_web": 30
  },
  "gpt-5:full_1": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 29,
   "accuracy": 0.9666666666666667,
   "mean_prompt_tokens": 2552.4,
   "mean_completion_tokens": 1837.7,
   "mean_latency_s": 30.37179694970449,
   "total_cost_usd": 0.6470249999999999,
   "mean_cost_usd": 0.021567499999999996,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.9666666666666667,
   "n_excl_web": 30
  },
  "gpt-5:full_10": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 27,
   "accuracy": 0.9,
   "mean_prompt_tokens": 16304.666666666666,
   "mean_completion_tokens": 1995.7,
   "mean_latency_s": 31.109782203038534,
   "total_cost_usd": 1.210135,
   "mean_cost_usd": 0.04033783333333333,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.9,
   "n_excl_web": 30
  },
  "gpt-5:full_30": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 28,
   "accuracy": 0.9333333333333333,
   "mean_prompt_tokens": 65781.13333333333,
   "mean_completion_tokens": 1978.6666666666667,
   "mean_latency_s": 35.896104788780214,
   "total_cost_usd": 3.0603925,
   "mean_cost_usd": 0.10201308333333332,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.9333333333333333,
   "n_excl_web": 30
  },
  "gpt-5:full_all": {
   "n": 30,
   "n_errors": 30,
   "errors_sample": [
    "HTTP 400: {\"detail\":\"Input tokens exceed the configured limit of 272000 tokens. Your messages resulted in 340087 tokens. Please reduce the length of the message",
    "HTTP 400: {\"detail\":\"Input tokens exceed the configured limit of 272000 tokens. Your messages resulted in 340091 tokens. Please reduce the length of the message",
    "HTTP 400: {\"detail\":\"Input tokens exceed the configured limit of 272000 tokens. Your messages resulted in 340093 tokens. Please reduce the length of the message"
   ],
   "correct": 0,
   "accuracy": 0.0,
   "mean_prompt_tokens": null,
   "mean_completion_tokens": null,
   "mean_latency_s": null,
   "total_cost_usd": 0,
   "mean_cost_usd": null,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.0,
   "n_excl_web": 30
  },
  "gpt-5:index": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 21,
   "accuracy": 0.7,
   "mean_prompt_tokens": 13958.9,
   "mean_completion_tokens": 2467.366666666667,
   "mean_latency_s": 36.067172185579935,
   "total_cost_usd": 1.26366875,
   "mean_cost_usd": 0.042122291666666666,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.7,
   "n_excl_web": 30,
   "opened_target": 22,
   "mean_opens": 0.9,
   "wrong_breakdown": {
    "no_skill_opened": 3,
    "only_other_skills_opened": 4,
    "target_opened_but_wrong": 2,
    "error_no_answer": 0
   },
   "right_without_target": 1,
   "other_opened": [
    [
     "ai_session_search_archive",
     2
    ],
    [
     "skill_session_keeper",
     1
    ],
    [
     "music_library_dedup",
     1
    ],
    [
     "skill_outlook",
     1
    ]
   ]
  },
  "gpt-5:index_rep2": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 21,
   "accuracy": 0.7,
   "mean_prompt_tokens": 14043.8,
   "mean_completion_tokens": 2770.5333333333333,
   "mean_latency_s": 32.44706338246663,
   "total_cost_usd": 1.3578025000000002,
   "mean_cost_usd": 0.04526008333333334,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.7,
   "n_excl_web": 30,
   "opened_target": 22,
   "mean_opens": 0.9,
   "wrong_breakdown": {
    "no_skill_opened": 4,
    "only_other_skills_opened": 4,
    "target_opened_but_wrong": 1,
    "error_no_answer": 0
   },
   "right_without_target": 0,
   "other_opened": [
    [
     "ai_session_search_archive",
     2
    ],
    [
     "skill_agentic_patrol",
     1
    ],
    [
     "music_library_dedup",
     1
    ],
    [
     "skill_outlook",
     1
    ]
   ]
  },
  "gpt-5:index_fixed": {
   "n": 17,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 14,
   "accuracy": 0.8235294117647058,
   "mean_prompt_tokens": 15934.529411764706,
   "mean_completion_tokens": 2858.764705882353,
   "mean_latency_s": 29.356163319419412,
   "total_cost_usd": 0.8245987499999999,
   "mean_cost_usd": 0.04850580882352941,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.8235294117647058,
   "n_excl_web": 17,
   "opened_target": 14,
   "mean_opens": 0.9411764705882353,
   "wrong_breakdown": {
    "no_skill_opened": 1,
    "only_other_skills_opened": 2,
    "target_opened_but_wrong": 0,
    "error_no_answer": 0
   },
   "right_without_target": 0,
   "other_opened": [
    [
     "skill_session_keeper",
     1
    ],
    [
     "music_library_dedup",
     1
    ]
   ]
  },
  "grok-4-fast:none": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 0,
   "accuracy": 0.0,
   "mean_prompt_tokens": 268.8333333333333,
   "mean_completion_tokens": 205.7,
   "mean_latency_s": 5.672764650980631,
   "total_cost_usd": 0.02550875,
   "mean_cost_usd": 0.0008502916666666666,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.0,
   "n_excl_web": 30
  },
  "grok-4-fast:full_1": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 26,
   "accuracy": 0.8666666666666667,
   "mean_prompt_tokens": 2615.4333333333334,
   "mean_completion_tokens": 130.6,
   "mean_latency_s": 3.6573637962341308,
   "total_cost_usd": 0.10787375,
   "mean_cost_usd": 0.003595791666666667,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.8666666666666667,
   "n_excl_web": 30
  },
  "grok-4-fast:full_10": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 26,
   "accuracy": 0.8666666666666667,
   "mean_prompt_tokens": 15499.533333333333,
   "mean_completion_tokens": 207.93333333333334,
   "mean_latency_s": 4.739522075653076,
   "total_cost_usd": 0.5968274999999998,
   "mean_cost_usd": 0.019894249999999995,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.8666666666666667,
   "n_excl_web": 30
  },
  "grok-4-fast:full_30": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 26,
   "accuracy": 0.8666666666666667,
   "mean_prompt_tokens": 61640.8,
   "mean_completion_tokens": 286.4,
   "mean_latency_s": 6.912205696105957,
   "total_cost_usd": 2.3330099999999994,
   "mean_cost_usd": 0.07776699999999998,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.8666666666666667,
   "n_excl_web": 30
  },
  "grok-4-fast:full_all": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 19,
   "accuracy": 0.6333333333333333,
   "mean_prompt_tokens": 321924.8333333333,
   "mean_completion_tokens": 383.6333333333333,
   "mean_latency_s": 16.565650081634523,
   "total_cost_usd": 12.10095375,
   "mean_cost_usd": 0.403365125,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.6333333333333333,
   "n_excl_web": 30
  },
  "grok-4-fast:index": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 13,
   "accuracy": 0.43333333333333335,
   "mean_prompt_tokens": 10923.466666666667,
   "mean_completion_tokens": 267.43333333333334,
   "mean_latency_s": 6.738795423507691,
   "total_cost_usd": 0.4296874999999999,
   "mean_cost_usd": 0.014322916666666663,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.43333333333333335,
   "n_excl_web": 30,
   "opened_target": 13,
   "mean_opens": 0.5666666666666667,
   "wrong_breakdown": {
    "no_skill_opened": 12,
    "only_other_skills_opened": 4,
    "target_opened_but_wrong": 1,
    "error_no_answer": 0
   },
   "right_without_target": 1,
   "other_opened": [
    [
     "skill_circle_post",
     1
    ],
    [
     "ai_session_search_archive",
     1
    ],
    [
     "delayed_execution",
     1
    ],
    [
     "skill_outlook",
     1
    ]
   ]
  },
  "grok-4-fast:index_rep2": {
   "n": 30,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 13,
   "accuracy": 0.43333333333333335,
   "mean_prompt_tokens": 10342.933333333332,
   "mean_completion_tokens": 267.3333333333333,
   "mean_latency_s": 7.20461102326711,
   "total_cost_usd": 0.40790999999999994,
   "mean_cost_usd": 0.013596999999999998,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.43333333333333335,
   "n_excl_web": 30,
   "opened_target": 13,
   "mean_opens": 0.5,
   "wrong_breakdown": {
    "no_skill_opened": 14,
    "only_other_skills_opened": 2,
    "target_opened_but_wrong": 1,
    "error_no_answer": 0
   },
   "right_without_target": 1,
   "other_opened": [
    [
     "skill_firecrawl",
     1
    ],
    [
     "skill_circle_post",
     1
    ]
   ]
  },
  "grok-4-fast:index_fixed": {
   "n": 26,
   "n_errors": 0,
   "errors_sample": [],
   "correct": 9,
   "accuracy": 0.34615384615384615,
   "mean_prompt_tokens": 11478.923076923076,
   "mean_completion_tokens": 304.38461538461536,
   "mean_latency_s": 8.612644323935875,
   "total_cost_usd": 0.39285,
   "mean_cost_usd": 0.015109615384615385,
   "n_web_flagged": 0,
   "accuracy_excl_web": 0.34615384615384615,
   "n_excl_web": 26,
   "opened_target": 10,
   "mean_opens": 0.5,
   "wrong_breakdown": {
    "no_skill_opened": 12,
    "only_other_skills_opened": 3,
    "target_opened_but_wrong": 2,
    "error_no_answer": 0
   },
   "right_without_target": 1,
   "other_opened": [
    [
     "skill_tavily",
     1
    ],
    [
     "skill_circle_post",
     1
    ],
    [
     "ai_session_search_archive",
     1
    ]
   ]
  }
 },
 "analysis": {
  "deepseek-v4.1-flash": {
   "index_fixed": {
    "failed_not_opened": 6,
    "now_correct": [
     "T01",
     "T22",
     "T24",
     "T29"
    ],
    "now_opened_target": [
     "T01",
     "T22",
     "T24",
     "T29"
    ],
    "still_opened_nothing": [],
    "passing_sample": 10,
    "passing_sample_still_correct": 10,
    "passing_sample_broke": []
   },
   "index_cue": {
    "n": 23,
    "correct": 22,
    "opened_target": 21,
    "opened_nothing": 2
   },
   "index_no_cue": {
    "n": 7,
    "correct": 2,
    "opened_target": 2,
    "opened_nothing": 1
   },
   "full_1_correct": 28,
   "dilution": {
    "full_10": {
     "lost_vs_full_1": [
      "T16"
     ],
     "gained_vs_full_1": [
      "T12",
      "T15"
     ],
     "lost_target_pos": {
      "T16": 4
     }
    },
    "full_30": {
     "lost_vs_full_1": [],
     "gained_vs_full_1": [
      "T12",
      "T15"
     ],
     "lost_target_pos": {}
    },
    "full_all": {
     "lost_vs_full_1": [],
     "gained_vs_full_1": [
      "T12",
      "T15"
     ],
     "lost_target_pos": {}
    }
   },
   "index_right_without_target": [
    "T17"
   ],
   "index_opened_counts": {
    "1": 22,
    "2": 3,
    "0": 3,
    "4": 2
   }
  },
  "gpt-5": {
   "index_fixed": {
    "failed_not_opened": 7,
    "now_correct": [
     "T10",
     "T23",
     "T24",
     "T29"
    ],
    "now_opened_target": [
     "T10",
     "T23",
     "T24",
     "T29"
    ],
    "still_opened_nothing": [
     "T28"
    ],
    "passing_sample": 10,
    "passing_sample_still_correct": 10,
    "passing_sample_broke": []
   },
   "index_cue": {
    "n": 23,
    "correct": 19,
    "opened_target": 20,
    "opened_nothing": 3
   },
   "index_no_cue": {
    "n": 7,
    "correct": 2,
    "opened_target": 2,
    "opened_nothing": 1
   },
   "full_1_correct": 29,
   "dilution": {
    "full_10": {
     "lost_vs_full_1": [
      "T15",
      "T23"
     ],
     "gained_vs_full_1": [],
     "lost_target_pos": {
      "T15": 0,
      "T23": 4
     }
    },
    "full_30": {
     "lost_vs_full_1": [
      "T14"
     ],
     "gained_vs_full_1": [],
     "lost_target_pos": {
      "T14": 29
     }
    }
   },
   "index_right_without_target": [
    "T17"
   ],
   "index_opened_counts": {
    "1": 25,
    "0": 4,
    "2": 1
   }
  },
  "grok-4-fast": {
   "index_fixed": {
    "failed_not_opened": 16,
    "now_correct": [
     "T19",
     "T26"
    ],
    "now_opened_target": [
     "T15",
     "T19",
     "T26",
     "T30"
    ],
    "still_opened_nothing": [
     "T06",
     "T09",
     "T10",
     "T12",
     "T14",
     "T24",
     "T27",
     "T28",
     "T29"
    ],
    "passing_sample": 10,
    "passing_sample_still_correct": 7,
    "passing_sample_broke": [
     "T03",
     "T07",
     "T23"
    ]
   },
   "index_cue": {
    "n": 23,
    "correct": 12,
    "opened_target": 12,
    "opened_nothing": 11
   },
   "index_no_cue": {
    "n": 7,
    "correct": 1,
    "opened_target": 1,
    "opened_nothing": 2
   },
   "full_1_correct": 26,
   "dilution": {
    "full_10": {
     "lost_vs_full_1": [
      "T15",
      "T26"
     ],
     "gained_vs_full_1": [
      "T02",
      "T08"
     ],
     "lost_target_pos": {
      "T15": 0,
      "T26": 2
     }
    },
    "full_30": {
     "lost_vs_full_1": [
      "T05",
      "T23",
      "T26"
     ],
     "gained_vs_full_1": [
      "T02",
      "T08",
      "T30"
     ],
     "lost_target_pos": {
      "T05": 17,
      "T23": 21,
      "T26": 9
     }
    },
    "full_all": {
     "lost_vs_full_1": [
      "T01",
      "T05",
      "T06",
      "T10",
      "T12",
      "T13",
      "T20",
      "T23",
      "T24",
      "T26",
      "T28"
     ],
     "gained_vs_full_1": [
      "T02",
      "T08",
      "T16",
      "T30"
     ],
     "lost_target_pos": {
      "T01": 29,
      "T05": 64,
      "T06": 23,
      "T10": 31,
      "T12": 67,
      "T13": 46,
      "T20": 33,
      "T23": 113,
      "T24": 119,
      "T26": 74,
      "T28": 51
     }
    }
   },
   "index_right_without_target": [
    "T17"
   ],
   "index_opened_counts": {
    "1": 17,
    "0": 13
   }
  }
 }
}