[
 {
  "id": "T01",
  "answer_hash": "bb807953273d",
  "correct": true,
  "reason": "The answer accurately states that RT-DETR V2 calls torch.float64 and provides the --device cpu flag as the workaround.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies RT-DETR V2's torch.float64 operation as the MPS crash cause and prescribes the --device cpu flag as the workaround.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T02",
  "answer_hash": "aa7699224ca1",
  "correct": false,
  "reason": "The answer identifies the Keychain item as `com.apple.handoff.identity` instead of the required `handoff-own-encryption-key`.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer names the Keychain item as `com.apple.handoff.identity` instead of the required `handoff-own-encryption-key`, contradicting the ground-truth rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T03",
  "answer_hash": "fe99d1b91799",
  "correct": false,
  "reason": "The answer fails to mention deleting `tmp/openreview_token.json` as specified in the reference document.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer recommends re-authenticating via a login command instead of the required step of deleting `tmp/openreview_token.json` and retrying.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T04",
  "answer_hash": "b74290648c31",
  "correct": false,
  "reason": "The answer does not state the required 2-3 second interval, stating instead that it does not have the information.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer does not state the required 2-3 second interval and instead says it lacks the information.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T05",
  "answer_hash": "b097b5cb3409",
  "correct": false,
  "reason": "The answer fails to mention sending a GET request to fetch the resource list and locating the resource by content match, instead describing how to sniff traffic with Playwright.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer recommends intercepting traffic and querying a status/lookup endpoint by creation_uuid, rather than the required approach of sending a GET to fetch the resource list and finding the new resource by content match.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T06",
  "answer_hash": "077155bcd823",
  "correct": false,
  "reason": "The answer specifies 30 seconds instead of the recommended 30 minutes (1800 seconds).",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer states 30 seconds, contradicting the required default of about 30 minutes (1800s).",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T07",
  "answer_hash": "6af5b7fead81",
  "correct": false,
  "reason": "The answer fails to state that forcing determinism on long text payloads may cause repetition or mention provider sampling strategies.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer never mentions that forcing determinism on long text payloads can cause repetition, which is the specific rule the excerpt requires.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T08",
  "answer_hash": "c5e329f4f796",
  "correct": true,
  "reason": "The answer correctly identifies the recommended minimum padding as at least 24px.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the recommended minimum padding of 24px or more, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T09",
  "answer_hash": "11e9596e03c1",
  "correct": false,
  "reason": "The answer does not provide the required configuration key, value, or type, stating that it lacks the necessary documentation.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer does not state the required key `web_search = \"disabled\"` or that the value must be a string, instead declining to answer.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T10",
  "answer_hash": "ad72628ba678",
  "correct": false,
  "reason": "The answer states that it does not have the information and fails to mention the limit of 50 pairs.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer does not state the 50-pair limit and instead says the information is unavailable.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T11",
  "answer_hash": "ef14e3b23bbb",
  "correct": true,
  "reason": "The answer accurately states that an unrecognized `--gpu` value is sent as A100 and not rejected.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the unrecognized --gpu value is sent as A100 and not rejected, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T12",
  "answer_hash": "8ba9a054c8f2",
  "correct": false,
  "reason": "The answer does not explain that error 40104 is caused by an unverified account or that the user needs to complete verification in the DataForSEO panel.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer does not state that error 40104 means the account is authenticated but unverified, nor does it mention completing verification in the DataForSEO panel.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T13",
  "answer_hash": "3499755a9758",
  "correct": true,
  "reason": "The answer correctly identifies the `--execute` flag and explains that collection is dry-run by default.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the `--execute` flag is needed because collection is dry-run by default, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T14",
  "answer_hash": "b2e3710be723",
  "correct": false,
  "reason": "The answer fails to identify the correct live store path, `~/Library/Group Containers/group.com.apple.calendar/Calendar.sqlitedb`, instead directing the user to sub-folders under `~/Library/Calendars`.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer recommends copying from ~/Library/Calendars/Calendar.sqlitedb (the decoy) instead of the required ~/Library/Group Containers/group.com.apple.calendar/Calendar.sqlitedb, contradicting the ground truth.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T15",
  "answer_hash": "a9646acaa831",
  "correct": false,
  "reason": "The answer fails to mention creating a matplotlib structure draft and passing it as the input image (-i).",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer never states the specific required workflow of first creating a matplotlib structure draft and passing it as the `-i` input image, instead giving a generic staged reference-driven workflow.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T16",
  "answer_hash": "8630ab36eee5",
  "correct": true,
  "reason": "The answer accurately states that the sync step only reads .m4a and .qta files, satisfying the rubric.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states that sync only reads .m4a and .qta files, matching the ground truth rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T17",
  "answer_hash": "1339183554f2",
  "correct": false,
  "reason": "The answer fails to mention pairing the watch with the Mac in Device Hub at least once, instead prescribing steps via the iPhone Watch app.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer describes enabling Developer Mode via the iPhone Watch app and restarting, but omits the required step of pairing the watch with the Mac in Device Hub at least once.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T18",
  "answer_hash": "810e9dd85654",
  "correct": true,
  "reason": "The answer correctly identifies that the CLI maps nulls to 0 and specifies using scripts/request_metrics.py.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the native CLI maps nulls to 0 and prescribes scripts/request_metrics.py as the lossless alternative, matching the rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T19",
  "answer_hash": "1b017f0d376f",
  "correct": false,
  "reason": "The answer lists battery trends as merely one of several speculative options alongside other fields and commands, rather than clearly identifying the battery trend across snapshots or a watch window as the definitive method.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer suggests inspecting various alternative fields and commands but does not state the required rule that docking/charging should be determined by a rising battery trend across snapshots or a watch window.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T20",
  "answer_hash": "36904057348e",
  "correct": false,
  "reason": "The answer fails to state that subagents are skipped by default and does not identify the specific required flag `--include-subagent`.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer never states the required flag `--include-subagent` (it guesses `--include-subagents` as an uncertain alternative) and instead recommends editing SQL, contradicting the documented rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T21",
  "answer_hash": "e982ded5798f",
  "correct": false,
  "reason": "The answer states the strict upper limit is 25 words instead of the required 22 words.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer states a 25-word cap, contradicting the excerpt's hard gate of 22 words.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T22",
  "answer_hash": "b4f814b4667e",
  "correct": true,
  "reason": "The answer correctly identifies `--source-root` as the command-line option and the current working directory as the default resolution path.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states both required elements: --source-root as the option and current working directory as the default resolution base, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T23",
  "answer_hash": "082adae16a33",
  "correct": false,
  "reason": "The answer fails to mention that the status endpoint only returns PowerState.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer says the skill lacks TV-specific endpoints but never states the key rule that volume/mute are unreadable due to protocol limitations and get_status returns PowerState only.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T24",
  "answer_hash": "c844964e858b",
  "correct": true,
  "reason": "The answer correctly identifies exit code 10 and specifies that a human must refresh the cookie.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states exit code 10 for expired-cookie auth failure and says a human must refresh the cookie, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T25",
  "answer_hash": "be572e97cd33",
  "correct": false,
  "reason": "The answer does not provide the required maximum duration of 7200 seconds (or 2 hours) and instead states that it lacks the information.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer fails to state the required 7200-second (2-hour) limit and instead says the constraint is unknown.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T26",
  "answer_hash": "3364bd7375e6",
  "correct": false,
  "reason": "The answer failed to identify UTC as the default timezone, stating instead that the information could not be determined.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer does not state the default timezone is UTC, instead claiming it cannot be determined.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T27",
  "answer_hash": "9ac38dd620cd",
  "correct": false,
  "reason": "The answer incorrectly claims that the `--after` flag does not exist, missing that it is a pagination cursor expecting a UUID value.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer claims --after is not a valid flag and that --created_after should be used, contradicting the rule that --after is a pagination cursor expecting a UUID value.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T28",
  "answer_hash": "1ba562bec64c",
  "correct": false,
  "reason": "The answer fails to mention that include and exclude domain filters cannot be combined in the CLI.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer does not state the key rule that include and exclude domain filters cannot be combined, nor does it specify using the -site: operator inside the query text as the required workaround.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T29",
  "answer_hash": "d35a08cce86a",
  "correct": false,
  "reason": "The answer fails to mention that ffmpeg sums the channels at about +3 dB and that this can cause false clipping to be reported.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer never states that ffmpeg's -ac 1 sums correlated stereo channels at about +3 dB, instead claiming a +6 dBFS sum with 0.5 scaling, and does not mention the false clipping report.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T30",
  "answer_hash": "45c74758352a",
  "correct": false,
  "reason": "The answer fails to mention the specific cause stated in the excerpt: that the Chrome user-data-dir contains stale session tokens from a previous test.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer describes a generic token-refresh loop and never identifies the specific root cause from the excerpt: stale session tokens in the CDP Chrome user-data-dir from a previous test.",
  "judge_secondary": "deepseek-v4.1-flash"
 }
]