[
 {
  "id": "T01",
  "answer_hash": "fd170f973849",
  "correct": false,
  "reason": "The answer fails to mention RT-DETR V2 and torch.float64 as required by the rubric.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer correctly gives the `--device cpu` workaround but misidentifies the cause, failing to mention RT-DETR V2 and torch.float64 as the specific unsupported operation.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T02",
  "answer_hash": "ef25c8d4db07",
  "correct": false,
  "reason": "The answer fails to mention the required Keychain item named 'handoff-own-encryption-key', instead suggesting 'com.apple.continuity'.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer instructs deleting com.apple.continuity* items, not the required Keychain item named handoff-own-encryption-key.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T03",
  "answer_hash": "b078249a8cce",
  "correct": false,
  "reason": "The answer fails to mention deleting `tmp/openreview_token.json`, which is the specific required step specified in the excerpt.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer recommends re-authenticating with a login command instead of the required step of deleting `tmp/openreview_token.json` and retrying.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T04",
  "answer_hash": "341d01f56b56",
  "correct": false,
  "reason": "The answer does not state the required 2-3 second interval, stating instead that it lacks the information.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer does not state the required 2-3 second interval and instead says it lacks the information.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T05",
  "answer_hash": "7c3e4d83f09a",
  "correct": false,
  "reason": "The answer fails to mention sending a GET request to fetch the list and finding the resource by content match, instead suggesting intercepting network traffic with Playwright.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer recommends capturing XHR traffic and re-issuing a request with the creation_uuid, rather than the required approach of sending a GET to fetch the list and finding the new resource by content match.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T06",
  "answer_hash": "077155bcd823",
  "correct": false,
  "reason": "The answer states the default wake-up interval is 30 seconds instead of the recommended 30 minutes (1800s).",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer states 30 seconds, contradicting the required default of about 30 minutes (1800s).",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T07",
  "answer_hash": "1064b9638f40",
  "correct": false,
  "reason": "The answer fails to mention that forcing determinism on long text payloads specifically causes repetition, as required by the rubric.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer never mentions that forcing determinism on long text payloads can cause repetition, which is the specific rule the excerpt requires.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T08",
  "answer_hash": "b700f2d40f9f",
  "correct": true,
  "reason": "The answer correctly states the recommended minimum padding of 24px (or more) as specified in the excerpt and rubric.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the recommended minimum padding is 24px, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T09",
  "answer_hash": "eec62fb78e76",
  "correct": false,
  "reason": "The answer does not provide the required configuration key, value, or value type, stating that it lacks the necessary information.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer declines to provide the configuration, failing to state the required key `web_search = \"disabled\"` or that the value must be a string.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T10",
  "answer_hash": "62297e42559d",
  "correct": true,
  "reason": "The answer clearly states that the limit is 50 pairs, which matches the excerpt and rubric.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the maximum is 50 pairs, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T11",
  "answer_hash": "960558934bd7",
  "correct": true,
  "reason": "The answer accurately states that an unrecognized --gpu value is sent as A100 and not rejected.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the unrecognized --gpu value is sent as A100 and not rejected, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T12",
  "answer_hash": "fe0e3f566fa0",
  "correct": true,
  "reason": "The answer accurately states that the error is caused by the account being authenticated but not verified, and that the fix is to complete verification in the DataForSEO panel.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the account is authenticated but not verified and instructs completing verification in the DataForSEO panel, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T13",
  "answer_hash": "bb3bcb4fe32e",
  "correct": true,
  "reason": "The answer correctly identifies the `--execute` flag and explains that live collection is dry-run by default.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the `--execute` flag is required because live collection is dry-run by default, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T14",
  "answer_hash": "cb2a047bd83a",
  "correct": false,
  "reason": "The answer fails to mention the correct path `~/Library/Group Containers/group.com.apple.calendar/Calendar.sqlitedb` and does not explain that the file under `~/Library/Calendars` failed because it is a 0-byte decoy file.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer recommends copying from ~/Library/Calendars/Calendar.sqlitedb (or a UUID subfolder) instead of the correct live store path ~/Library/Group Containers/group.com.apple.calendar/Calendar.sqlitedb, and never mentions that the ~/Library/Calendars file is a 0-byte decoy.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T15",
  "answer_hash": "3be7a1b0a72b",
  "correct": false,
  "reason": "The answer fails to mention creating a matplotlib structure draft and passing it as an input image (-i).",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer never mentions creating a matplotlib structure draft or passing it as the -i input image, instead proposing a text-only blueprint workflow that contradicts the required rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T16",
  "answer_hash": "805778a1855a",
  "correct": true,
  "reason": "The answer accurately states that sync only reads .qta and .m4a files.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states that sync only reads .m4a and .qta files, matching the ground truth rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T17",
  "answer_hash": "6e66643411e8",
  "correct": false,
  "reason": "The answer fails to mention pairing the watch with the Mac in Device Hub at least once.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer omits the required step of pairing the watch with the Mac in Device Hub at least once, instead describing an iPhone Watch app procedure that contradicts the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T18",
  "answer_hash": "092f56daa49d",
  "correct": true,
  "reason": "The answer accurately states that the standard CLI maps nulls to 0 and correctly identifies `scripts/request_metrics.py` as the script to use instead.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the native CLI maps nulls to 0 and prescribes scripts/request_metrics.py for the inactivity check, matching the rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T19",
  "answer_hash": "2c71a850227c",
  "correct": false,
  "reason": "The answer fails to mention observing the battery trend (rising battery percentage across snapshots or a watch window) to determine docking and charging status.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer never mentions using the battery trend (rising percentage across snapshots or a watch window), instead suggesting searching for alternative fields, which contradicts the ground-truth rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T20",
  "answer_hash": "0d5999783050",
  "correct": false,
  "reason": "The answer fails to mention the `--include-subagent` flag required to include subagent fan-out sessions, instead inventing separate directory paths and CLI flags.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer never mentions the required `--include-subagent` flag or that subagent sessions are skipped by default; it invents a `--fanout` flag instead.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T21",
  "answer_hash": "3e6be96055cc",
  "correct": false,
  "reason": "The answer states the upper limit is 25 words instead of the required 22 words.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer states a 25-word cap, contradicting the excerpt's hard gate of 22 words.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T22",
  "answer_hash": "207d82b0e02d",
  "correct": true,
  "reason": "The answer correctly identifies '--source-root' as the option and specifies that paths resolve against the CLI's current working directory when omitted.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly names --source-root and states that omitted paths resolve against the CLI's current working directory, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T23",
  "answer_hash": "848505e49882",
  "correct": false,
  "reason": "The answer does not state that volume and mute state cannot be read or that get_status only returns PowerState.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer does not state that volume and mute cannot be read due to protocol limitations nor that get_status returns only PowerState; it instead says the schema must be inspected.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T24",
  "answer_hash": "45584cb8bc58",
  "correct": true,
  "reason": "The answer accurately states exit code 10 and that the resolution requires a human to refresh the cookie.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states exit code 10 for expired-cookie auth failure and says a human must refresh the cookie, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T25",
  "answer_hash": "a626a2fdd372",
  "correct": false,
  "reason": "The answer does not provide the required duration limit of 7200 seconds (or 2 hours), stating instead that it lacks this information.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer does not state the 7200-second (2-hour) limit and instead claims no duration restriction information is available.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T26",
  "answer_hash": "6f8be51a1c75",
  "correct": false,
  "reason": "The answer claims the information is not available and fails to identify UTC as the default timezone.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer fails to state the required default timezone of UTC, instead saying it cannot be determined.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T27",
  "answer_hash": "96c99151e56a",
  "correct": false,
  "reason": "The answer incorrectly claims that the `--after` flag does not exist, rather than explaining that it is a pagination cursor expecting a UUID value.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer claims --after is an unrecognized flag and not supported, contradicting the ground truth that it is a pagination cursor expecting a UUID value.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T28",
  "answer_hash": "0e52fdeedee0",
  "correct": false,
  "reason": "The answer does not provide the rule or state that include and exclude domain filters cannot be combined, nor does it mention using the `-site:` operator in the query text.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer does not state the rule that include and exclude domain filters cannot be combined, nor does it mention using the -site: operator inside the query text.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T29",
  "answer_hash": "85e57a19f6b8",
  "correct": false,
  "reason": "The answer claims downmixing causes a +6 dBFS peak increase instead of stating that ffmpeg sums the channels at about +3 dB, and it fails to mention that this causes false clipping.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer claims the peak can double (+6 dB) from phase-coherent summing, contradicting the excerpt's rule that correlated stereo sums at about +3 dB, and it never mentions the false clipping report.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T30",
  "answer_hash": "aa96afb18900",
  "correct": false,
  "reason": "The answer fails to mention that the issue is caused by stale session tokens from a previous test remaining in the Chrome user-data-dir.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer describes a generic token-refresh loop and never mentions the specific root cause from the excerpt: stale session tokens in the Chrome user-data-dir from a previous test.",
  "judge_secondary": "deepseek-v4.1-flash"
 }
]