[
 {
  "id": "T01",
  "answer_hash": "6da03608d44c",
  "correct": false,
  "reason": "The answer fails to mention both the specific model (RT-DETR V2) and the unsupported operation (torch.float64) that causes the crash.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer omits the specific cause (RT-DETR V2 invoking torch.float64, which MPS doesn't support), only vaguely attributing the crash to the MPS backend.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T02",
  "answer_hash": "c6e79db8fdc3",
  "correct": false,
  "reason": "The answer does not mention the required Keychain item name 'handoff-own-encryption-key'.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer instructs deleting a Keychain item named com.apple.continuity.identities/identity, not the required handoff-own-encryption-key item.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T03",
  "answer_hash": "143441aff859",
  "correct": false,
  "reason": "The answer fails to mention deleting `tmp/openreview_token.json`, instead suggesting running `openreview login`.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer instructs running `openreview login` instead of the required step of deleting `tmp/openreview_token.json` and retrying the command.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T04",
  "answer_hash": "c6d5632a72dc",
  "correct": false,
  "reason": "The answer claims not to have the specific interval data and suggests a 5-10 second delay instead of the required 2-3 second interval.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer does not state the required 2-3 second interval and instead suggests a different 5-10 second delay, contradicting the ground truth rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T05",
  "answer_hash": "2948d6934127",
  "correct": false,
  "reason": "The answer invents polling by creation_uuid rather than stating to send a GET request to fetch the list and locate the resource by content match.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer recommends polling a creation_uuid lookup endpoint or SSE/webhook, not the required GET list request followed by content matching to find the new resource's ID.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T06",
  "answer_hash": "c978d6db3372",
  "correct": false,
  "reason": "The answer fails to state the recommended default wake-up interval of 30 minutes (1800 seconds), stating instead that the information is not provided.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer fails to state the required default interval of about 30 minutes (1800s) and instead claims no guidance exists.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T07",
  "answer_hash": "cbe122831777",
  "correct": false,
  "reason": "The answer fails to mention that forcing determinism specifically on long text payloads can cause repetition, as required by the rubric.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer never mentions the key rule that forcing determinism on long text payloads can cause repetition; it instead gives generic reasons about exploration and brittleness.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T08",
  "answer_hash": "ecdd31bcac61",
  "correct": false,
  "reason": "The answer fails to state the recommended minimum padding of 24px and incorrectly claims the information is not provided.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer recommends 16px (or 20px) instead of the required minimum of 24px, contradicting the ground-truth rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T09",
  "answer_hash": "3c42fdc72f49",
  "correct": false,
  "reason": "The answer incorrectly specifies using a boolean value (`false`) instead of the required string value (`\"disabled\"`).",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer uses `web_search = false` with a boolean value, contradicting the required string value `\"disabled\"`.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T10",
  "answer_hash": "d816e57c710f",
  "correct": false,
  "reason": "The answer fails to state the limit of 50 pairs and claims the information is not provided.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer denies any documented limit and does not state the required maximum of 50 pairs.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T11",
  "answer_hash": "83b626477909",
  "correct": false,
  "reason": "The answer incorrectly claims the CLI continues with the default accelerator (usually T4), rather than sending it as A100.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer claims the unrecognized value is ignored and defaults to T4, contradicting the rule that it is sent as A100.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T12",
  "answer_hash": "9e43f746f878",
  "correct": false,
  "reason": "The answer incorrectly identifies the error as insufficient funds/sandbox quota exceeded and fails to mention that the account is unverified and requires verification in the DataForSEO panel.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer misidentifies 40104 as an insufficient funds/sandbox quota error and recommends switching to sandbox, rather than stating the account is authenticated but unverified and must complete verification in the DataForSEO panel.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T13",
  "answer_hash": "dede68534f79",
  "correct": false,
  "reason": "The answer fails to mention the `--execute` flag and that live collection defaults to a dry-run.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer claims the needed flag is --target, but the reference rule requires --execute because live collection is dry-run by default.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T14",
  "answer_hash": "7177d86a5440",
  "correct": false,
  "reason": "The answer provides an incorrect path (`~/Library/Calendars/<UUID>/Calendar.sqlitedb`) instead of `~/Library/Group Containers/group.com.apple.calendar/Calendar.sqlitedb` and fails to mention that the decoy file is a 0-byte file.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer gives the wrong path (~/Library/Calendars/<UUID>/Calendar.sqlitedb) and never states the correct live store path ~/Library/Group Containers/group.com.apple.calendar/Calendar.sqlitedb nor that the decoy is a 0-byte file.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T15",
  "answer_hash": "bd0a12c6b180",
  "correct": false,
  "reason": "The answer fails to mention creating a matplotlib structure draft and passing it as an input image (-i).",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer describes a text-model-then-prompt workflow and never mentions creating a matplotlib structure draft passed as the -i input image, which is the required rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T16",
  "answer_hash": "50f86dfd1356",
  "correct": false,
  "reason": "The answer fails to include the required '.qta' extension and incorrectly lists '.aac' and '.mp4' instead.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer lists .m4a, .aac, and .mp4 but omits the required .qta extension, contradicting the rule that sync reads only .qta and .m4a files.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T17",
  "answer_hash": "77620453284a",
  "correct": false,
  "reason": "The answer fails to mention pairing the Apple Watch with a Mac in Device Hub at least once, as required by the excerpt.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer describes enabling Developer Mode via the paired iPhone and Watch app, but omits the required step of pairing the watch with the Mac in Device Hub at least once.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T18",
  "answer_hash": "9d0dba1c5702",
  "correct": false,
  "reason": "The answer fails to mention that the CLI maps nulls to 0 and suggests a custom bash script instead of `scripts/request_metrics.py`.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer never mentions that the native CLI maps upstream nulls to 0 and instead recommends a custom check-idle.sh script rather than scripts/request_metrics.py.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T19",
  "answer_hash": "7cd7f17bee48",
  "correct": false,
  "reason": "The answer fails to mention monitoring the battery trend or rising battery percentage across snapshots or a watch window, and instead suggests checking status fields.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer recommends using a `status` sub-command and `dock_status`/`battery_state` fields, contradicting the ground truth rule to use the battery trend (rising percentage across snapshots or a watch window).",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T20",
  "answer_hash": "20e0afa44b5e",
  "correct": false,
  "reason": "The answer provides the incorrect flag `--include-subagents` instead of the exact flag `--include-subagent` specified in the excerpt.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer uses the wrong flag name `--include-subagents` instead of the required `--include-subagent`, and fabricates an incorrect mechanism for why sessions are skipped.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T21",
  "answer_hash": "14d28ec9260a",
  "correct": false,
  "reason": "The answer incorrectly claims there is no strict upper limit, missing the required limit of 22 words.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer denies any strict upper limit, directly contradicting the excerpt's hard gate of 22 words.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T22",
  "answer_hash": "822d739010ba",
  "correct": false,
  "reason": "The answer incorrectly identifies the option as `--base-dir` instead of `--source-root`.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer names the wrong option (`--base-dir` instead of `--source-root`), though it correctly states the fallback is the current working directory.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T23",
  "answer_hash": "05cb150ff261",
  "correct": false,
  "reason": "The answer incorrectly claims that volume and mute state can be retrieved, directly contradicting the ground truth that they are not readable and only PowerState is returned.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer claims volume and mute are readable via the status endpoint, directly contradicting the rule that they cannot be read and get_status returns PowerState only.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T24",
  "answer_hash": "477a80941a47",
  "correct": false,
  "reason": "The answer incorrectly identifies the exit code as 3 instead of 10 and suggests using a CLI flag instead of a manual human refresh.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer states exit code 3 instead of the required exit code 10 and prescribes a CLI re-login rather than a human cookie refresh.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T25",
  "answer_hash": "b4f986834ce9",
  "correct": false,
  "reason": "The answer fails to state the maximum duration of 7200 seconds (or 2 hours) and claims the material does not specify a limit.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer fails to state the required 7200-second (2-hour) limit and instead claims no maximum duration is specified.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T26",
  "answer_hash": "d422b15a3705",
  "correct": false,
  "reason": "The answer fails to state that the default timezone is UTC, claiming instead that the information is not available.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer fails to state the required default timezone of UTC, instead claiming the information is unavailable.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T27",
  "answer_hash": "ccace936f75b",
  "correct": false,
  "reason": "The answer incorrectly states that `--after` expects a Unix timestamp instead of explaining that it is a pagination cursor expecting a UUID value.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer claims --after expects a Unix timestamp, contradicting the ground truth that it is a pagination cursor expecting a UUID value.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T28",
  "answer_hash": "392a5e00e6b1",
  "correct": false,
  "reason": "The answer fails to mention that include and exclude domain filters cannot be combined in the CLI.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer does not state the key rule that --include-domain and --exclude-domain flags cannot be combined, instead suggesting site:/-site: operators in the query without mentioning the CLI flag constraint.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T29",
  "answer_hash": "e1b15001e5d8",
  "correct": false,
  "reason": "The answer states that the signal can reach +6 dBFS instead of mentioning that ffmpeg sums the channels at about +3 dB, and it fails to mention false clipping.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer claims the summed mono signal can reach almost +6 dBFS and never states the excerpt's key rule that correlated stereo sums at about +3 dB, nor that this causes false clipping reports.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T30",
  "answer_hash": "357ecfbe7770",
  "correct": false,
  "reason": "The answer provides generic debugging steps and possible causes rather than identifying the specific cause: stale session tokens in the Chrome user-data-dir causing the SSO SDK to repeatedly try and refresh the invalid token.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer lists generic SSO causes and never mentions the Chrome user-data-dir containing stale session tokens from a previous test, which is the specific ground-truth cause.",
  "judge_secondary": "deepseek-v4.1-flash"
 }
]