[
 {
  "id": "T01",
  "answer_hash": "7aa7a3a4ee7d",
  "correct": true,
  "reason": "The answer states the crash is caused by RT-DETR V2 invoking torch.float64 unsupported by MPS and prescribes the --device cpu flag as the workaround, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states that the crash is caused by the RT-DETR V2 model calling unsupported torch.float64 operations on MPS, and provides the --device cpu flag as the workaround.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T02",
  "answer_hash": "8283e7e11de8",
  "correct": true,
  "reason": "The answer explicitly names the Keychain item `handoff-own-encryption-key` and states that deleting it resets the Mac's local Handoff identity key, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies the specific Keychain item as handoff-own-encryption-key.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T03",
  "answer_hash": "5344df6a33d4",
  "correct": true,
  "reason": "The answer states deleting tmp/openreview_token.json and re-running the query, matching the required delete-and-retry step.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly instructs deleting `tmp/openreview_token.json` and retrying the query.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T04",
  "answer_hash": "9c61c345a0d2",
  "correct": true,
  "reason": "The answer clearly states the required 2 to 3 second interval between successive downloads, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly specifies the 2 to 3 second interval required between downloads.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T05",
  "answer_hash": "0e260e401492",
  "correct": true,
  "reason": "The answer explicitly recommends the POST-then-GET fetch-and-content-match pattern to obtain the real ID and warns against relying on the POST response's id/creation_uuid.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately instructs not to rely on the POST response/creation_uuid directly and explains how to send a GET request to fetch the list and identify the resource via content matching.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T06",
  "answer_hash": "0fbd4efd48b7",
  "correct": true,
  "reason": "The answer clearly states the recommended default wake-up interval is about 30 minutes (1800 seconds), matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the recommended default wake-up interval as about 30 minutes (1800 seconds).",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T07",
  "answer_hash": "f2b2ab91810a",
  "correct": true,
  "reason": "The answer explicitly states that forcing determinism on long text payloads causes repetition and that providers/models have their own sampling strategies, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly mentions that forcing determinism on long generation tasks can cause repetition, matching the rubric requirements.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T08",
  "answer_hash": "20e9bdf9c73c",
  "correct": true,
  "reason": "The answer explicitly states the recommended minimum padding is 24px or more, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly states the recommended minimum padding of 24px or more as specified in the excerpt.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T09",
  "answer_hash": "afde16fbcf9c",
  "correct": true,
  "reason": "The answer states the bare top-level key `web_search = \"disabled\"` with a string value and notes booleans/off/on/enabled are rejected, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly specifies setting `web_search = \"disabled\"` as a top-level key and notes that the value type must be a string.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T10",
  "answer_hash": "85b9240e4957",
  "correct": true,
  "reason": "The answer clearly states the maximum is 50 pairs and correctly notes larger sets must be split into independent plans with fresh review and authorization.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly states the limit of 50 pairs.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T11",
  "answer_hash": "a6ff6ea3d529",
  "correct": true,
  "reason": "The answer states the unrecognized --gpu value is not rejected and is sent as A100, matching the excerpt exactly.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states that the unrecognized `--gpu` value is not rejected and is sent as A100.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T12",
  "answer_hash": "8f2b121ae728",
  "correct": true,
  "reason": "The answer states error 40104 means the account is authenticated but not verified and instructs completing verification in the DataForSEO panel, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states that the account is authenticated but not verified, and that the fix is to complete verification in the DataForSEO panel.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T13",
  "answer_hash": "15c05e8c9bd1",
  "correct": true,
  "reason": "The answer states the --execute flag is required and explains that live collection is dry-run by default, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies the `--execute` flag and explains that live collection runs in dry-run mode by default.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T14",
  "answer_hash": "9a5620b46286",
  "correct": true,
  "reason": "The answer states the correct live path `~/Library/Group Containers/group.com.apple.calendar/Calendar.sqlitedb` and explains the failure because `~/Library/Calendars/Calendar.sqlitedb` is a 0-byte decoy file.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer provides the correct Group Containers path and explains that the file under ~/Library/Calendars is a 0-byte decoy.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T15",
  "answer_hash": "c3541ad0bd7b",
  "correct": true,
  "reason": "The answer clearly states the draft-first workflow: create a matplotlib structure draft, pass it as the input image (-i), and explains that text-only generation is unreliable, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states that you should generate a matplotlib structure draft first, pass it as the `-i` input image, and explains that generating from text alone produces unreliable results.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T16",
  "answer_hash": "2f255b1df62e",
  "correct": true,
  "reason": "The answer clearly states the sync only processes .m4a and .qta files, matching the ground truth rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states that the sync step only reads `.m4a` and `.qta` files.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T17",
  "answer_hash": "44c91141d541",
  "correct": true,
  "reason": "The answer states the watch must first be paired with the Mac in Device Hub/Xcode and then restarted, matching the excerpt's required steps.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly states that the user must pair the watch with the Mac in Device Hub and then restart the Apple Watch.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T18",
  "answer_hash": "b6a37f079439",
  "correct": true,
  "reason": "The answer clearly states the native CLI maps upstream nulls to 0 and recommends using scripts/request_metrics.py instead, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies that the standard CLI maps upstream nulls to 0 and explicitly states to use `scripts/request_metrics.py` instead.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T19",
  "answer_hash": "0467ecd4711a",
  "correct": true,
  "reason": "The answer's primary method explicitly states to determine charging/docked status by checking whether the battery percentage is rising across a watch window or between snapshots, matching the ground-truth rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly identifies checking the battery trend (rising battery percentage across a watch window or between snapshots) as the primary method.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T20",
  "answer_hash": "30839d816f05",
  "correct": true,
  "reason": "The answer clearly states subagent fan-out sessions are excluded/skipped by default and that the --include-subagent flag includes them, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states that subagent fan-out sessions are excluded by default and that the `--include-subagent` flag must be used to include them.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T21",
  "answer_hash": "b524a79b7ef5",
  "correct": true,
  "reason": "The answer explicitly states the hard gate of 22 words as the strict upper limit, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states that the strict upper limit is 22 words.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T22",
  "answer_hash": "566f7602a312",
  "correct": true,
  "reason": "The answer correctly identifies --source-root as the option and states that omitted paths resolve against the CLI's current working directory, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies `--source-root` as the option and states that paths resolve against the CLI's current working directory if it is omitted.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T23",
  "answer_hash": "19eb9fa63d98",
  "correct": true,
  "reason": "The answer clearly states volume and mute cannot be read due to a protocol limitation and that get_status returns PowerState only, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states that volume and mute state are not readable and that get_status returns PowerState only.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T24",
  "answer_hash": "d6fc8042df88",
  "correct": true,
  "reason": "The answer states exit code 10 and that resolution requires a human to refresh the cookie per docs/cookie_refresh.md, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies exit code 10 and states that a human must refresh the cookie.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T25",
  "answer_hash": "331ce9c81ff5",
  "correct": true,
  "reason": "The answer clearly states the 7200-second (2-hour) maximum and that longer files are rejected before models load, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly states the limit of 7,200 seconds (2 hours) before the file is rejected.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T26",
  "answer_hash": "1caa154308ab",
  "correct": true,
  "reason": "The answer clearly states the default timezone is UTC, matching the excerpt and rubric.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states that the default timezone is UTC.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T27",
  "answer_hash": "aeb1894fb7a7",
  "correct": true,
  "reason": "The answer clearly states that --after/--before are pagination cursors expecting a UUID cursor value, not ISO timestamps, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately explains that `--after` is a pagination cursor expecting a UUID cursor value rather than a timestamp.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T28",
  "answer_hash": "8fb802391a15",
  "correct": true,
  "reason": "The answer states that Firecrawl's --include-domain and --exclude-domain flags are mutually exclusive and instructs using the -site: operator inside the query text, matching the ground truth rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states that include and exclude domain flags cannot be combined in Firecrawl and instructs to use the `-site:` operator directly inside the query text.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T29",
  "answer_hash": "7139268a7895",
  "correct": true,
  "reason": "The answer states ffmpeg sums correlated stereo channels by roughly +3 dB and causes false clipping reports, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately explains that ffmpeg sums correlated stereo channels at about +3 dB, leading to false clipping reports.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T30",
  "answer_hash": "7505da042487",
  "correct": true,
  "reason": "The answer clearly states the loop is caused by stale session tokens in the browser profile/user-data-dir and that the SSO SDK repeatedly uses and refreshes the invalid token after 401s, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately explains that stale session tokens in the Chrome user-data-dir from a previous run cause the SSO SDK to repeatedly use and attempt to refresh the invalid token upon receiving 401 errors.",
  "judge_secondary": "gemini-3.8-flash"
 }
]