[
 {
  "id": "T01",
  "answer_hash": "98e4ae4369ea",
  "correct": true,
  "reason": "The answer accurately identifies RT-DETR V2 calling torch.float64 as the cause of the crash and provides the `--device cpu` flag as the workaround.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the crash is caused by Docling's RT-DETR V2 model calling torch.float64 unsupported by MPS and prescribes the --device cpu flag as the workaround.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T02",
  "answer_hash": "f8b83b52e5e4",
  "correct": true,
  "reason": "The answer correctly identifies the Keychain item as `handoff-own-encryption-key` and explains how to delete it and restart Handoff.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly names the Keychain item `handoff-own-encryption-key` and describes deleting it to reset the Mac's local Handoff identity key, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T03",
  "answer_hash": "b38fdeec33e1",
  "correct": true,
  "reason": "The answer accurately states to delete `tmp/openreview_token.json` and retry the query once.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states to delete `tmp/openreview_token.json` and retry once, matching the ground truth rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T04",
  "answer_hash": "65a2bd59c10e",
  "correct": true,
  "reason": "The answer accurately states the 2-3 second interval required by the excerpt.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the 2–3 second interval between successive downloads, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T05",
  "answer_hash": "0a44fbf17f29",
  "correct": true,
  "reason": "The answer correctly instructs not to rely on the POST response ID/uuid, and to send a GET request to fetch the list and locate the resource by matching its content.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states not to use the POST response's id and to follow with a GET, finding the new resource by content match, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T06",
  "answer_hash": "d491f64060b7",
  "correct": true,
  "reason": "The answer clearly states the recommended default wake-up interval of about 30 minutes (1800 seconds).",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the recommended default inspection wake-up interval is about 30 minutes (1800 seconds), matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T07",
  "answer_hash": "37b3a2e6b560",
  "correct": true,
  "reason": "The answer directly states that forcing determinism on long text payloads can cause repetition and notes provider sampling strategies.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states that forcing determinism on long text payloads can cause repetition and that providers/models have their own sampling strategies, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T08",
  "answer_hash": "e31a3a3e6d6d",
  "correct": true,
  "reason": "The answer correctly states that the recommended minimum padding is 24px or more.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the recommended minimum padding is 24px or more, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T09",
  "answer_hash": "304ab0cce8d4",
  "correct": true,
  "reason": "The answer correctly specifies setting `web_search = \"disabled\"` as a top-level key and states that the value must be a string.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the bare top-level key `web_search = \"disabled\"` with a string value and notes booleans/other values are rejected, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T10",
  "answer_hash": "880bead89066",
  "correct": true,
  "reason": "The answer correctly identifies that the maximum number allowed is 50 pairs.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the maximum is 50 pairs per delete manifest, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T11",
  "answer_hash": "21304cd6b901",
  "correct": true,
  "reason": "The answer accurately states that an unrecognized `--gpu` value is sent as A100 and is not rejected.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states that on CLI 0.7.4 an unrecognized --gpu value is sent as A100 and not rejected, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T12",
  "answer_hash": "9ea67bfd475b",
  "correct": true,
  "reason": "The answer correctly identifies that the account is authenticated but not verified and explains that the fix is to complete account verification in the DataForSEO panel.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states error 40104 means the account is authenticated but not verified and that the fix is to complete verification in the DataForSEO panel, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T13",
  "answer_hash": "6698a1d86df1",
  "correct": true,
  "reason": "The answer correctly identifies the `--execute` flag and explains that live collection is dry-run by default.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the required `--execute` flag and explains that live collection is dry-run by default, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T14",
  "answer_hash": "12d09334e909",
  "correct": true,
  "reason": "The answer correctly identifies the path under Group Containers and explains that the file under ~/Library/Calendars fails because it is a 0-byte file.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the correct path ~/Library/Group Containers/group.com.apple.calendar/Calendar.sqlitedb and explains the decoy is a 0-byte file, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T15",
  "answer_hash": "33bbd71b66e1",
  "correct": true,
  "reason": "The answer accurately states that one should first create a matplotlib structure draft, pass it as the input image (-i), and explains that generating from text alone yields unreliable results.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the required workflow—first create a matplotlib structure draft and pass it as the `-i` input image—and explains that text-prompt-only generation is unreliable, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T16",
  "answer_hash": "e9d419bd7b52",
  "correct": true,
  "reason": "The answer correctly states that the sync step only reads .m4a and .qta files.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states that sync reads only .m4a and .qta files, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T17",
  "answer_hash": "16a2970c7f69",
  "correct": true,
  "reason": "The answer accurately states both required steps: pairing the watch with the Mac via Device Hub and restarting the Apple Watch.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states both required steps from the excerpt: pairing the watch with the Mac via Device Hub at least once, and restarting the Apple Watch to make the Developer Mode toggle appear.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T18",
  "answer_hash": "f33e278f9a80",
  "correct": true,
  "reason": "The answer accurately states that the CLI maps upstream nulls to 0 and identifies scripts/request_metrics.py as the script to use instead.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the native CLI maps upstream nulls to 0 and names scripts/request_metrics.py as the required alternative for scale-to-zero/inactivity checks.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T19",
  "answer_hash": "c00f68d9a29f",
  "correct": true,
  "reason": "The answer accurately states that one should observe the battery trend to see if it is rising across a watch window or between snapshots.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the documented method is to infer docked/charging from the battery trend (rising across a watch window or between snapshots), matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T20",
  "answer_hash": "495f92b3cca8",
  "correct": true,
  "reason": "The answer clearly states that subagent fan-out sessions are skipped by default and explains how to include them using the `--include-subagent` flag.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states subagent fan-out sessions are skipped by default and that the --include-subagent flag is required to include them, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T21",
  "answer_hash": "47374799d47f",
  "correct": true,
  "reason": "The answer accurately states that the strict upper limit is 22 words.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the hard ceiling of 22 words for ordinary spoken sentences, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T22",
  "answer_hash": "5671e5aa8cd6",
  "correct": true,
  "reason": "The answer accurately identifies `--source-root` as the option to define the base directory and states that relative paths resolve against the CLI's current working directory if omitted.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the option is `--source-root` and that omitted paths resolve against the CLI's current working directory, matching the excerpt exactly.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T23",
  "answer_hash": "0742cbe98d58",
  "correct": true,
  "reason": "The answer accurately states that volume and mute state are not readable due to protocol limitations and that the status endpoint returns PowerState only.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states volume and mute cannot be read due to a protocol limitation and that get_status returns PowerState only, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T24",
  "answer_hash": "368ee86eea4d",
  "correct": true,
  "reason": "The answer correctly identifies exit code 10 and states that it requires a human to refresh the cookie.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states exit code 10 for expired-cookie auth failure and that it requires a human to refresh the cookie per docs/cookie_refresh.md, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T25",
  "answer_hash": "5366ccf50799",
  "correct": true,
  "reason": "The answer accurately states the limit as 7200 seconds (2 hours) based on the excerpt.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the maximum duration is 7200 seconds (2 hours) and that over-length files are rejected before a model loads, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T26",
  "answer_hash": "1958b0770f82",
  "correct": true,
  "reason": "The answer correctly states that the default timezone is UTC.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the default timezone is UTC for both calendar invite and calendar update, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T27",
  "answer_hash": "c7f0d96b1ffe",
  "correct": true,
  "reason": "The answer accurately states that `--after` is a pagination cursor expecting a UUID cursor value rather than an ISO timestamp filter.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states that --after is a pagination cursor expecting a UUID cursor value, not an ISO timestamp filter, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T28",
  "answer_hash": "95a6a609180a",
  "correct": true,
  "reason": "The answer clearly states that Firecrawl's `--include-domain` and `--exclude-domain` filters cannot be combined and provides the workaround of using the `-site:` operator directly within the query text.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer contradicts the ground truth by claiming the Tavily CLI supports combining include and exclude domain flags, whereas the excerpt states include and exclude domain filters cannot be combined.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T29",
  "answer_hash": "ebc5e8662af4",
  "correct": true,
  "reason": "The answer explains that ffmpeg sums correlated channels by about +3 dB, which can lead to false clipping reports.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states ffmpeg's downmix sums the channels (raising peak ~+3 dB) and produces a false clipping report, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T30",
  "answer_hash": "a287ff4b3505",
  "correct": true,
  "reason": "The answer accurately explains that the user-data-dir contains stale session tokens from a previous test and that the SSO SDK repeatedly attempts to use and refresh the invalid token after receiving 401 errors.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the Chrome user-data-dir contains stale session tokens from a previous test and that the SSO SDK repeatedly uses/refreshes the invalid token after 401s, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 }
]