[
 {
  "id": "T01",
  "answer_hash": "3313eac494c5",
  "correct": true,
  "reason": "The answer accurately mentions the RT-DETR V2 model, the unsupported torch.float64 call, and the `--device cpu` flag.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the crash is caused by Docling's RT-DETR V2 model calling torch.float64 unsupported by MPS and prescribes the --device cpu flag as the workaround, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T02",
  "answer_hash": "4e5d7a5dfdf3",
  "correct": true,
  "reason": "The answer correctly identifies the Keychain item as `handoff-own-encryption-key` and advises restarting Handoff.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly identifies the Keychain item handoff-own-encryption-key and instructs deleting it then restarting Handoff to reset the local identity key, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T03",
  "answer_hash": "90be3f864f42",
  "correct": true,
  "reason": "The answer explicitly instructs to delete `tmp/openreview_token.json` and retry the query once.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states to delete `tmp/openreview_token.json` and retry the command once, matching the ground truth rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T04",
  "answer_hash": "56bc4aa8d33c",
  "correct": true,
  "reason": "The answer correctly specifies the 2–3 second interval as required by the excerpt and rubric.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the required 2–3 second interval between successive downloads, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T05",
  "answer_hash": "d60faf0416f3",
  "correct": true,
  "reason": "The answer clearly states not to rely on the POST response/creation_uuid and explains how to fetch the list via GET and find the resource by content match.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states not to use the POST response's creation_uuid as the ID and prescribes the POST→GET→content-match pattern to obtain the real ID, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T06",
  "answer_hash": "1a37907e7b16",
  "correct": true,
  "reason": "The answer clearly states the recommended default wake-up interval as about 30 minutes (1800 seconds).",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the recommended default inspection wake-up interval is about 30 minutes (1800 seconds), matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T07",
  "answer_hash": "a58fcc4e52d1",
  "correct": true,
  "reason": "The answer accurately states that forcing determinism on long text payloads can cause repetition, matching the rubric requirements.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states that forcing determinism on long text payloads can cause repetition and that providers have their own sampling strategies, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T08",
  "answer_hash": "e3ad57decab5",
  "correct": true,
  "reason": "The answer clearly states the recommended minimum padding of 24px or more as specified in the excerpt.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the recommended minimum padding is 24px or more, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T09",
  "answer_hash": "67d23d6f7df1",
  "correct": true,
  "reason": "The answer correctly specifies setting `web_search = \"disabled\"` as a top-level key and states that the value type must be a string.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the bare top-level key `web_search = \"disabled\"` with a string value and notes booleans/other values are rejected, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T10",
  "answer_hash": "af5442f17030",
  "correct": true,
  "reason": "The answer directly states that the limit is 50 pairs per delete manifest.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the maximum is 50 pairs per delete manifest, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T11",
  "answer_hash": "30f7faf88716",
  "correct": true,
  "reason": "The answer accurately states that an unrecognized `--gpu` value is not rejected and is sent as A100.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the unrecognized --gpu value is not rejected and is silently sent as A100, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T12",
  "answer_hash": "f2248369cea0",
  "correct": true,
  "reason": "The answer accurately states that the account is authenticated but not verified and explains that the fix is to complete verification in the DataForSEO panel.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states error 40104 means the account is authenticated but not verified and instructs completing verification in the DataForSEO panel, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T13",
  "answer_hash": "4e2db81c0203",
  "correct": true,
  "reason": "The answer correctly identifies the `--execute` flag and explains that live collection is dry-run by default.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the `--execute` flag is required because live collection is dry-run by default, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T14",
  "answer_hash": "790c9940a26b",
  "correct": true,
  "reason": "The answer correctly identifies the correct path (`~/Library/Group Containers/group.com.apple.calendar/Calendar.sqlitedb`) and explains that the file under `~/Library/Calendars` fails because it is a 0-byte decoy file.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies the live store path ~/Library/Group Containers/group.com.apple.calendar/Calendar.sqlitedb and explains that ~/Library/Calendars/Calendar.sqlitedb is a 0-byte decoy on current macOS.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T15",
  "answer_hash": "4e075eea8384",
  "correct": true,
  "reason": "The answer accurately includes all required elements: creating a matplotlib structure draft first, passing it via the -i flag, and explaining that generating from text alone yields unreliable results.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the required workflow—first create a matplotlib structure draft and pass it as the input image via -i—and explains that text-only prompting produces unreliable results.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T16",
  "answer_hash": "37acce294a6c",
  "correct": true,
  "reason": "The answer correctly identifies that sync only reads `.qta` and `.m4a` files.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states sync reads only .qta and .m4a files, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T17",
  "answer_hash": "08dcf6138be1",
  "correct": true,
  "reason": "The answer correctly states both required steps: pairing the watch with the Mac in Device Hub at least once, and restarting the Apple Watch.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the watch must be paired with the Mac at least once and then restarted for the Developer Mode toggle to appear, matching the excerpt's required steps.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T18",
  "answer_hash": "9f4a2f2a7bd1",
  "correct": true,
  "reason": "The answer accurately states that the native CLI maps upstream nulls to 0 and correctly identifies `scripts/request_metrics.py` as the script to use instead.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the native CLI maps upstream nulls to 0 and prescribes scripts/request_metrics.py for the scale-to-zero/inactivity check, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T19",
  "answer_hash": "31aed6eceb7e",
  "correct": true,
  "reason": "The answer accurately states that you should observe the battery trend to see if the battery percentage is rising across a watch window or between snapshots.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the rule that docked/charging state must be inferred from a rising battery trend across a watch window or between snapshots, not from the null fields.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T20",
  "answer_hash": "e51a7fab927b",
  "correct": true,
  "reason": "The answer accurately states that subagent fan-out sessions are skipped by default and can be included using the `--include-subagent` flag.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states subagent fan-out sessions are skipped by default and that the --include-subagent flag includes them, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T21",
  "answer_hash": "1c5786ac0ab1",
  "correct": true,
  "reason": "The answer correctly identifies the strict upper limit as 22 words.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the strict upper limit is 22 words and quotes the hard gate that no ordinary spoken sentence may exceed 22 words.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T22",
  "answer_hash": "2662e4567900",
  "correct": true,
  "reason": "The answer correctly identifies `--source-root` as the option to define the base directory and specifies that relative paths resolve against the CLI's current working directory when omitted.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states --source-root as the option and that omitted relative paths resolve against the CLI's current working directory, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T23",
  "answer_hash": "9dc0de149398",
  "correct": true,
  "reason": "The answer directly states that volume and mute state are not readable due to a protocol limitation and that the status endpoint returns PowerState only.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states volume and mute cannot be read due to a protocol limitation and that get_status returns only PowerState, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T24",
  "answer_hash": "b309d6fbc595",
  "correct": true,
  "reason": "The answer correctly identifies exit code 10 and explains that it requires a human to refresh the cookie.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states exit code 10 indicates an expired session cookie and that it requires a human to refresh the cookie per docs/cookie_refresh.md, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T25",
  "answer_hash": "0153d2126901",
  "correct": true,
  "reason": "The answer accurately states the maximum duration limit of 7200 seconds (2 hours) before a file is rejected.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the 7200-second (2-hour) per-file limit and correctly notes files longer than that are rejected before a model loads.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T26",
  "answer_hash": "03374e624353",
  "correct": true,
  "reason": "The answer clearly states that the default timezone is UTC.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the default timezone is UTC for both calendar invite and calendar update, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T27",
  "answer_hash": "7aaf64436770",
  "correct": true,
  "reason": "The answer correctly explains that `--after` is a pagination cursor that expects a UUID cursor value rather than an ISO timestamp.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states that --after is a pagination cursor expecting UUID cursor values, not ISO timestamps, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T28",
  "answer_hash": "34f3426fc5f1",
  "correct": true,
  "reason": "The answer accurately states that include and exclude domain filters cannot be combined and provides the workaround of using the `-site:` operator in the query text.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states that --include-domain and --exclude-domain cannot be combined (mutually exclusive, Firecrawl limitation) and prescribes using the -site: operator inside the query text, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T29",
  "answer_hash": "fe09c53d26c8",
  "correct": true,
  "reason": "The answer explains that downmixing adds about +3 dB for correlated channels and causes non-clipping files to be falsely reported as clipping.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states that correlated stereo downmix sums channels at about +3 dB and falsely reports a compliant mix as clipping, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T30",
  "answer_hash": "972fdc1fab15",
  "correct": true,
  "reason": "The answer accurately states that the Chrome user-data-dir contains stale session tokens from a previous run and explains the cycle of the SSO SDK using, failing, and refreshing the invalid token upon receiving 401 errors.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the Chrome user-data-dir contains stale session tokens from a previous test and describes the SSO SDK repeatedly using and refreshing the invalid token after 401s, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 }
]