[
 {
  "id": "T01",
  "answer_hash": "2c6a93cf9b40",
  "correct": true,
  "reason": "The answer accurately identifies RT-DETR V2 invoking torch.float64 as the cause of the crash and specifies --device cpu as the workaround flag.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the crash is caused by Docling's RT-DETR V2 model calling torch.float64 (unsupported by MPS) and prescribes the --device cpu flag as the workaround, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T02",
  "answer_hash": "d0a3c3f53865",
  "correct": true,
  "reason": "The answer correctly identifies the Keychain item as `handoff-own-encryption-key`.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly names the Keychain item `handoff-own-encryption-key` and describes deleting it to reset the Mac's local Handoff identity key, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T03",
  "answer_hash": "dd5433795c56",
  "correct": true,
  "reason": "The answer correctly instructs deleting tmp/openreview_token.json and retrying the command.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states to delete `tmp/openreview_token.json` and retry once, matching the ground truth rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T04",
  "answer_hash": "2b10c7be4408",
  "correct": true,
  "reason": "The answer accurately states that an interval of 2–3 seconds should be maintained between downloads to avoid triggering anti-scraping protections.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the 2–3 second interval between successive downloads, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T05",
  "answer_hash": "6202ab512631",
  "correct": false,
  "reason": "The answer provides a generic list of hypothetical reverse-engineering strategies rather than stating the required rule to fetch the list via GET and find the resource by content match.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer recommends many speculative approaches (filtering by creation_uuid, job polling, Location headers, websocket) rather than clearly stating the ground-truth rule to send a GET list request and find the new resource by content match.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T06",
  "answer_hash": "3ea999650eac",
  "correct": true,
  "reason": "The answer accurately states the recommended default wake-up interval as about 30 minutes (1800 seconds).",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the recommended default inspection wake-up interval is about 30 minutes (1800 seconds), matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T07",
  "answer_hash": "5e64eb4598ba",
  "correct": false,
  "reason": "The answer fails to state that forcing determinism on long text payloads can cause repetition, relying instead on a general LLM engineering explanation.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer never mentions the specific rule that forcing determinism on long text payloads may cause repetition; it discusses degenerate loops and diversity loss but not the excerpt's stated gotcha.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T08",
  "answer_hash": "b9d0c9a0ef36",
  "correct": true,
  "reason": "The answer correctly identifies 24px as the recommended minimum padding.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the recommended minimum padding is 24px, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T09",
  "answer_hash": "c12f03059ddb",
  "correct": false,
  "reason": "The answer declines to provide the specific configuration and incorrectly suggests that the value is typically a boolean rather than the required string `web_search = \"disabled\"`.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer refuses to state the required rule, instead suggesting a boolean `false` value and nested placement, contradicting the required `web_search = \"disabled\"` string value.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T10",
  "answer_hash": "4664ec80c4db",
  "correct": true,
  "reason": "The answer correctly states the limit of 50 pairs per delete manifest.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the maximum is 50 pairs per delete manifest, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T11",
  "answer_hash": "82b6560d67f5",
  "correct": true,
  "reason": "The answer accurately states that an unrecognized --gpu value is not rejected and is sent as A100.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the unrecognized --gpu value is not rejected and is silently sent as A100, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T12",
  "answer_hash": "1f7981e7a716",
  "correct": true,
  "reason": "The answer correctly identifies that the account is authenticated but not verified and directs the user to complete verification in the DataForSEO panel.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states error 40104 means the account is authenticated but not verified and instructs completing verification in the DataForSEO panel, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T13",
  "answer_hash": "f0ae66592077",
  "correct": true,
  "reason": "The answer correctly identifies the `--execute` flag and explains that live collection is dry-run by default.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the required flag is `--execute` and explains that the CLI is dry-run by default, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T14",
  "answer_hash": "1c814bde3709",
  "correct": true,
  "reason": "The answer provides the correct group container path and explains that the file under ~/Library/Calendars failed because it is a 0-byte decoy file.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the correct group container path and explains the ~/Library/Calendars file is a 0-byte decoy, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T15",
  "answer_hash": "4fabcfb7a625",
  "correct": false,
  "reason": "The answer admits to not knowing the workflow for GPT Image 2.5 and fails to mention creating a matplotlib structure draft or passing it as the -i input image.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer never mentions creating a matplotlib structure draft or passing it as the `-i` input image, instead recommending HTML/CSS/SVG composition, so it does not state the required rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T16",
  "answer_hash": "6047e6e7df06",
  "correct": true,
  "reason": "The answer clearly states that the sync step only reads `.qta` and `.m4a` files.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the sync reads exactly two extensions, .qta and .m4a, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T17",
  "answer_hash": "fe19d2e738b7",
  "correct": true,
  "reason": "The answer accurately states both required steps: pairing the watch with the Mac via Device Hub and restarting the Apple Watch.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states both required steps from the excerpt: pairing the watch with the Mac in Device Hub at least once, and restarting the Apple Watch, matching the ground-truth rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T18",
  "answer_hash": "c54d14608c07",
  "correct": true,
  "reason": "The answer clearly states that the native CLI maps upstream nulls to 0 and explicitly identifies `scripts/request_metrics.py` as the script to use instead.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the native CLI maps upstream nulls to 0 and prescribes scripts/request_metrics.py for the inactivity check, matching the rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T19",
  "answer_hash": "bf0c0c1979f6",
  "correct": true,
  "reason": "The answer clearly states that the battery trend (a rising battery percentage across a watch window or between snapshots) is the way to determine if the vacuum is docked and charging.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly identifies the battery trend (rising battery percentage across a watch window or between snapshots) as the authoritative way to determine docked/charging state, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T20",
  "answer_hash": "3870f8b66369",
  "correct": true,
  "reason": "The answer correctly states that subagent fan-out sessions are skipped by default and explains that they can be included using the `--include-subagent` flag.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states subagent fan-out sessions are skipped by default and that the `--include-subagent` flag is required to include them, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T21",
  "answer_hash": "509e1d123398",
  "correct": true,
  "reason": "The answer correctly states that the strict upper limit is 22 words.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the strict upper limit is 22 words for an ordinary spoken sentence, matching the excerpt's hard gate.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T22",
  "answer_hash": "c533a932a6fb",
  "correct": true,
  "reason": "The answer correctly identifies `--source-root` as the option to define the base directory and specifies that paths resolve against the CLI's current working directory if omitted.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states --source-root as the option and that omitted relative paths resolve against the CLI's current working directory, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T23",
  "answer_hash": "e6297a23d87e",
  "correct": true,
  "reason": "The answer accurately states that volume and mute state are not readable due to protocol limitations and that the status endpoint returns PowerState only.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states volume and mute are not readable due to a protocol limitation and that get_status returns PowerState only, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T24",
  "answer_hash": "9ef7cb563f5e",
  "correct": true,
  "reason": "The answer correctly identifies exit code 10 and states that a human must refresh the cookie.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states exit code 10 for expired cookie auth failure and says it requires a human to refresh the cookie per docs/cookie_refresh.md, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T25",
  "answer_hash": "cbe28985dc35",
  "correct": true,
  "reason": "The answer correctly states the limit of 7200 seconds (2 hours) per file as required.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the 7200-second (2-hour) per-file limit and that rejection occurs before a model loads, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T26",
  "answer_hash": "7101679512a6",
  "correct": false,
  "reason": "The answer fails to state that the default timezone is UTC, claiming instead that it cannot answer the question.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer never states the default timezone is UTC; it claims the tool is unknown and recommends requiring an explicit timezone or using /etc/localtime instead.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T27",
  "answer_hash": "a2b5ba33be71",
  "correct": true,
  "reason": "The answer accurately explains that `--after` is a pagination cursor expecting a UUID cursor value rather than an ISO timestamp filter.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states that --after is a Resend pagination cursor expecting a UUID cursor value, not an ISO timestamp, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T28",
  "answer_hash": "5fad400ec5a3",
  "correct": false,
  "reason": "The answer fails to state the constraint that include and exclude domain filters cannot be combined.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer never states the key rule that --include-domain and --exclude-domain cannot be combined, and instead suggests structured include/exclude parameters as a cleaner option, contradicting the constraint.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T29",
  "answer_hash": "0a8535dbc3e8",
  "correct": false,
  "reason": "The answer fails to mention that ffmpeg sums correlated stereo channels at about +3 dB causing false clipping, and incorrectly claims that identical correlated channels result in (L + L)/2 = L.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer claims ffmpeg's downmix attenuates correlated content and never states the key rule that -ac 1 sums correlated stereo channels at about +3 dB, causing false clipping reports.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T30",
  "answer_hash": "77c6a2e09c03",
  "correct": true,
  "reason": "The answer directly identifies that the Chrome user-data-dir contains stale session tokens from a previous test and explains how the SSO SDK repeatedly attempts to use and refresh the invalid token upon receiving 401 errors.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the CDP Chrome user-data-dir contains stale session tokens from a previous test, causing the SSO SDK to repeatedly use and refresh the invalid token on 401s, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 }
]