[
 {
  "id": "T01",
  "answer_hash": "b469b2ca2b58",
  "correct": true,
  "reason": "The answer states the crash is caused by Docling's RT-DETR V2 model calling torch.float64 unsupported by MPS, and prescribes the --device cpu flag as the workaround.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies the RT-DETR V2 model and torch.float64 as the cause of the crash, and provides the --device cpu flag as the workaround.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T02",
  "answer_hash": "98e15b3876e6",
  "correct": true,
  "reason": "The answer explicitly names the Keychain item handoff-own-encryption-key and states that deleting it resets the Mac's local Handoff identity key, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies the Keychain item named handoff-own-encryption-key.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T03",
  "answer_hash": "85b0a4a5fb6f",
  "correct": true,
  "reason": "The answer explicitly instructs deleting tmp/openreview_token.json and retrying the command once, matching the ground truth rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly instructs to delete tmp/openreview_token.json and re-run the query.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T04",
  "answer_hash": "16f52a3c7e60",
  "correct": true,
  "reason": "The answer clearly states the required 2–3 second interval between downloads, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly specifies the 2-3 second interval required between downloads to prevent triggering anti-scraping protections.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T05",
  "answer_hash": "e78742bdd820",
  "correct": true,
  "reason": "The answer explicitly advises not to use the POST response ID and to send a GET list request and locate the new resource by content matching, satisfying both rubric points.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states not to rely on the POST response for the ID and explains that one should send a GET request to fetch the list and identify the resource by matching its content.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T06",
  "answer_hash": "d67071dd9f54",
  "correct": true,
  "reason": "The answer clearly states the recommended default wake-up interval is about 30 minutes (1,800 seconds), matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly states the recommended default wake-up interval as about 30 minutes (1,800 seconds).",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T07",
  "answer_hash": "5a8420f587fc",
  "correct": true,
  "reason": "The answer clearly states that forcing determinism (temperature 0) on long text payloads causes repetition and notes providers' own sampling strategies, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies that forcing determinism on long text payloads can cause repetition loops, while also noting provider-specific sampling defaults.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T08",
  "answer_hash": "7d620b111d21",
  "correct": true,
  "reason": "The answer explicitly states the recommended minimum padding is 24px or more, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies the recommended minimum padding as 24px or more.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T09",
  "answer_hash": "1c2c2b8512c7",
  "correct": true,
  "reason": "The answer states the bare top-level key `web_search = \"disabled\"` with a string value and explicitly notes booleans/alternate strings are rejected, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly states to set `web_search = \"disabled\"` and explicitly specifies that the value must be a string.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T10",
  "answer_hash": "659e1f2cfa66",
  "correct": true,
  "reason": "The answer clearly states the maximum is 50 pairs and correctly notes that larger sets must be split into independent plans with fresh review and authorization.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately identifies the maximum limit of 50 pairs per delete manifest.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T11",
  "answer_hash": "ad818a615383",
  "correct": true,
  "reason": "The answer clearly states the unrecognized --gpu value is sent as A100 and not rejected, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states that an unrecognized --gpu value is sent as A100 rather than being rejected, matching the excerpt.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T12",
  "answer_hash": "e98cc73f421e",
  "correct": true,
  "reason": "The answer states the account is authenticated but not verified and instructs completing verification in the DataForSEO dashboard, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states that error 40104 is caused by an authenticated but unverified account and explains that completing account verification in the DataForSEO dashboard/panel fixes it.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T13",
  "answer_hash": "2219029b9e05",
  "correct": true,
  "reason": "The answer explicitly states the `--execute` flag is needed and explains that live collection runs in dry-run mode by default, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies the `--execute` flag and explains that commands run in dry-run mode by default.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T14",
  "answer_hash": "89a80df8a851",
  "correct": true,
  "reason": "The answer correctly identifies the live path ~/Library/Group Containers/group.com.apple.calendar/Calendar.sqlitedb and explains the ~/Library/Calendars/Calendar.sqlitedb decoy is a 0-byte file.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer provides the correct group container path and explains that the file under ~/Library/Calendars is a 0-byte decoy.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T15",
  "answer_hash": "c55445aee8e6",
  "correct": true,
  "reason": "The answer clearly states the required workflow—create a matplotlib structure draft and pass it as the input image via the -i flag—and explains that generating from text alone produces unreliable results.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately explains the workflow of creating a matplotlib draft, passing it via the -i flag, and notes that generating from text alone yields unreliable results.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T16",
  "answer_hash": "97288cf91dc2",
  "correct": true,
  "reason": "The answer clearly states that sync reads only .m4a and .qta files, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately specifies that the sync step only reads .m4a and .qta files, satisfying the rubric.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T17",
  "answer_hash": "2d893ed03230",
  "correct": true,
  "reason": "The answer explicitly includes pairing the watch with the Mac via Device Hub and restarting the Apple Watch, matching the required steps from the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly states both required steps: pairing the watch with the Mac in Device Hub and restarting the Apple Watch.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T18",
  "answer_hash": "ddb069aedbe0",
  "correct": true,
  "reason": "The answer states the native CLI maps upstream nulls to 0 and prescribes scripts/request_metrics.py for scale-to-zero/inactivity checks, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly explains that the native CLI maps upstream nulls to 0 and specifies that `scripts/request_metrics.py` should be used instead.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T19",
  "answer_hash": "7ff87dcaec62",
  "correct": true,
  "reason": "The answer explicitly recommends using the battery trend (rising battery_percent across a watch window or between snapshots) to determine docked/charging status, matching the ground truth rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states to use the battery trend, looking for rising battery percentage across a watch window or between snapshots.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T20",
  "answer_hash": "2244703172d3",
  "correct": true,
  "reason": "The answer states subagent fan-out sessions are skipped by default and that the --include-subagent flag includes them, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly explains that subagent fan-out sessions are skipped by default and can be included using the `--include-subagent` flag.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T21",
  "answer_hash": "71d29f716900",
  "correct": true,
  "reason": "The answer explicitly states the hard gate that no ordinary spoken sentence may exceed 22 words, matching the excerpt exactly.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states the strict upper limit of 22 words for an ordinary spoken sentence.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T22",
  "answer_hash": "18223ef9ae96",
  "correct": true,
  "reason": "The answer correctly identifies --source-root as the option and states that omitted paths resolve against the CLI's current working directory, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies `--source-root` as the option and the CLI's current working directory as the default resolution path.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T23",
  "answer_hash": "a4ba2d90932e",
  "correct": true,
  "reason": "The answer clearly states volume and mute are not readable due to a protocol limitation and that get_status returns PowerState only, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states that volume and mute status are not readable and that the status endpoint returns PowerState only.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T24",
  "answer_hash": "fccbfee2e9c6",
  "correct": true,
  "reason": "The answer correctly states exit code 10 for expired-cookie auth failure and that resolution requires a human to refresh the cookie, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies exit code 10 and states that a human must refresh the session cookie.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T25",
  "answer_hash": "ece6d659cc66",
  "correct": true,
  "reason": "The answer clearly states the 7,200-second (2-hour) per-file limit and that files exceeding it are rejected before the model loads, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly states that the maximum duration before rejection is 7,200 seconds (2 hours).",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T26",
  "answer_hash": "7bbf9d0aa9aa",
  "correct": true,
  "reason": "The answer clearly states the default timezone is UTC, matching the excerpt and rubric.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly states that the default timezone is UTC.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T27",
  "answer_hash": "ae2968f41214",
  "correct": true,
  "reason": "The answer clearly states that --after is a pagination cursor expecting a UUID cursor value, not an ISO timestamp filter, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly states that `--after` is a pagination cursor rather than a timestamp filter and that it expects a UUID cursor value.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T28",
  "answer_hash": "55e905698e80",
  "correct": true,
  "reason": "The answer states that Firecrawl's --include-domain and --exclude-domain flags are mutually exclusive and instructs using the -site: operator inside the query text, matching the ground truth rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states that include and exclude domain flags cannot be combined and explains how to use the `-site:` operator directly inside the query text.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T29",
  "answer_hash": "8f9c244d4731",
  "correct": true,
  "reason": "The answer states ffmpeg sums correlated stereo channels by about +3 dB and causes false clipping reports, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly explains that downmixing correlated stereo sums channels by about +3 dB, leading to false clipping reports.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T30",
  "answer_hash": "0dea6f9c7f86",
  "correct": true,
  "reason": "The answer clearly states the Chrome user-data-dir contains stale session tokens from a previous session and that the SSO SDK repeatedly uses/refreshes the invalid token after 401s, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately identifies that stale session tokens in the Chrome user data directory cause the SSO SDK to repeatedly try and refresh the invalid token after receiving 401s.",
  "judge_secondary": "gemini-3.8-flash"
 }
]