[
 {
  "id": "T01",
  "answer_hash": "aa2613397955",
  "correct": true,
  "reason": "The answer correctly identifies RT-DETR V2 invoking torch.float64 as the MPS-unsupported cause and prescribes the --device cpu flag as the workaround.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies the RT-DETR V2 model and torch.float64 as the cause of the crash, and provides the `--device cpu` flag as the workaround.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T02",
  "answer_hash": "151b10860f76",
  "correct": true,
  "reason": "The answer explicitly names the Keychain item handoff-own-encryption-key and instructs deleting it to reset the Mac's local Handoff identity key, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies the Keychain item as 'handoff-own-encryption-key'.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T03",
  "answer_hash": "90a933923100",
  "correct": true,
  "reason": "The answer states deleting tmp/openreview_token.json and re-running the query, matching the required steps.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly instructs deleting `tmp/openreview_token.json` and retrying the command.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T04",
  "answer_hash": "31241e89556e",
  "correct": true,
  "reason": "The answer clearly states the required 2 to 3 second interval between successive downloads, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly states that a 2 to 3 second interval should be maintained between downloads.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T05",
  "answer_hash": "44b01d46dd03",
  "correct": true,
  "reason": "The answer explicitly says not to use the POST response's id/creation_uuid and instructs sending a GET list request afterward to find the new resource by content match, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states not to rely directly on the immediate response and instructs to send a GET request to fetch the list and match the resource by content.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T06",
  "answer_hash": "25beda5d7aef",
  "correct": true,
  "reason": "The answer clearly states the recommended default wake-up interval is about 30 minutes (1,800 seconds), matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies the recommended default wake-up interval as about 30 minutes (1,800 seconds).",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T07",
  "answer_hash": "5c961ea32274",
  "correct": true,
  "reason": "The answer explicitly states that forcing determinism on long text payloads causes pathological repetition and notes providers' own sampling strategies, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately explains that forcing determinism on long text outputs can cause repetition and mentions provider sampling strategies.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T08",
  "answer_hash": "cfd6d20f781d",
  "correct": true,
  "reason": "The answer clearly states the recommended padding is 24px or more, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies the recommended padding of 24px or more.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T09",
  "answer_hash": "8f13cec493f0",
  "correct": true,
  "reason": "The answer states the bare top-level key `web_search = \"disabled\"` with a string value and explicitly rejects booleans and other variants, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly specifies setting `web_search = \"disabled\"` as a top-level key and states that the value type must be a string.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T10",
  "answer_hash": "54d17b5b6fce",
  "correct": true,
  "reason": "The answer clearly states the maximum is 50 pairs, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states that the maximum number of image pairs allowed in a single delete manifest is 50 pairs.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T11",
  "answer_hash": "73965038af10",
  "correct": true,
  "reason": "The answer clearly states the unrecognized --gpu value is not rejected and is sent as A100, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly states that an unrecognized --gpu value is not rejected and is sent as A100.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T12",
  "answer_hash": "802ae703b97c",
  "correct": true,
  "reason": "The answer states the account is authenticated but not verified and instructs completing verification in the DataForSEO panel, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately explains that the account is authenticated but not verified, and instructs the user to complete verification in the DataForSEO dashboard/panel.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T13",
  "answer_hash": "13f822eef509",
  "correct": true,
  "reason": "The answer states the required `--execute` flag and explains that the CLI defaults to dry-run mode, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies the `--execute` flag and explains that live collection runs in dry-run mode by default.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T14",
  "answer_hash": "e8520a0f5dd9",
  "correct": true,
  "reason": "The answer correctly identifies the live path ~/Library/Group Containers/group.com.apple.calendar/Calendar.sqlitedb and explains that ~/Library/Calendars/Calendar.sqlitedb is a 0-byte decoy file.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer provides the correct Group Containers path and explains that the file under ~/Library/Calendars failed because it is a 0-byte decoy file.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T15",
  "answer_hash": "babc0a366223",
  "correct": true,
  "reason": "The answer clearly states the matplotlib draft workflow, passing it as the -i input image, and that text-only generation produces unreliable results.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer covers all required points: generating a matplotlib draft, passing it via the -i flag, and explaining that generating from text alone yields unreliable results.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T16",
  "answer_hash": "f3109bc3c3d7",
  "correct": true,
  "reason": "The answer clearly states sync reads only .m4a and .qta files, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies that the sync step only reads .m4a and .qta files.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T17",
  "answer_hash": "e4db116b4066",
  "correct": true,
  "reason": "The answer includes both required steps: pairing the watch with the Mac in Device Hub at least once and restarting the Apple Watch, then confirms the Developer Mode toggle appears in Settings > Privacy & Security.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies both pairing the Apple Watch with the Mac via Device Hub and restarting the watch to make the option appear.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T18",
  "answer_hash": "ebd37089e721",
  "correct": true,
  "reason": "The answer states the native CLI maps upstream nulls to 0 and recommends using scripts/request_metrics.py instead, matching the rule exactly.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies that the native CLI maps null values to 0 and specifies that scripts/request_metrics.py should be used instead.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T19",
  "answer_hash": "8b57f67b52eb",
  "correct": true,
  "reason": "The answer's primary method explicitly states to monitor the battery trend, verifying that battery_percent increases over time across a watch window or between snapshots, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states that the battery trend (observing an increasing battery percentage across a watch window or between snapshots) should be used to determine if the vacuum is docked and charging.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T20",
  "answer_hash": "05ca36049f7f",
  "correct": true,
  "reason": "The answer states subagent fan-out sessions are skipped by default and that the `--include-subagent` flag includes them, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states that subagent fan-out sessions are skipped by default and can be included using the `--include-subagent` flag.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T21",
  "answer_hash": "d3ca4aaf3ac2",
  "correct": true,
  "reason": "The answer explicitly states the hard gate that no ordinary spoken sentence may exceed 22 words, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly states the strict upper limit of 22 words as specified in the excerpt.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T22",
  "answer_hash": "7ef1ae872670",
  "correct": true,
  "reason": "The answer explicitly states --source-root as the option and that omitted relative paths resolve against the CLI's current working directory, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies `--source-root` as the command-line option and states that it defaults to the CLI's current working directory when omitted.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T23",
  "answer_hash": "ce027f8a4570",
  "correct": true,
  "reason": "The answer clearly states volume and mute cannot be read due to protocol limitations and that get_status returns only PowerState, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly states that volume and mute state cannot be retrieved and that the status endpoint only returns PowerState.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T24",
  "answer_hash": "bda8e62b61a8",
  "correct": true,
  "reason": "The answer correctly states exit code 10 for the expired-cookie auth failure and that it must be resolved by a human manually refreshing the cookie per docs/cookie_refresh.md.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies exit code 10 as indicating an expired cookie and states that a human must manually refresh it.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T25",
  "answer_hash": "4af0980be7a5",
  "correct": true,
  "reason": "The answer clearly states the 7200-second (2-hour) per-file limit and that files exceeding it are rejected before any model loads, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies the maximum duration as 7,200 seconds (2 hours) before the file is rejected.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T26",
  "answer_hash": "55463748c272",
  "correct": true,
  "reason": "The answer clearly states the default timezone is UTC, matching the excerpt and rubric.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly and accurately states that the default timezone is UTC.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T27",
  "answer_hash": "c36cb97dddcc",
  "correct": true,
  "reason": "The answer clearly states that --after is a pagination cursor expecting a UUID email ID, not an ISO timestamp, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states that `--after` is a pagination cursor rather than a timestamp filter and expects a UUID.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T28",
  "answer_hash": "df3cf757a98e",
  "correct": false,
  "reason": "The answer claims Tavily allows combining --include-domain and --exclude-domain, contradicting the rule that include and exclude domain filters cannot be combined; it only applies the correct workaround to Firecrawl.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly states that include and exclude domain flags cannot be combined in Firecrawl and instructs using the `-site:` operator directly inside the search query.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T29",
  "answer_hash": "75c6f3dd086b",
  "correct": true,
  "reason": "The answer states ffmpeg's -ac 1 sums correlated channels by roughly +3 dB, causing false clipping reports, matching the excerpt's rule.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately states that ffmpeg sums correlated channels at approximately +3 dB, leading to false reports of clipping.",
  "judge_secondary": "gemini-3.8-flash"
 },
 {
  "id": "T30",
  "answer_hash": "5b420aac18a0",
  "correct": true,
  "reason": "The answer clearly identifies the stale user-data-dir/session tokens from a previous session as the cause and describes the SSO SDK repeatedly using and refreshing the invalid token after 401s, matching the excerpt.",
  "judge": "deepseek-v4.1-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer accurately explains that stale session tokens in the Chrome user-data directory from a previous session cause the SSO SDK to repeatedly attempt using and refreshing the invalid token upon receiving 401 errors.",
  "judge_secondary": "gemini-3.8-flash"
 }
]