[
 {
  "id": "T01",
  "answer_hash": "69ef077d545e",
  "correct": true,
  "reason": "The answer correctly identifies the RT-DETR V2 model calling torch.float64 as the cause of the crash and provides the `--device cpu` flag as the workaround.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies RT-DETR V2 calling torch.float64 as the MPS crash cause and prescribes the --device cpu flag as the workaround.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T02",
  "answer_hash": "a4c7bd011785",
  "correct": true,
  "reason": "The answer correctly identifies the specific Keychain item named handoff-own-encryption-key.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly names the Keychain item handoff-own-encryption-key and describes deleting it to reset the Mac's local Handoff identity key, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T03",
  "answer_hash": "6afb323e4eff",
  "correct": true,
  "reason": "The answer correctly states to delete the file `tmp/openreview_token.json` and retry the query.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states deleting `tmp/openreview_token.json` and retrying the query, matching the required rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T04",
  "answer_hash": "4c9862aa50c1",
  "correct": true,
  "reason": "The answer accurately states the 2–3 second interval required between downloads to prevent triggering anti-scraping protections.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the required 2–3 second interval between successive downloads, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T05",
  "answer_hash": "bbb3ad849181",
  "correct": true,
  "reason": "The answer accurately instructs not to rely on the POST response's creation_uuid and to fetch the list endpoint to locate the newly created item by matching content.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states not to rely on the POST response's creation_uuid as the ID and instructs sending a GET list request after POST to find the new resource by content match, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T06",
  "answer_hash": "86809155f57e",
  "correct": true,
  "reason": "The answer accurately states the recommended default wake-up interval of 30 minutes (1800 seconds).",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the recommended default interval as 30 minutes (1800 seconds), matching the excerpt exactly.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T07",
  "answer_hash": "34112a769776",
  "correct": true,
  "reason": "The answer directly includes the required reasons from the excerpt, explicitly noting that forcing determinism on long text payloads may cause repetition.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer quotes the excerpt verbatim, including that forcing determinism on long text payloads may cause repetition, and correctly advises not defaulting to temperature 0.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T08",
  "answer_hash": "b8807e183a66",
  "correct": true,
  "reason": "The answer correctly identifies the recommended minimum padding as 24px or more based on the excerpt.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the recommended minimum padding is 24px or more, matching the excerpt exactly.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T09",
  "answer_hash": "7382f331da8d",
  "correct": true,
  "reason": "The answer correctly identifies setting the top-level key `web_search = \"disabled\"` using a string value type.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the bare top-level key `web_search = \"disabled\"` with a string value and notes booleans and other strings are rejected, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T10",
  "answer_hash": "e334c05f39e0",
  "correct": true,
  "reason": "The answer accurately states that the limit is 50 pairs per delete manifest, matching the excerpt and rubric.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the maximum is 50 pairs per delete manifest, matching the excerpt exactly.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T11",
  "answer_hash": "b60400660ea0",
  "correct": true,
  "reason": "The answer accurately states that an unrecognized value is sent as A100 and not rejected.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the unrecognized --gpu value is sent as A100 and not rejected, matching the ground truth rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T12",
  "answer_hash": "822733bdaa9a",
  "correct": true,
  "reason": "The answer accurately explains that the account is authenticated but unverified and directs the user to complete verification in the DataForSEO dashboard/panel.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the account is authenticated but not verified and instructs completing verification in the DataForSEO panel, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T13",
  "answer_hash": "2f0e701609cc",
  "correct": true,
  "reason": "The answer correctly identifies the `--execute` flag and explains that it is required because the commands operate in dry-run mode by default.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the `--execute` flag is needed because the CLI defaults to dry-run mode, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T14",
  "answer_hash": "100e3e7d0855",
  "correct": true,
  "reason": "The answer provides the correct live store path and correctly explains that the file under ~/Library/Calendars is a 0-byte decoy.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer gives the correct group-container path and explains the ~/Library/Calendars file is a 0-byte decoy, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T15",
  "answer_hash": "b8b65a539c82",
  "correct": true,
  "reason": "The answer accurately states that one should first create a matplotlib structure draft, pass it via the `-i` input, and that prompting from text alone produces unreliable results.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the required workflow (create a matplotlib structure draft and pass it as the -i input image) and explains that text-only prompting produces unreliable results, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T16",
  "answer_hash": "faaf2cb5319c",
  "correct": true,
  "reason": "The answer correctly states that the sync step reads only .qta and .m4a files.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states that sync reads only .qta and .m4a files, matching the ground truth rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T17",
  "answer_hash": "af9f45aba035",
  "correct": true,
  "reason": "The answer correctly states both required steps: pairing the watch with the Mac in Device Hub at least once and restarting the Apple Watch.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer includes both required steps—pairing the watch with the Mac via Device Hub at least once and restarting the Apple Watch—so the Developer Mode toggle appears.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T18",
  "answer_hash": "fc0b18b98432",
  "correct": true,
  "reason": "The answer correctly explains that the CLI maps nulls to 0 and specifies using scripts/request_metrics.py instead.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the native CLI maps nulls to 0 and prescribes scripts/request_metrics.py, matching the rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T19",
  "answer_hash": "d8df15c78dbe",
  "correct": true,
  "reason": "The answer accurately explains how to use the battery trend to check for a rising battery level between snapshots or across a watch window.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states to use the battery trend (rising battery percentage across snapshots or a watch window) to determine docked/charging status, matching the excerpt and rubric.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T20",
  "answer_hash": "e3aa94b24393",
  "correct": true,
  "reason": "The answer accurately states that subagent fan-out sessions are skipped by default and can be included using the `--include-subagent` flag.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states subagent fan-out sessions are skipped by default and that adding the `--include-subagent` flag includes them, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T21",
  "answer_hash": "c199efe21fb8",
  "correct": true,
  "reason": "The answer correctly identifies the strict upper limit as 22 words.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the hard gate limit of 22 words, matching the excerpt exactly.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T22",
  "answer_hash": "d3b83278e523",
  "correct": true,
  "reason": "The answer accurately specifies `--source-root` as the option and the current working directory as the default.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states both required elements: --source-root as the option and current working directory as the default resolution base, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T23",
  "answer_hash": "e652028a766c",
  "correct": true,
  "reason": "The answer accurately states that volume and mute state cannot be read and that the status endpoint only returns PowerState.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states volume and mute cannot be read due to a protocol limitation and that get_status returns only PowerState, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T24",
  "answer_hash": "6482233c4e5a",
  "correct": true,
  "reason": "The answer correctly identifies exit code 10 and states that a human must refresh the session cookie.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states exit code 10 for expired cookie auth failure and that a human must refresh the cookie, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T25",
  "answer_hash": "704887e12edd",
  "correct": true,
  "reason": "The answer accurately states the maximum allowed duration of 7200 seconds (2 hours) before the file is rejected.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the 7200-second (2-hour) limit and that longer files are rejected before a model loads, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T26",
  "answer_hash": "59eb487dcd17",
  "correct": true,
  "reason": "The answer correctly states that the default timezone is UTC.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the default timezone is UTC, matching the excerpt and rubric.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T27",
  "answer_hash": "2a87cea0e30c",
  "correct": true,
  "reason": "The answer accurately states that `--after` is a pagination cursor that expects a UUID cursor value rather than an ISO timestamp.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states that --after is a pagination cursor expecting a UUID cursor value, not an ISO timestamp, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T28",
  "answer_hash": "c2190b4636c2",
  "correct": true,
  "reason": "The answer correctly states that include and exclude domain flags cannot be combined and explains to use the `-site:` operator inside the query text instead.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states that --include-domain and --exclude-domain cannot be combined and applies the -site: operator inside the query text as the workaround, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T29",
  "answer_hash": "fdaf518c549f",
  "correct": true,
  "reason": "The answer accurately states that ffmpeg sums the channels at about +3 dB for correlated stereo, causing audio to be falsely reported as clipping.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states that ffmpeg's -ac 1 sums correlated stereo channels at about +3 dB, causing a compliant mix to be falsely reported as clipping, matching the excerpt and rubric.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T30",
  "answer_hash": "9fe69791a91e",
  "correct": true,
  "reason": "The answer accurately identifies that stale tokens in the Chrome user-data-dir cause the SSO SDK to repeatedly use, fail with 401, and refresh the token in an infinite loop.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the Chrome user-data-dir contains stale session tokens from a previous session and that the SSO SDK repeatedly uses/refreshes the invalid token after 401s, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 }
]