[
 {
  "id": "T01",
  "answer_hash": "92918bab9b86",
  "correct": true,
  "reason": "The answer accurately identifies RT-DETR V2 calling torch.float64 as the cause and provides the --device cpu flag as the workaround.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies RT-DETR V2 calling torch.float64 as the MPS-unsupported cause and prescribes the --device cpu flag as the workaround.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T02",
  "answer_hash": "790dbcb880ea",
  "correct": true,
  "reason": "The answer clearly identifies the Keychain item as `handoff-own-encryption-key` and mentions restarting Handoff, satisfying the rubric.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly names the Keychain item `handoff-own-encryption-key` and describes deleting it to reset the Mac's local Handoff identity key, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T03",
  "answer_hash": "08f9911c5013",
  "correct": true,
  "reason": "The answer correctly specifies deleting `tmp/openreview_token.json` and retrying the command.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states to delete tmp/openreview_token.json and retry once, matching the ground truth rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T04",
  "answer_hash": "1a317223f47e",
  "correct": true,
  "reason": "The answer accurately states that a 2–3 second interval should be maintained between successive downloads.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the required 2–3 second interval between successive downloads, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T05",
  "answer_hash": "855fe1649dce",
  "correct": false,
  "reason": "The answer provides a generic troubleshooting list of several possibilities instead of stating the specific rule to send a GET request to fetch the list and find the resource by content match, and it even actively discourages client-side list matching.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer recommends many alternatives (headers, nested fields, polling, treating the uuid as the ID) and only mentions list matching as one option, rather than clearly stating the rule to send a GET list after POST and find the resource by content match.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T06",
  "answer_hash": "2a8f45e8e70d",
  "correct": true,
  "reason": "The answer clearly states the recommended default wake-up interval as approximately 30 minutes (1800 seconds).",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the recommended default inspection wake-up interval is about 30 minutes (1800 seconds), matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T07",
  "answer_hash": "ecada50dc960",
  "correct": true,
  "reason": "The answer quotes the exact rule and clearly explains that forcing determinism on long text payloads can cause repetition.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states that forcing determinism on long text payloads may cause repetition and that providers have their own sampling strategies, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T08",
  "answer_hash": "e8d20cc72ce0",
  "correct": true,
  "reason": "The answer clearly states the recommended minimum padding of 24px or more.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the recommended minimum padding is 24px or more, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T09",
  "answer_hash": "c4d27385620b",
  "correct": false,
  "reason": "The answer does not state that the configuration requires `web_search = \"disabled\"` as a string, and instead claims the information is unknown while suggesting boolean flags.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer never states the required key `web_search = \"disabled\"` or that the value must be a string; it instead speculates about booleans and other config shapes, contradicting the ground truth.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T10",
  "answer_hash": "41d832958188",
  "correct": true,
  "reason": "The answer correctly states that the maximum is 50 pairs per delete manifest.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the maximum is 50 pairs per delete manifest, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T11",
  "answer_hash": "0df903b65696",
  "correct": true,
  "reason": "The answer clearly states that an unrecognized `--gpu` value is not rejected and is sent as A100.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the unrecognized --gpu value is not rejected and is silently sent as A100, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T12",
  "answer_hash": "d412fa6216c8",
  "correct": true,
  "reason": "The answer accurately states that the account is authenticated but not verified and explains that verification must be completed in the DataForSEO panel.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states error 40104 means the account is authenticated but not verified and instructs completing verification in the DataForSEO panel, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T13",
  "answer_hash": "6cb0538a01db",
  "correct": true,
  "reason": "The answer correctly identifies the `--execute` flag and explains that it is needed because live collection is dry-run by default.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the required flag is `--execute` and explains it is needed because the CLI is dry-run by default, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T14",
  "answer_hash": "2bfc66f911ab",
  "correct": true,
  "reason": "The answer provides the correct group container path and explains that the file under ~/Library/Calendars failed because it is a 0-byte decoy.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the correct group container path and explains the ~/Library/Calendars file is a 0-byte decoy, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T15",
  "answer_hash": "3e142c55d819",
  "correct": false,
  "reason": "The answer does not mention creating a matplotlib structure draft or passing it as the -i input image, and instead proposes an unrelated HTML/CSS workflow.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer never states the required workflow of first creating a matplotlib structure draft and passing it as the -i input image, instead proposing an unrelated HTML/CSS composition approach.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T16",
  "answer_hash": "aaa70ed42fba",
  "correct": true,
  "reason": "The answer correctly identifies that the sync step reads only .m4a and .qta files.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states sync reads only .m4a and .qta files, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T17",
  "answer_hash": "c0cd8cec64bf",
  "correct": true,
  "reason": "The answer correctly states both required steps: pairing the watch with the Mac via Device Hub and restarting the Apple Watch.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states both required steps from the excerpt: pairing the watch with the Mac via Device Hub at least once, and restarting the Apple Watch so the Developer Mode toggle appears.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T18",
  "answer_hash": "0a34435ef928",
  "correct": true,
  "reason": "The answer clearly states that the native CLI maps nulls to 0 and specifies using scripts/request_metrics.py instead.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the native CLI maps upstream nulls to 0 and prescribes scripts/request_metrics.py for evaluating inactivity/scale-to-zero.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T19",
  "answer_hash": "3a493681963a",
  "correct": true,
  "reason": "The answer clearly states to use the battery trend by checking for a rising battery percentage between snapshots or across a watch window.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the authoritative method is the battery trend, describing rising battery percentage between snapshots or across a watch window, matching the excerpt and rubric.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T20",
  "answer_hash": "8b8acaf68da7",
  "correct": true,
  "reason": "The answer clearly states that subagent fan-out sessions are skipped by default and can be included using the `--include-subagent` flag.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states subagent fan-out sessions are skipped by default and that re-running export with the `--include-subagent` flag includes them, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T21",
  "answer_hash": "f34bc58a8941",
  "correct": true,
  "reason": "The answer clearly identifies 22 words as the strict upper limit.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the strict upper limit is 22 words for an ordinary spoken sentence, matching the excerpt's hard gate.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T22",
  "answer_hash": "14fd0ad6c0f1",
  "correct": true,
  "reason": "The answer correctly identifies `--source-root` as the option and states that relative paths default to resolving against the CLI's current working directory.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states --source-root as the option and that omitted paths resolve against the CLI's current working directory, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T23",
  "answer_hash": "6bdb46185fae",
  "correct": true,
  "reason": "The answer clearly states that volume and mute state are not readable due to protocol limitations and that the endpoint returns PowerState only.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states volume and mute state are not readable due to a protocol limitation and that get_status returns PowerState only, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T24",
  "answer_hash": "d9f9c4a63807",
  "correct": true,
  "reason": "The answer correctly identifies exit code 10 and states that a human must refresh the session cookie.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states exit code 10 for an expired cookie and correctly identifies resolution as a human step to refresh the cookie per docs/cookie_refresh.md.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T25",
  "answer_hash": "c6dd9a7127a9",
  "correct": true,
  "reason": "The answer correctly identifies the maximum duration as 7200 seconds (2 hours).",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the 7200-second (2-hour) per-file limit and that rejection occurs before a model loads, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T26",
  "answer_hash": "fb587d8db64f",
  "correct": false,
  "reason": "The answer fails to state that the default timezone is UTC, claiming instead that the commands do not exist.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer denies the commands exist and never states the default timezone is UTC, contradicting the ground truth.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T27",
  "answer_hash": "e21e1e46740c",
  "correct": true,
  "reason": "The answer accurately explains that `--after` is a pagination cursor rather than a timestamp filter and expects a UUID cursor value.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states that --after is a pagination cursor expecting a UUID value, not an ISO timestamp, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T28",
  "answer_hash": "06a84aba70b1",
  "correct": false,
  "reason": "The answer fails to state that include and exclude domain filters cannot be combined, and in fact incorrectly claims that passing both filters simultaneously is the standard approach.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer claims include and exclude domain filters compose and can be passed together, directly contradicting the rule that they cannot be combined and that -site: must be used in the query text.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T29",
  "answer_hash": "2fc3b6e0fa87",
  "correct": false,
  "reason": "The answer fails to state that ffmpeg downmixes correlated stereo by summing channels at about +3 dB, instead incorrectly describing the matrix as L*0.5 + R*0.5 or up to +6 dB.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer claims the downmix sums channels at 0.5 each (preserving amplitude) or up to +6 dB, contradicting the excerpt's specific rule that correlated stereo sums at about +3 dB, and it never states the +3 dB figure.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T30",
  "answer_hash": "849f15ca4b3d",
  "correct": true,
  "reason": "The answer accurately identifies that the Chrome user-data-dir contains stale tokens from a previous test and explains that the SSO SDK repeatedly attempts to use and refresh the invalid token upon getting 401s.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the Chrome user-data-dir contains stale session tokens from a previous test and that the SSO SDK repeatedly uses/refreshes the invalid token after 401s, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 }
]