[
 {
  "id": "T01",
  "answer_hash": "b74d9f2cb1e3",
  "correct": true,
  "reason": "The answer correctly identifies that the RT-DETR V2 model invokes torch.float64 and provides the --device cpu workaround flag.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer correctly identifies RT-DETR V2 calling torch.float64 as the cause and prescribes the --device cpu flag as the workaround.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T02",
  "answer_hash": "739411ea6adc",
  "correct": true,
  "reason": "The answer correctly identifies the Keychain item as `handoff-own-encryption-key`.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly names the Keychain item handoff-own-encryption-key and instructs deleting it to reset the Mac's local Handoff identity key, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T03",
  "answer_hash": "c3336d3cd434",
  "correct": true,
  "reason": "The answer accurately instructs the user to delete `tmp/openreview_token.json` and retry running the query.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly instructs deleting tmp/openreview_token.json and re-running the query, matching the ground truth rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T04",
  "answer_hash": "b5b80306cc54",
  "correct": true,
  "reason": "The answer correctly specifies the required 2-3 second interval.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the 2–3 second interval between successive downloads, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T05",
  "answer_hash": "a17bba0e357c",
  "correct": true,
  "reason": "The answer accurately states not to rely on the POST response or creation_uuid, and instructs sending a GET request to fetch the list and find the resource via content match.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly says not to rely on the POST response (creation_uuid) and instructs sending a GET list after POST and matching the new resource by content to get the real ID.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T06",
  "answer_hash": "1ae45b2d1829",
  "correct": true,
  "reason": "The answer accurately states the recommended default wake-up interval of 30 minutes (1800 seconds) in accordance with the excerpt and rubric.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the recommended default wake-up interval is 30 minutes (1800 seconds), matching the excerpt exactly.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T07",
  "answer_hash": "cd8ab2a7102f",
  "correct": true,
  "reason": "The answer accurately states that providers have their own sampling strategies and that forcing determinism on long text payloads can cause repetition.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states that forcing determinism on long text payloads can cause repetition, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T08",
  "answer_hash": "9158a48e780a",
  "correct": true,
  "reason": "The answer correctly identifies the recommended minimum padding as 24px (or more).",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the recommended minimum padding is 24px or more, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T09",
  "answer_hash": "740396df60cd",
  "correct": true,
  "reason": "The answer accurately states that `web_search = \"disabled\"` must be set as a top-level key using a string value.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the bare top-level key `web_search = \"disabled\"` with a string value and notes booleans/other forms are rejected, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T10",
  "answer_hash": "61392988e3a3",
  "correct": true,
  "reason": "The answer correctly states that the maximum number allowed is 50 pairs.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the maximum is 50 pairs, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T11",
  "answer_hash": "a125b268bad7",
  "correct": true,
  "reason": "The answer directly states that an unrecognized --gpu value is sent as A100 and not rejected.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the unrecognized --gpu value is not rejected and is sent as A100, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T12",
  "answer_hash": "746fee88cd97",
  "correct": true,
  "reason": "The answer accurately states that the account is authenticated but unverified and specifies completing verification in the DataForSEO panel to fix it.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the account is authenticated but not verified and instructs completing verification in the DataForSEO panel, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T13",
  "answer_hash": "5c73436f10fe",
  "correct": true,
  "reason": "The answer correctly identifies the `--execute` flag and explains that it is required because the CLI defaults to dry-run mode.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the `--execute` flag is required and explains that the CLI defaults to dry-run mode, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T14",
  "answer_hash": "38946a307172",
  "correct": true,
  "reason": "The answer correctly identifies the live store path and explains that the file under ~/Library/Calendars failed because it is a 0-byte decoy file.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the correct group-container path and explains the ~/Library/Calendars file is a 0-byte decoy, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T15",
  "answer_hash": "b4c8f494c051",
  "correct": true,
  "reason": "The answer accurately states the workflow of creating a matplotlib structure draft and passing it via the `-i` flag, and explains that generating from text alone produces unreliable results.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the two-stage workflow of first creating a matplotlib structure draft and passing it as the `-i` input image, and explains that text-only prompting produces unreliable results.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T16",
  "answer_hash": "6404dc9a2d21",
  "correct": true,
  "reason": "The answer accurately states that the sync step only processes `.m4a` and `.qta` files.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the sync reads only .m4a and .qta files, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T17",
  "answer_hash": "a7591aa7b395",
  "correct": true,
  "reason": "The answer correctly states both required steps: pairing the watch with the Mac using Device Hub and restarting the Apple Watch.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states both required steps from the excerpt: pairing the watch with the Mac via Device Hub at least once, and restarting the Apple Watch so the Developer Mode toggle appears.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T18",
  "answer_hash": "831397a8bc61",
  "correct": true,
  "reason": "The answer accurately states that the CLI maps upstream nulls to 0 and correctly identifies `scripts/request_metrics.py` as the script to use instead.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the native CLI maps upstream nulls to 0 and recommends using scripts/request_metrics.py instead, matching the rule exactly.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T19",
  "answer_hash": "a05665bfe6e2",
  "correct": true,
  "reason": "The answer clearly states to use the battery trend by checking for rising battery percentage across a watch window or between snapshots.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states that is_docked and charging_state are null and instructs using the battery trend (rising battery percentage across snapshots or a watch window) to determine docking/charging, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T20",
  "answer_hash": "a5d3521e6182",
  "correct": true,
  "reason": "The answer correctly states that subagent sessions are excluded/skipped by default and specifies using the `--include-subagent` flag to include them.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states subagent sessions are excluded by default and that the --include-subagent flag includes them, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T21",
  "answer_hash": "c7fce70620b2",
  "correct": true,
  "reason": "The answer correctly identifies the strict upper limit as 22 words.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer explicitly states the hard gate that no ordinary spoken sentence may exceed 22 words, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T22",
  "answer_hash": "e7dc62664104",
  "correct": true,
  "reason": "The answer correctly identifies the `--source-root` option and states that paths default to resolving against the current working directory.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states both required elements: --source-root as the option and the CLI's current working directory as the default resolution base.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T23",
  "answer_hash": "1890e8482f69",
  "correct": true,
  "reason": "The answer accurately states that volume and mute state cannot be read due to protocol limitations and that the status endpoint returns PowerState only.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states volume and mute are not readable due to a protocol limitation and that get_status returns PowerState only, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T24",
  "answer_hash": "18a43db44105",
  "correct": true,
  "reason": "The answer correctly identifies exit code 10 and states that a human must refresh the cookie.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states exit code 10 for expired cookie auth failure and says a human must refresh the cookie per docs/cookie_refresh.md, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T25",
  "answer_hash": "e7fa13eac751",
  "correct": true,
  "reason": "The answer accurately states that the maximum allowed duration before rejection is 7200 seconds (2 hours).",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the 7200-second (2-hour) limit and that files longer than that are rejected before the model loads, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T26",
  "answer_hash": "0562cae73ce5",
  "correct": true,
  "reason": "The answer correctly states that the default timezone is UTC.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the default timezone is UTC, matching the excerpt and rubric.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T27",
  "answer_hash": "494021c5a126",
  "correct": true,
  "reason": "The answer correctly explains that the flag is a pagination cursor expecting a UUID rather than an ISO timestamp.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states that --after is a pagination cursor expecting a UUID cursor value, not an ISO timestamp, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T28",
  "answer_hash": "2ab1460ef471",
  "correct": false,
  "reason": "The answer incorrectly instructs the user to combine the `--include-domain` and `--exclude-domain` flags, contradicting the rule that they cannot be combined and failing to mention using the `-site:` operator in the query text.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": false,
  "reason_secondary": "The answer claims the include and exclude domain flags can be combined, directly contradicting the rule that they cannot be combined and that the -site: operator must be used in the query text instead.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T29",
  "answer_hash": "21dfafa3f610",
  "correct": true,
  "reason": "The answer accurately states that downmixing correlated channels results in a ~3 dB increase, causing a false clipping report.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer states the downmix inflates the measured peak by about +3 dB for correlated stereo and causes a false clipping report, matching the excerpt's rule.",
  "judge_secondary": "deepseek-v4.1-flash"
 },
 {
  "id": "T30",
  "answer_hash": "7d17a5b81e6a",
  "correct": true,
  "reason": "The answer accurately states that the Chrome user-data-dir contains stale session tokens from a previous run and explains that the SSO SDK loops repeatedly trying to use and refresh the invalid token upon receiving 401 errors.",
  "judge": "gemini-3.8-flash",
  "correct_secondary": true,
  "reason_secondary": "The answer clearly states the Chrome user-data-dir contains stale SSO session tokens from a previous run and that the SSO SDK repeatedly uses and refreshes the invalid token after 401s, matching the excerpt.",
  "judge_secondary": "deepseek-v4.1-flash"
 }
]