{
"ai_session_search_archive": {
"tasks": [
{
"question": "In our archived AI session lookup workflow, why is it forbidden to reuse an earlier file list when executing a semantic search?",
"excerpt": "Do not reuse an old `tmp/*files*.txt`: a semantic-search file list is also a result allowlist, so an omitted file cannot be returned even if its vectors exist in the cache.",
"fact": "A semantic search file list functions as a result allowlist, preventing any omitted file from being returned even if its embeddings are already in the cache.",
"rubric": {
"must_include": [
"allowlist",
"omitted file cannot be returned even if its vectors/cache exist"
],
"reference_answer": "The file list serves as an allowlist for search results, meaning any omitted file cannot be returned even if its vectors already exist in the cache."
},
"excerpt_verbatim": true
},
{
"question": "When presenting search results from our unified AI session archive, can we generate application deep links for matching Claude Code or Codex sessions?",
"excerpt": "Antigravity / Claude Code / Codex / Second Mind IDs (UUIDs or non-`ses_` prefixes) do not satisfy this contract and cannot produce a deep link.",
"fact": "Claude Code and Codex session IDs cannot produce deep links because their IDs use UUIDs or non-ses_ prefixes that fail the deep link contract.",
"rubric": {
"must_include": [
"cannot produce a deep link",
"UUIDs or non-ses_ prefixes"
],
"reference_answer": "No, Claude Code and Codex session IDs use UUIDs or non-ses_ prefixes that do not satisfy the deep link specification, so they cannot produce deep links."
},
"excerpt_verbatim": true
}
],
"cost": 0.008871
},
"bestpractice_ai_debugging_diagnosis": {
"tasks": [
{
"question": "While using an AI coding assistant to resolve a bug, I noticed it keeps repeatedly editing the exact same block of code and referencing functions or dependencies that don't exist in our project. According to the AI debugging diagnosis guide, what root problem do these symptoms indicate?",
"excerpt": "### 1. Insufficient Context\n\nSymptoms:\n- AI's proposed solutions deviate from actual needs\n- Repeatedly modifying the same piece of code\n- Introducing nonexistent dependencies or functions",
"fact": "Repeatedly editing the same code and introducing nonexistent dependencies or functions are symptoms of insufficient context.",
"rubric": {
"must_include": [
"insufficient context"
],
"reference_answer": "These symptoms indicate that the AI was provided with insufficient context."
},
"excerpt_verbatim": true
},
{
"question": "When debugging issues with an AI assistant genuinely turn out to be architectural problems, what three actions should be attempted first before committing to a large-scale rewrite?",
"excerpt": "Even then, try first:\n- Local refactoring rather than large-scale rewrite\n- Increase test coverage\n- Improve documentation and comments",
"fact": "When facing a genuine architecture problem, you should first try local refactoring, increasing test coverage, and improving documentation and comments before attempting a large-scale rewrite.",
"rubric": {
"must_include": [
"local refactoring",
"increase test coverage",
"improve documentation"
],
"reference_answer": "Before attempting a large-scale rewrite, you should first try local refactoring, increasing test coverage, and improving documentation and comments."
},
"excerpt_verbatim": true
}
],
"cost": 0.005763
},
"bestpractice_ai_product_design": {
"tasks": [
{
"question": "We are setting up a multi-agent workflow that utilizes both O3 and Gemini. According to the design guidelines, what are the inherent strengths and weaknesses of each model that we should plan around?",
"excerpt": "- O3: good at search and exploration, but shallow analysis\n- Gemini: weak at search, but deep analysis and strong synthesis",
"fact": "O3 is effective at search and exploration but shallow in analysis, whereas Gemini is weak at search but excels at deep analysis and strong synthesis.",
"rubric": {
"must_include": [
"O3 is good at search/exploration but shallow at analysis",
"Gemini is weak at search but strong at deep analysis/synthesis"
],
"reference_answer": "O3 excels at search and exploration but provides shallow analysis, while Gemini is weak at search but provides deep analysis and strong synthesis."
},
"excerpt_verbatim": true
},
{
"question": "Before our engineering team begins training and developing models for a new AI feature, what three-step causal chain should guide our planning process?",
"excerpt": "Product form → evaluation method → model development strategy",
"fact": "The causal chain for AI product planning flows in sequence from product form to evaluation method to model development strategy.",
"rubric": {
"must_include": [
"Product form",
"Evaluation method",
"Model development strategy"
],
"reference_answer": "The planning causal chain progresses from product form to evaluation method to model development strategy."
},
"excerpt_verbatim": true
}
],
"cost": 0.01019925
},
"bestpractice_ai_programming_mindset": {
"tasks": [
{
"question": "When designing an AI system, why is a reasoning model's internal reflection considered insufficient on its own for managing external task execution?",
"excerpt": "Reasoning Model's \"reflection\" is stateless — it cannot perceive changes in the external world. By the time thinking ends, the world has already changed.",
"fact": "A reasoning model's reflection is stateless and cannot perceive external world changes while thinking.",
"rubric": {
"must_include": [
"stateless",
"cannot perceive changes in the external world"
],
"reference_answer": "A reasoning model's reflection is stateless and unable to perceive external world changes while it thinks, meaning the environment may have already changed by the time reasoning finishes."
},
"excerpt_verbatim": true
},
{
"question": "In AI-assisted programming, what trade-off is specifically cautioned against as the 'productivity trap'?",
"excerpt": "- **Productivity trap**: sacrificing developer mental bandwidth and flow state to save minor token costs is premature optimization",
"fact": "The productivity trap is prematurely optimizing to save minor token costs at the expense of developer mental bandwidth and flow state.",
"rubric": {
"must_include": [
"mental bandwidth",
"flow state",
"minor token costs"
],
"reference_answer": "The productivity trap is the premature optimization of sacrificing a developer's mental bandwidth and flow state just to save minor token costs."
},
"excerpt_verbatim": true
}
],
"cost": 0.01479975
},
"bestpractice_api_key_management_1password_cli": {
"tasks": [
{
"question": "Where should developers store secret references for local environment injection with 1Password CLI, and what file permission mode should be assigned to it?",
"excerpt": "Store references in the local file `~/.config/op/env.secrets`:\n\n```dotenv\nOPENAI_API_KEY=op://dev/dev-api-keys/openai_api_key\nTAVILY_API_KEY=op://dev/dev-api-keys/tavily_api_key\nGEMINI_API_KEY=op://dev/dev-api-keys/gemini_api_key\n```\n\nThis file contains no plaintext keys and can be safely stored locally (recommended permissions `600`).",
"fact": "Secret references should be stored in ~/.config/op/env.secrets with file permissions set to 600.",
"rubric": {
"must_include": [
"~/.config/op/env.secrets",
"600"
],
"reference_answer": "Secret references should be stored in `~/.config/op/env.secrets` with recommended permissions of `600`."
},
"excerpt_verbatim": true
},
{
"question": "When working with 1Password CLI, what command should I run to inspect existing field entries on the development API keys item to avoid typos, and which JSON property contains their names?",
"excerpt": "If you are not sure the field exists, list existing fields first: `op item get dev-api-keys --vault dev --format json` and inspect `fields[*].label`. This avoids silent failures from typos (e.g., `olama` vs `ollama`).",
"fact": "To check existing field labels and prevent typos, run `op item get dev-api-keys --vault dev --format json` and inspect `fields[*].label`.",
"rubric": {
"must_include": [
"op item get dev-api-keys --vault dev --format json",
"fields[*].label"
],
"reference_answer": "Run `op item get dev-api-keys --vault dev --format json` and inspect `fields[*].label`."
},
"excerpt_verbatim": true
}
],
"cost": 0.008912999999999999
},
"bestpractice_gui_automation": {
"tasks": [
{
"question": "When scraping dynamic web pages (such as infinite scroll) where HTML elements are dynamically removed, what four-step procedure should be followed to capture and extract the underlying data?",
"excerpt": "Dynamic pages (e.g., infinite scroll) have HTML elements that are dynamically removed. Solution:\n\n1. Open Dev Tools → Network Tab\n2. Perform the operation\n3. Export HAR\n4. Programmatically parse the HAR file",
"fact": "For dynamic pages where HTML elements are dynamically removed, data should be captured by opening Dev Tools' Network tab, performing the operation, exporting the HAR, and programmatically parsing the HAR file.",
"rubric": {
"must_include": [
"Open Dev Tools Network Tab",
"Export HAR",
"Programmatically parse the HAR file"
],
"reference_answer": "Open Dev Tools to the Network Tab, perform the operation, export the HAR file, and programmatically parse the HAR file."
},
"excerpt_verbatim": true
},
{
"question": "In GUI automation workflows involving AI, what specific user action causes the automation chain to break, and what two alternatives should be implemented instead?",
"excerpt": "When manual screenshot pasting is required, the automation chain breaks. Solution:\n\n- Let AI take screenshots itself (Playwright)\n- Use screenshot utilities for automatic capture",
"fact": "The automation chain breaks when manual screenshot pasting is required, which should be resolved by having the AI take screenshots itself (Playwright) or using screenshot utilities for automatic capture.",
"rubric": {
"must_include": [
"manual screenshot pasting",
"letting AI take screenshots itself (or Playwright)",
"screenshot utilities for automatic capture"
],
"reference_answer": "The chain breaks when manual screenshot pasting is required; this should be solved by letting AI capture screenshots itself (such as via Playwright) or using screenshot utilities for automatic capture."
},
"excerpt_verbatim": true
}
],
"cost": 0.0059655
},
"bestpractice_internal_visuals": {
"tasks": [
{
"question": "When styling inline elements like badges or chips in our Markdown files, why are we required to use compound classes like `.chip.ok` instead of just `.ok`?",
"excerpt": "When applying CSS classes to decorative inline elements such as chips, badges, or pills, bind two classes, such as `.vx-chip.ok` or `.chip.ok`, instead of using a bare single class such as `.ok`. The higher specificity prevents a parent container's `--fg` from overriding the semantic color.",
"fact": "Compound classes must be used on inline chips and badges to provide higher specificity and prevent the parent container's `--fg` from overriding the semantic color.",
"rubric": {
"must_include": [
"higher specificity",
"parent container's `--fg` overriding the semantic color"
],
"reference_answer": "Compound classes are required because their higher specificity prevents a parent container's `--fg` variable from overriding the element's semantic color."
},
"excerpt_verbatim": true
},
{
"question": "What is the upper word limit for text inside an individual card in our internal documents, and how should longer explanations be handled?",
"excerpt": "Do not put more than 150 words of long prose in a card. A card should contain only a chip, a heading, and one or two judgment sentences. Move long paragraphs into the Markdown body below the card.",
"fact": "A card must contain no more than 150 words of prose, and any long paragraphs should be moved into the Markdown body below the card.",
"rubric": {
"must_include": [
"150 words",
"Markdown body below the card"
],
"reference_answer": "Cards must not exceed 150 words of prose, and any longer paragraphs must be placed in the Markdown body below the card."
},
"excerpt_verbatim": true
}
],
"cost": 0.00984675
},
"bestpractice_interview_evaluation": {
"tasks": [
{
"question": "When assessing edge versus cloud architecture decisions, what deployment setup is specifically recommended if a system has high criticality and failure consequences are severe?",
"excerpt": "2. **Criticality**: severe failure consequences → Edge + redundancy",
"fact": "When system failure consequences are severe, the Edge vs Cloud decision framework recommends deploying to Edge with redundancy.",
"rubric": {
"must_include": [
"Edge",
"redundancy"
],
"reference_answer": "The system should be deployed on the Edge along with redundancy."
},
"excerpt_verbatim": true
},
{
"question": "When discussing cases where classical algorithms still hold value over modern alternatives, which classical algorithm demonstrated superior performance over Superglue in Magic Leap's experience?",
"excerpt": "In Magic Leap practice, ORB features outperformed cutting-edge approaches like Superglue — classical algorithms still have value in specific scenarios.",
"fact": "In Magic Leap practice, classical ORB features outperformed cutting-edge approaches such as Superglue.",
"rubric": {
"must_include": [
"ORB"
],
"reference_answer": "ORB features outperformed cutting-edge approaches like Superglue in Magic Leap's practice."
},
"excerpt_verbatim": true
}
],
"cost": 0.007313999999999999
},
"bestpractice_markdown_html_conversion": {
"tasks": [
{
"question": "I am styling the HTML generated from Markdown documentation to ensure it looks good on mobile screens. What is the recommended minimum padding to apply to the body or main container so text does not cling to the screen edges?",
"excerpt": "Ensure the HTML `body` or main container has sufficient `padding` (recommended 24px or more) to prevent text from hugging the screen edge on mobile devices.",
"fact": "The HTML body or main container should have a recommended padding of 24px or more for mobile adaptation.",
"rubric": {
"must_include": [
"24px"
],
"reference_answer": "The recommended padding for the HTML body or main container is at least 24px."
},
"excerpt_verbatim": true
},
{
"question": "I'm writing a script to concatenate several Markdown chapter files into a single file before converting it to HTML. How many line breaks need to be inserted between the merged files to avoid confusing the heading parser?",
"excerpt": "When merging multiple Markdown files into one large document, you must insert **at least two line breaks** (i.e., one blank line) between files.",
"fact": "Merging multiple Markdown files requires at least two line breaks (one blank line) between files.",
"rubric": {
"must_include": [
"at least two line breaks",
"one blank line"
],
"reference_answer": "You must insert at least two line breaks (equivalent to one blank line) between files."
},
"excerpt_verbatim": true
}
],
"cost": 0.006479250000000001
},
"bestpractice_pdf_to_markdown": {
"tasks": [
{
"question": "When converting PDFs using Docling on Apple Silicon macOS, what causes the layout detection phase to crash under the MPS backend, and what command-line flag should be passed to work around it?",
"excerpt": "On Apple Silicon, Docling's RT-DETR V2 model calls `torch.float64` during layout detection, which the MPS backend doesn't support, causing the entire PDF conversion to crash. The simplest workaround is to run Docling on CPU directly:\n\n```bash\ndocling input.pdf --to md --device cpu --output output_dir\n```",
"fact": "Docling crashes on Apple Silicon MPS because RT-DETR V2 calls unsupported torch.float64 during layout detection, which is resolved by specifying `--device cpu`.",
"rubric": {
"must_include": [
"RT-DETR V2",
"torch.float64",
"--device cpu"
],
"reference_answer": "The layout detection crashes because Docling's RT-DETR V2 model invokes torch.float64 (unsupported by MPS), which can be resolved by passing `--device cpu`."
},
"excerpt_verbatim": true
},
{
"question": "When using the local LM Studio vision-language OCR workflow for scanned PDFs, what default API endpoint and vision model name does the system expect?",
"excerpt": "LM Studio needs to be manually opened with a vision model loaded. Default API is `http://127.0.0.1:1234/v1`, default model name is `qwen/qwen3.5-35b-a3b`.",
"fact": "The local LM Studio VLM OCR fallback defaults to the `http://127.0.0.1:1234/v1` API endpoint and the `qwen/qwen3.5-35b-a3b` vision model.",
"rubric": {
"must_include": [
"http://127.0.0.1:1234/v1",
"qwen/qwen3.5-35b-a3b"
],
"reference_answer": "The workflow defaults to the API endpoint `http://127.0.0.1:1234/v1` and the model name `qwen/qwen3.5-35b-a3b`."
},
"excerpt_verbatim": true
}
],
"cost": 0.023567249999999998
},
"bestpractice_product_decision_analysis": {
"tasks": [
{
"question": "When producing an analysis report under the product decision reverse-engineering methodology, how should the final judgment layer be structured across different audiences or roles?",
"excerpt": "3. **Judgment layer**: conclusions based on the analysis. Provide separate judgments for different roles (Builder / Consumer / Competitor).",
"fact": "The judgment layer of the analysis report must provide separate conclusions for three distinct roles: Builder, Consumer, and Competitor.",
"rubric": {
"must_include": [
"separate judgments/conclusions for different roles",
"Builder, Consumer, and Competitor"
],
"reference_answer": "The judgment layer should present separate conclusions tailored to three specific roles: Builder, Consumer, and Competitor."
},
"excerpt_verbatim": true
},
{
"question": "In the five-step product deconstruction framework, what specific thought experiment is recommended during Step 4 to quickly reveal the core trade-offs of a decision?",
"excerpt": "**Reversal test**: if it had made the opposite choice, who would benefit and who would lose? This thought experiment quickly exposes the core of the trade-off.",
"fact": "The reversal test thought experiment asks who would benefit and who would lose if the product had made the opposite choice.",
"rubric": {
"must_include": [
"reversal test",
"asking who would benefit and who would lose if the opposite choice had been made"
],
"reference_answer": "The framework uses the 'reversal test,' a thought experiment asking who would benefit and who would lose if the product had made the opposite choice."
},
"excerpt_verbatim": true
}
],
"cost": 0.0027495
},
"bestpractice_skill_writing": {
"tasks": [
{
"question": "When defining boundary conditions for handling exceptional failures in an AI agent skill, what are the two minimum degradation scenarios that must be explicitly covered?",
"excerpt": "At minimum cover \"no data\" and \"tool unavailable\" two degradation scenarios",
"fact": "A skill file's boundary conditions must at minimum account for the two degradation scenarios of \"no data\" and \"tool unavailable\".",
"rubric": {
"must_include": [
"no data",
"tool unavailable"
],
"reference_answer": "Boundary conditions must at minimum cover two degradation scenarios: \"no data\" and \"tool unavailable\"."
},
"excerpt_verbatim": true
},
{
"question": "When authoring an agent skill, step-by-step sequential instructions are discouraged. Under what specific condition is it acceptable to retain explicit sequence requirements between steps?",
"excerpt": "Only retain sequence requirements when a step's order materially affects the outcome (e.g., \"must complete X before Y because Y depends on X's output\").",
"fact": "Sequence requirements should only be retained in a skill file when the order of steps materially affects the outcome, such as when a later step depends on an earlier step's output.",
"rubric": {
"must_include": [
"order materially affects the outcome",
"depends on the output"
],
"reference_answer": "Explicit sequence requirements should only be retained when a step's order materially affects the outcome, such as when a step depends on the output of a preceding step."
},
"excerpt_verbatim": true
}
],
"cost": 0.01181325
},
"bestpractice_staged_approach": {
"tasks": [
{
"question": "When preparing data during Stage 1 of an AI-assisted automation pipeline, is it acceptable to stream or hold the fetched source data directly in memory when transitioning into Stage 2 batch processing?",
"excerpt": "- Fully pull source data to local\n- Isolate from the online system\n- Data must be written to disk before entering the next stage\n- Goal: ensure subsequent processing has a stable data foundation",
"fact": "Data collected in Stage 1 must be written to disk before proceeding to the next stage.",
"rubric": {
"must_include": [
"written to disk",
"before entering the next stage"
],
"reference_answer": "No, data must be written to disk before entering the next stage to ensure a stable data foundation."
},
"excerpt_verbatim": true
},
{
"question": "Before executing any destructive action in the automation workflow, what four checks must be confirmed according to the Dry Run checklist?",
"excerpt": "Dry Run checklist:\n- [ ] Confirm operation scope (which files/records are affected)\n- [ ] Confirm operation content (what specific changes)\n- [ ] Confirm rollback capability (backup or version control exists)\n- [ ] Confirm execution environment (not production)",
"fact": "The Dry Run checklist requires confirming operation scope, operation content, rollback capability, and the execution environment.",
"rubric": {
"must_include": [
"operation scope",
"operation content",
"rollback capability and execution environment (not production)"
],
"reference_answer": "The Dry Run checklist requires confirming the operation scope, operation content, rollback capability (backup or version control exists), and execution environment (not production)."
},
"excerpt_verbatim": true
}
],
"cost": 0.009150750000000001
},
"bestpractice_temporal_info_verification": {
"tasks": [
{
"question": "When configuring a Tavily search to verify potentially unfamiliar or newer software versions or model names, what specific parameter values should be used for the search depth and the maximum number of results?",
"excerpt": "Use Tavily search, `search_depth=\"advanced\"`, `max_results=5`.",
"fact": "When running a Tavily search for verification, search_depth must be set to \"advanced\" and max_results must be set to 5.",
"rubric": {
"must_include": [
"search_depth=\"advanced\"",
"max_results=5"
],
"reference_answer": "The Tavily search should be configured with `search_depth=\"advanced\"` and `max_results=5`."
},
"excerpt_verbatim": true
},
{
"question": "When conducting a physical anchor check on AI-generated orbital parameters for a geosynchronous satellite, what specific position and altitude requirements should be used to verify validity?",
"excerpt": "Physical common sense (geosynchronous satellites must be above the equator at approximately 35,786 km) revealed the AI's hallucination.",
"fact": "Geosynchronous satellites must be positioned directly above the equator at approximately 35,786 km.",
"rubric": {
"must_include": [
"above the equator",
"35,786 km"
],
"reference_answer": "Geosynchronous satellites must be located directly above the equator at an altitude of approximately 35,786 km."
},
"excerpt_verbatim": true
}
],
"cost": 0.00714675
},
"compressor": {
"tasks": [
{
"question": "I am trying to parse output from Apple Compressor using the command line by running `Compressor -monitor -batchid \"12345\" -once -format json`, but it errors with `Invalid parameter: -format`. What specific command-line argument positioning issue is causing this?",
"excerpt": "On at least some Compressor versions, `-format json` must be placed immediately after `-monitor`. Placing it at the end can fail with `Invalid parameter: -format`.",
"fact": "-format json must be placed immediately after -monitor on the command line, or Compressor may return an 'Invalid parameter: -format' error.",
"rubric": {
"must_include": [
"-format json must be placed immediately after -monitor"
],
"reference_answer": "In Compressor CLI, `-format json` must be placed immediately after the `-monitor` flag rather than at the end of the command."
},
"excerpt_verbatim": true
},
{
"question": "In Apple Compressor `.compressorsetting` files, what does a value of `-100` for width, height, or video-encode bounds represent?",
"excerpt": "If a `.compressorsetting` file configures `video-encode` `bounds` or `automatic` width and height to `-100`, the preset dynamically inherits the source resolution (\"same as source\" / adaptive) rather than enforcing a fixed dimension.",
"fact": "A width, height, or bounds value of -100 in a .compressorsetting file dynamically inherits the source resolution ('same as source' / adaptive).",
"rubric": {
"must_include": [
"same as source",
"inherits the source resolution"
],
"reference_answer": "A value of -100 means the preset dynamically inherits the resolution of the source file (acting as \"same as source\" / adaptive) rather than using a fixed dimension."
},
"excerpt_verbatim": true
}
],
"cost": 0.003983250000000001
},
"delayed_execution": {
"tasks": [
{
"question": "I have a scheduled job that requires AI judgment and want to set it up properly using the ecosystem tools. What is the recommended workflow and combination of tools to handle this delayed AI task?",
"excerpt": "For delayed tasks requiring AI judgment, the correct combination is: first use `opencode_skill submit --dry-run` to pre-check the submission chain, then use `process-launcher` to trigger the real `opencode_skill submit` at the future time.",
"fact": "Delayed tasks requiring AI judgment must first be pre-checked with `opencode_skill submit --dry-run` and then scheduled via `process-launcher` to run `opencode_skill submit`.",
"rubric": {
"must_include": [
"opencode_skill submit --dry-run",
"process-launcher",
"opencode_skill submit"
],
"reference_answer": "First use `opencode_skill submit --dry-run` to pre-check the submission chain, and then use `process-launcher` to trigger `opencode_skill submit` at the future time."
},
"excerpt_verbatim": true
},
{
"question": "I need to schedule a command for delayed execution, but the tool does not offer a dry-run, preview, or check option. According to the safety rules, what must be done before proceeding?",
"excerpt": "Before delayed execution, run the target command's dry-run / check / preview mode; if no pre-check capability exists, explicitly state the risk.",
"fact": "If the command to be run after a delay lacks a dry-run or pre-check capability, the risk must be explicitly stated.",
"rubric": {
"must_include": [
"explicitly state the risk"
],
"reference_answer": "If no pre-check capability exists for the target command, you must explicitly state the risk."
},
"excerpt_verbatim": true
}
],
"cost": 0.0066359999999999995
},
"deployment_github_actions_koyeb": {
"error": "bad json",
"raw": "```json\n{\n \"tasks\": [\n {\n \"question\": \"I have a Dockerized application, but when deploying it to Koyeb via GitHub Actions, the build logs indicate that it is installing Python instead of building from my Dockerfile. Which specific action input parameter must be set to resolve this, and why does this issue occur?\",\n \"excerpt\": \"3. **Explicitly specify `git-builder: \"docker\"`** — otherwise Koyeb detects Python/Node files and falls back to buildpack\",\n \"fact\": \"You must explicitly specify git-builder as \\\"docker\\\" in the GitHub Actions step, otherwise Koyeb detects Python or Node files and defaults to using a buildpack.\",\n \"rubric\": {\n \"must_include\": [\n \"git-builder: \\\"docker\\\" (or git-builder set to docker)\",\n \"falls back to buildpack (or detects Python/Node files)\"\n ],\n \"reference_answer\": \"You must set `git-builder: \\\"docker\\\"` because otherwise Koyeb automatically detects Python/Node files and falls back to using buildpack.\"\n }\n },\n {\n \"question\": \"While configuring the `service-regions` input for a Koyeb deployment in GitHub Actions, the deployment failed with the error `cannot mix continental and metropolitan regions`. What causes this error, and which codes should be used if targeting continental regions?\",\n \"excerpt\": \"| `cannot mix continental and metropolitan regions` | Mixed `na` with `fra`, etc. | Use only continental regions (`na`/`eu`/`ap`) |\",\n \"fact\": \"Mixing continental region codes with metropolitan region codes causes an error, and continental deployments must only use na, eu, or ap.\",\n \"rubric\": {\n \"must_include\": [\n \"mixing continental regions with metropolitan regions (such as na with fra)\",\n \"na, eu, ap\"\n ],\n \"reference_answer\": \"The error is caused by mixing continental regions with metropolitan regions (e.g., combining na with fra), and only continental regions (`na`, `eu`, `ap`) should be used.\"\n }\n }\n ]\n}\n```"
},
"growth_analytics": {
"tasks": [
{
"question": "When checking our Kit Welcome Sequence metrics, I noticed that the sequence subscriber count hovers around 10 rather than growing over time as our total subscribers increase. Has the automation stalled, and how should this count be interpreted?",
"excerpt": "`subscriber_count` is a flow metric (subscribers exit after completing all 3 emails); with organic subscription of 2-3/day, the steady-state in-sequence count is 8-15. Judge health by whether it stays in that band; do not expect it to grow with cumulative subscriptions.",
"fact": "The Welcome Sequence subscriber count is a flow metric whose subscribers exit after three emails, so a steady-state count between 8 and 15 is expected healthy behavior rather than cumulative growth.",
"rubric": {
"must_include": [
"It is a flow metric where subscribers exit after completing all 3 emails (it does not accumulate)",
"A steady-state count of 8 to 15 is normal and healthy for organic subscription rates"
],
"reference_answer": "The automation is working properly because the count is a flow metric where subscribers exit after completing all 3 emails, and maintaining a steady-state band of 8 to 15 indicates healthy performance."
},
"excerpt_verbatim": true
},
{
"question": "I ran the GSC metrics script without any extra site parameters to check search performance for yage.ai, but the returned numbers don't match the site. What domain does the tool query by default, and how can I target yage.ai?",
"excerpt": "Default query target remains `sc-domain:grapeot.me`; use `--site sc-domain:yage.ai` or set `GSC_SITE_URL` when querying yage.ai",
"fact": "The GSC tool defaults to querying sc-domain:grapeot.me, and querying yage.ai requires using the --site sc-domain:yage.ai flag or setting the GSC_SITE_URL environment variable.",
"rubric": {
"must_include": [
"The default query target is `sc-domain:grapeot.me`",
"Specify yage.ai by passing `--site sc-domain:yage.ai` or setting the `GSC_SITE_URL` environment variable"
],
"reference_answer": "The tool targets sc-domain:grapeot.me by default; to query yage.ai, you must pass `--site sc-domain:yage.ai` or set the `GSC_SITE_URL` environment variable."
},
"excerpt_verbatim": true
}
],
"cost": 0.0131505
},
"mac_universal_clipboard": {
"tasks": [
{
"question": "Universal Clipboard is still broken on my Mac after trying basic restarts and toggling Handoff off and on. Is there a specific Keychain item I can delete to reset my Mac's local Handoff identity key before resorting to signing out of iCloud?",
"excerpt": "Consider deleting the Keychain item named `handoff-own-encryption-key`, then restart Handoff. This resets the Mac's local Handoff identity key.",
"fact": "Deleting the Keychain item named `handoff-own-encryption-key` resets the Mac's local Handoff identity key.",
"rubric": {
"must_include": [
"handoff-own-encryption-key"
],
"reference_answer": "You can delete the Keychain item named handoff-own-encryption-key and then restart Handoff to reset the local identity key."
},
"excerpt_verbatim": true
},
{
"question": "When running Continuity diagnostic commands in zsh to inspect Universal Clipboard logs, why is it recommended to invoke `/usr/bin/log` by its full path rather than calling `log` directly?",
"excerpt": "Use `/usr/bin/log` rather than bare `log` in zsh, because `log` can resolve to a shell builtin or function in some environments.",
"fact": "The full path `/usr/bin/log` is recommended in zsh because bare `log` can resolve to a shell builtin or function.",
"rubric": {
"must_include": [
"builtin",
"function"
],
"reference_answer": "The full path `/usr/bin/log` should be used because bare `log` can resolve to a shell builtin or function in some zsh environments."
},
"excerpt_verbatim": true
}
],
"cost": 0.007275749999999999
},
"openreview": {
"tasks": [
{
"question": "While using the OpenReview CLI tool, I ran into an HTTP 403 error caused by an expired authentication token. What specific step should I take to resolve this error and run my query?",
"excerpt": "If a 403 occurs due to an expired token, delete `tmp/openreview_token.json` and retry once.",
"fact": "To resolve a 403 error caused by an expired token, delete `tmp/openreview_token.json` and retry once.",
"rubric": {
"must_include": [
"delete `tmp/openreview_token.json`",
"retry the command"
],
"reference_answer": "Delete `tmp/openreview_token.json` and retry the command once."
},
"excerpt_verbatim": true
},
{
"question": "I retrieved accepted papers from a conference venue using the OpenReview CLI, but cannot find the authors' affiliated institutions anywhere in the paper output. Why is this information missing from the paper objects, and what workflow must be used to get it?",
"excerpt": "1. **Affiliation is not on the paper.** Papers only carry authorids; institution info is in the profile. Flow: papers → extract authorids → batch-query profiles → read history.",
"fact": "Paper records in OpenReview only include author IDs rather than affiliations, requiring you to extract the IDs and batch-query author profiles to obtain institution history.",
"rubric": {
"must_include": [
"papers only carry author IDs (authorids) / affiliation is not stored on paper records",
"extract author IDs and query the author profiles to read institution history"
],
"reference_answer": "Affiliations are not included on paper records (which only contain author IDs), so you must extract the author IDs and query the author profiles to retrieve their institution history."
},
"excerpt_verbatim": true
}
],
"cost": 0.01666125
},
"playwright_ajax_capture": {
"tasks": [
{
"question": "I'm automating a rich text editor (TipTap/ProseMirror) in my Playwright script, but setting innerHTML directly on the editor doesn't trigger React state and the submit button doesn't fire. What is the recommended way to enter new text and clear existing text?",
"excerpt": "Use `page.keyboard.type(text, delay=30)` to type via the keyboard, which triggers React state properly. First `page.click(editor)` to focus, then `keyboard.type`. To clear existing content: `keyboard.press(\"Meta+A\")` + `keyboard.press(\"Backspace\")`.",
"fact": "For TipTap/ProseMirror editors, focus the editor and type using page.keyboard.type, and clear content using Meta+A followed by Backspace.",
"rubric": {
"must_include": [
"Focus the editor and use keyboard typing (`page.keyboard.type`) rather than setting innerHTML",
"Clear existing content using `keyboard.press(\"Meta+A\")` followed by `keyboard.press(\"Backspace\")`"
],
"reference_answer": "Click the editor to focus and type via `page.keyboard.type` to trigger React state, clearing any existing content beforehand with `keyboard.press(\"Meta+A\")` followed by `keyboard.press(\"Backspace\")`."
},
"excerpt_verbatim": true
},
{
"question": "When reverse-engineering an internal API, I noticed that creating a resource returns a response without an id field (only a creation_uuid), breaking subsequent actions that need the ID. How should I reliably obtain the real ID of the newly created resource?",
"excerpt": "Don't use the response's `id` directly after POST. If you need the new resource's ID, send a GET (fetch list) after POST and find the just-created resource by content match.",
"fact": "If a POST response lacks a resource ID, send a GET list request after the POST and locate the newly created item by matching its content.",
"rubric": {
"must_include": [
"Do not rely directly on the POST response ID or uuid",
"Send a GET request to fetch the list and find the resource by content match"
],
"reference_answer": "Send a GET request to fetch the resource list after the POST request and locate the new resource's real ID by matching its content."
},
"excerpt_verbatim": true
}
],
"cost": 0.013034249999999999
},
"project_scaffold": {
"tasks": [
{
"question": "I'm setting up the README for a project that we plan to publish to public GitHub. What mandatory statement needs to be added to the Privacy section to declare that it is safe to open-source?",
"excerpt": "In the README's Privacy section, write clearly: `This repository is designed to be publishable with only fake examples.`",
"fact": "The Privacy section of the README in a public repo must explicitly state that the repository is designed to be publishable with only fake examples.",
"rubric": {
"must_include": [
"This repository is designed to be publishable with only fake examples."
],
"reference_answer": "The README's Privacy section must state: \"This repository is designed to be publishable with only fake examples.\""
},
"excerpt_verbatim": true
},
{
"question": "Our open-source repository contains several distinct agent skills. What is the rule for exposing them at the global workspace level to keep discovery clean?",
"excerpt": "If the public repo contains multiple skills, expose exactly one root/router skill to the global workspace level. Use that root skill to link to focused local files inside the repo. Do not symlink every focused skill globally; that makes discovery noisy and makes private/public overlays harder to reason about.",
"fact": "Repositories containing multiple skills must expose only one root/router skill to the global workspace level and avoid symlinking every individual skill globally.",
"rubric": {
"must_include": [
"expose exactly one root/router skill to the global workspace level",
"link to focused local files rather than symlinking every skill globally"
],
"reference_answer": "You must expose exactly one root or router skill globally that links to focused local files inside the repo, rather than symlinking each individual skill."
},
"excerpt_verbatim": true
}
],
"cost": 0.014247000000000001
},
"share_report": {
"error": "bad json",
"raw": "```json\n{\n \"tasks\": [\n {\n \"question\": \"I am preparing the English translation of a report whose Chinese slug is quantum-algorithms-guide-20260415. What is the exact convention for naming the English article's slug?\",\n \"excerpt\": \"Naming: insert `-en` before the date in the Chinese slug, e.g. `attention-residuals-intuition-20260318` → `attention-residuals-intuition-en-20260318`.\",\n \"fact\": \"English slugs are formed by inserting `-en` directly before the date suffix of the Chinese slug.\",\n \"rubric\": {\n \"must_include\": [\n \"insert `-en` before the date\",\n \"quantum-algorithms-guide-en-20260415\"\n ],\n \"reference_answer\": \"The English slug must insert `-en` immediately before the date suffix, resulting in quantum-algorithms-guide-en-20260415.\"\n }\n },\n {\n \"question\": \"Why shouldn't SEO metadata, such as the canonical link, be injected into article pages using pandoc's `--include-in-header` flag during generation?\",\n \"excerpt\": \"pandoc's `--embed-resources` sees `` as an external resource to fetch, base64-inlining the remote HTML into the generated HTML, triggering exponential file bloat.\",\n \"fact\": \"Pandoc's `--embed-resources` flag mistakenly fetches canonical links as external assets and base64-inlines the target HTML into the page, causing exponential file bloat.\",\n \"rubric\": {\n \"must_include\": [\n \"--embed-resources\",\n \"base64-inlines the remote HTML or external resource\",\n \"file bloat\"\n ],\n \"reference_answer\": \"Because pandoc's `--embed-resources` treats the canonical link as an external resource to fetch and base64-inlines the remote HTML, causing severe file bloat.\"\n }\n }\n ]\n}\n```"
},
"workflow_astra_coordinator": {
"tasks": [
{
"question": "Under what specific conditions does the coordinator role activate for an incoming task?",
"excerpt": "This role activates only when both conditions hold:\n1. Trusted runtime context confirms the actual primary model is GPT-6 Astra, rather than merely an agent label or display name.\n2. The user explicitly asks the main thread to act as coordinator or PI for the current task.",
"fact": "The coordinator role only activates if trusted runtime context confirms the primary model is GPT-6 Astra and the user explicitly asks the main thread to act as coordinator or PI.",
"rubric": {
"must_include": [
"primary model is GPT-6 Astra",
"user explicitly asks"
],
"reference_answer": "It activates only when trusted runtime context confirms the actual primary model is GPT-6 Astra and the user explicitly asks the main thread to act as coordinator or PI for the task."
},
"excerpt_verbatim": true
},
{
"question": "When an agent initiates first contact with another session but cannot determine its own session identifier, what should it do?",
"excerpt": "If the sender's own session id is unknown, mark it unknown. Do not guess. Discovering one's own address is not a prerequisite for sending.",
"fact": "If an agent's own session ID is unknown when sending first contact, it must mark it unknown and proceed without guessing.",
"rubric": {
"must_include": [
"mark it unknown",
"do not guess"
],
"reference_answer": "The sender must mark its session ID as unknown and proceed with sending, rather than guessing or waiting to discover its own address."
},
"excerpt_verbatim": true
}
],
"cost": 0.00884025
},
"workflow_bilibili_whisper_transcription": {
"tasks": [
{
"question": "When running the video transcription script, only the full text is saved by default. Which command-line flag should I pass if I explicitly need segmented text included in the output?",
"excerpt": "By default keep only the full text; add this only when segmented text is explicitly needed:\n\n```bash\n--include-segments\n```",
"fact": "Passing the `--include-segments` flag to the transcription script is required when segmented text is explicitly needed alongside the default full text.",
"rubric": {
"must_include": [
"--include-segments"
],
"reference_answer": "You must pass the `--include-segments` flag to include segmented text in the output."
},
"excerpt_verbatim": true
},
{
"question": "When downloading audio tracks from Bilibili for batch transcription, how long of an interval should be maintained between successive downloads to prevent triggering anti-scraping protections?",
"excerpt": "2. **Single-threaded download** → Download audio one by one, 2-3 second interval between each, to avoid triggering anti-scraping",
"fact": "Audio downloads should be executed single-threaded with a 2 to 3 second interval between each download to avoid anti-scraping triggers.",
"rubric": {
"must_include": [
"2-3 second"
],
"reference_answer": "You should maintain a 2 to 3 second interval between each download."
},
"excerpt_verbatim": true
}
],
"cost": 0.0069225
},
"workflow_cognitive_profile_extraction": {
"tasks": [
{
"question": "What is the recommended pre-execution check regarding the current model before running the cognitive profile extraction workflow, and what should be done if it fails?",
"excerpt": "Pre-execution check: Confirm whether the current model ID contains opus.\n\n- Is Opus → Continue. Your context window is extremely precious; your core capabilities are design, quality assurance, and writing. All research, data processing, and code writing must be delegated to sub-agents, and parallel by default. Writing — including axiom text, indices, and final reports — must be done by you personally; do not outsource.\n- Not Opus → Pause and ask the user:\n\n > \"This workflow is designed to be executed by Opus — Opus's context window and writing capability are core assumptions of the process. Your current model is not Opus. Are you sure you want to continue? If this is a model selection mistake, it is recommended to switch to Opus before starting.\"",
"fact": "The workflow requires checking if the model ID contains 'opus', and if not, pausing to ask the user for confirmation and advising a switch to Opus.",
"rubric": {
"must_include": [
"check whether the model ID contains 'opus'",
"pause and ask the user whether to continue / recommend switching to Opus"
],
"reference_answer": "Check whether the current model ID contains 'opus'; if it does not, pause and ask the user if they want to continue while recommending they switch to Opus."
},
"excerpt_verbatim": false
},
{
"question": "When extracting a target person's cognitive profile across multiple rounds, how many convergence signals must be satisfied before deciding to converge, and what is recommended if only two are met?",
"excerpt": "Converge when 3 of the 4 signals are met. When 2 are met, recommend one more round of lightweight verification for confirmation.",
"fact": "Convergence requires meeting at least 3 of the 4 signals; if only 2 are met, one more round of lightweight verification is recommended.",
"rubric": {
"must_include": [
"3 of the 4 signals (or 3 signals)",
"recommend one more round of lightweight verification"
],
"reference_answer": "You should converge when 3 of the 4 convergence signals are met, and recommend one additional round of lightweight verification if only 2 are met."
},
"excerpt_verbatim": false
}
],
"cost": 0.0066254999999999994
},
"workflow_deep_research_survey": {
"tasks": [
{
"question": "When setting up parallel sub-agents to research different dimensions of a topic, what minimum amount of overlap should be designed between their assigned dimensions, and what is the purpose of this requirement?",
"excerpt": "Dimensions must have ≥50% overlap so different agents have the opportunity to discover different interpretations of the same information or contradictory conclusions.",
"fact": "Research dimensions for sub-agents must have at least 50% overlap to enable discovering divergent interpretations or contradictory conclusions.",
"rubric": {
"must_include": [
"at least 50% (or ≥50%) overlap",
"discovering different interpretations or contradictory conclusions"
],
"reference_answer": "Dimensions must have at least 50% overlap so different agents have the opportunity to discover different interpretations of the same information or contradictory conclusions."
},
"excerpt_verbatim": true
},
{
"question": "In the deep research workflow's source-level fact-check phase, what three verdict classifications must be used when evaluating each primary source against its paraphrase in fact_check.md?",
"excerpt": "3. Write `tmp//fact_check.md`. For every item, record the source, current paraphrase, verdict (`consistent`, `partial deviation`, or `hallucination`), and original excerpt.",
"fact": "Each item logged during the source-level fact-check must receive a verdict of consistent, partial deviation, or hallucination.",
"rubric": {
"must_include": [
"consistent",
"partial deviation",
"hallucination"
],
"reference_answer": "The three verdict classifications used in fact_check.md are consistent, partial deviation, and hallucination."
},
"excerpt_verbatim": true
}
],
"cost": 0.0138135
},
"workflow_knowledge_flywheel": {
"tasks": [
{
"question": "Our knowledge extraction process has hit a stagnation point where the data we need to extract is extremely rare. What specific breakthrough method should we apply to overcome this stagnation point?",
"excerpt": "Data flywheel \"stagnation point\" (extremely rare data) breakthrough method:\n Use VLM for initial harvesting\n Correct via Human-in-the-loop\n Form cold-start seed data",
"fact": "Overcoming a stagnation point caused by extremely rare data requires initial harvesting with a VLM, human-in-the-loop correction, and forming cold-start seed data.",
"rubric": {
"must_include": [
"VLM for initial harvesting",
"human-in-the-loop correction",
"cold-start seed data"
],
"reference_answer": "Use a VLM for initial harvesting, refine it through human-in-the-loop correction, and form cold-start seed data."
},
"excerpt_verbatim": true
},
{
"question": "We are setting up local infrastructure using a 5090 cluster and vLLM to serve a 32B parameter model with a 128k context window. How many GPUs are required for this configuration?",
"excerpt": "5090 cluster + vLLM: 32B model with 128k context needs only two GPUs",
"fact": "A 32B model with a 128k context window deployed on a 5090 cluster using vLLM requires only two GPUs.",
"rubric": {
"must_include": [
"two GPUs"
],
"reference_answer": "It requires only two GPUs."
},
"excerpt_verbatim": true
}
],
"cost": 0.011943
},
"workflow_parallel_subagents": {
"tasks": [
{
"question": "What is the recommended percentage overlap between subagents when splitting research or creative tasks versus code or execution tasks?",
"excerpt": "| Task Type | Overlap Range | Reason |\n|-----------|---------------|--------|\n| Research/Creative tasks | 30% - 50% | Cross-validation, gap-filling |\n| Code/Execution tasks | 0% - 20% | Efficiency priority, reduce duplication |",
"fact": "Research and creative tasks should have a 30% to 50% overlap across subagents, whereas code and execution tasks should have a 0% to 20% overlap.",
"rubric": {
"must_include": [
"30% to 50% (or 30-50%) for research or creative tasks",
"0% to 20% (or 0-20%) for code or execution tasks"
],
"reference_answer": "Research and creative tasks should have an overlap of 30% to 50%, while code and execution tasks should have an overlap of 0% to 20%."
},
"excerpt_verbatim": true
},
{
"question": "Why shouldn't you default to setting temperature to 0 on subagents to increase stability?",
"excerpt": "Do not default to setting `temperature: 0` for agents \"for stability.\" Many providers/models have their own sampling strategies; DS4 also does deterministic handling for protocol structures, but forcing determinism on long text payloads may cause repetition.",
"fact": "Forcing determinism on long text payloads by setting temperature to 0 can cause repetition.",
"rubric": {
"must_include": [
"repetition",
"forcing determinism on long text payloads"
],
"reference_answer": "You should not default to setting temperature to 0 because providers have their own sampling strategies and forcing determinism on long text payloads can cause repetition."
},
"excerpt_verbatim": true
}
],
"cost": 0.00492825
},
"workflow_public_consensus_net_income_audit": {
"tasks": [
{
"question": "In the consensus net income audit table, under what conditions should the revision Direction column be marked as Up, Down, or Flat rather than Not Comparable?",
"excerpt": "Write Up / Down / Flat only when both current and baseline are comparable; otherwise write `Not Comparable`",
"fact": "The Direction column may only be marked as Up, Down, or Flat when both the current estimate and the baseline are comparable.",
"rubric": {
"must_include": [
"both the current estimate and the baseline must be comparable",
"write Not Comparable if they are not comparable"
],
"reference_answer": "You may only mark Up, Down, or Flat when both the current estimate and the baseline are comparable; otherwise, you must record Not Comparable."
},
"excerpt_verbatim": true
},
{
"question": "While extracting consensus figures from MarketScreener, I noticed that the finances page provides quarterly or semi-annual tables while the calendar page displays an annual row. Which presentation should take precedence for the full-year figure?",
"excerpt": "When a direct calendar annual row is available, prefer it; do not pass Q4 or semi-annual values off as full-year.",
"fact": "A direct annual row on the calendar page must be prioritized over Q4 or semi-annual data from the finances page.",
"rubric": {
"must_include": [
"prefer the direct calendar annual row",
"do not use or substitute Q4 or semi-annual values as full-year figures"
],
"reference_answer": "When a direct calendar annual row is available, prefer it, and do not pass off Q4 or semi-annual values as full-year numbers."
},
"excerpt_verbatim": true
}
],
"cost": 0.011437499999999998
},
"workflow_research_paper_survey_writing": {
"tasks": [
{
"question": "When analyzing a research paper's technical niche and identifying which layer of the stack it affects, how does placing the paper at a lower layer change how its impact propagates?",
"excerpt": "Which layer it is on determines the impact propagation path — lower-layer improvements have wider reach but slower propagation.",
"fact": "Lower-layer improvements have wider reach but slower propagation across the stack.",
"rubric": {
"must_include": [
"wider reach (or broader reach / affects more systems)",
"slower propagation (or takes longer to propagate / spread)"
],
"reference_answer": "Improvements at lower layers of the technical stack have a wider reach, but their impact propagates more slowly."
},
"excerpt_verbatim": true
},
{
"question": "When conducting a niche analysis of a research paper, how do the strategic market consequences differ between establishing a brand-new capability versus driving down the cost of an existing capability?",
"excerpt": "The strategic implications are entirely different: the former opens new markets; the latter reallocates existing ones.",
"fact": "Creating a new capability opens new markets, whereas compressing the cost of an existing capability reallocates existing markets.",
"rubric": {
"must_include": [
"creating a new capability opens new markets",
"cost compression reallocates existing markets"
],
"reference_answer": "Introducing a new capability opens new markets, whereas cost compression reallocates existing markets."
},
"excerpt_verbatim": true
}
],
"cost": 0.0156165
},
"workflow_watchdog": {
"tasks": [
{
"question": "I am dispatching several long-running background sub-agent tasks in my workflow. What is the recommended default wake-up interval to set for checking in on their progress?",
"excerpt": "After dispatching a task, set a reasonable inspection wake-up, default about 30 minutes (1800s).",
"fact": "The default inspection wake-up interval after dispatching a task is about 30 minutes (1800 seconds).",
"rubric": {
"must_include": [
"30 minutes",
"1800"
],
"reference_answer": "The recommended default inspection wake-up interval is about 30 minutes (1800 seconds)."
},
"excerpt_verbatim": true
},
{
"question": "When monitoring a background subprocess in Claude Code and discovering it has gotten stuck in a repetitive loop without progress, what specific tool or command should be used to terminate the task?",
"excerpt": "Kill it (use `TaskStop` in Claude Code), and proceed with the partial results, or switch methods and redo.",
"fact": "In the Claude Code harness, TaskStop is used to kill a background task that is stuck in a loop.",
"rubric": {
"must_include": [
"TaskStop"
],
"reference_answer": "In Claude Code, you should terminate the stuck process using TaskStop."
},
"excerpt_verbatim": true
}
],
"cost": 0.00457425
},
"antigravity_cli": {
"tasks": [
{
"question": "I am trying to run the Antigravity CLI (`agy`), but `agy models` fails because I am not authenticated. What command or environment variable should I use to sign in?",
"excerpt": "- **Authentication Channel**: `agy` authenticates via Antigravity subscription credentials stored in the system keyring. It does not read `GEMINI_API_KEY`. If `agy models` fails to list models, sign in via the Antigravity desktop application. Do not fall back to `GEMINI_API_KEY` (that routes to a distinct API billing path).",
"fact": "Antigravity CLI does not support an `agy login` subcommand or `GEMINI_API_KEY`; authentication must be completed by signing in through the Antigravity desktop application.",
"rubric": {
"must_include": [
"sign in via the Antigravity desktop application",
"does not read or use GEMINI_API_KEY"
],
"reference_answer": "You cannot authenticate using `GEMINI_API_KEY` or an `agy login` command; instead, you must sign in via the Antigravity desktop application so credentials are saved in the system keyring."
},
"excerpt_verbatim": true
},
{
"question": "What is the default timeout if I execute a headless task with `agy --print` without specifying the `--print-timeout` flag in Antigravity CLI 1.2.0?",
"excerpt": "| `--print-timeout` | Internal execution timeout; 1.2.0 defaults to **0** (wait until the turn completes). Set explicitly (e.g. `10m`) so a stuck turn cannot hang forever; keep the outer wrapper timeout higher so logs can flush. |",
"fact": "In Antigravity CLI 1.2.0, `--print-timeout` defaults to 0, which waits until the turn completes.",
"rubric": {
"must_include": [
"0",
"wait until the turn completes"
],
"reference_answer": "The default `--print-timeout` value is 0, which means it will wait indefinitely until the turn completes."
},
"excerpt_verbatim": true
}
],
"cost": 0.01179225
},
"claude_code": {
"tasks": [
{
"question": "I want to run an automated Claude Code command without it discovering any CLAUDE.md files, loading plugins, executing hooks, or reading from the keychain. What CLI flag enables this isolated behavior?",
"excerpt": "| `--bare` | Skip hooks, plugins, `CLAUDE.md` discovery, and keychain reads. |",
"fact": "The `--bare` flag skips hooks, plugins, CLAUDE.md discovery, and keychain reads.",
"rubric": {
"must_include": [
"`--bare`"
],
"reference_answer": "You should use the `--bare` flag to skip hooks, plugins, CLAUDE.md discovery, and keychain reads."
},
"excerpt_verbatim": true
},
{
"question": "When invoking Claude Code in non-interactive mode, what value should I pass to the `--tools` flag if I need to completely disable all built-in tools?",
"excerpt": "| `--tools` | Built-in allow list (e.g., `Bash,Edit,Read`). `default` preserves all tools; `\"\"` disables tools. |",
"fact": "Passing an empty string (`\"\"`) to the `--tools` flag disables all built-in tools.",
"rubric": {
"must_include": [
"empty string",
"\"\""
],
"reference_answer": "Pass an empty string (`\"\"`) to the `--tools` flag to disable all tools."
},
"excerpt_verbatim": true
}
],
"cost": 0.010587
},
"codex_cli": {
"tasks": [
{
"question": "I am upgrading my automation script to Codex CLI 0.156.1, and it previously used `--full-auto` or `--ask-for-approval`. Since those flags are now rejected, what configuration override and flag should I use instead to ensure headless non-interactive execution?",
"excerpt": "Never include any of these in exec commands. Control isolation via `--sandbox` and the approval policy via the config override `-c approval_policy=\"never\"`.",
"fact": "In Codex CLI 0.156.1, execution isolation must be controlled via `--sandbox` and the approval policy set using the config override `-c approval_policy=\"never\"` instead of removed approval flags.",
"rubric": {
"must_include": [
"-c approval_policy=\"never\"",
"--sandbox"
],
"reference_answer": "Use `--sandbox` to control isolation and `-c approval_policy=\"never\"` to set the approval policy."
},
"excerpt_verbatim": true
},
{
"question": "My custom OpenAI-compatible backend is failing with HTTP 400 because `web_search` is not supported. How exactly should I disable `web_search` in the configuration file, and what value type must be used?",
"excerpt": "Disable it as a bare top-level key `web_search = \"disabled\"` in the base config or in the profile file the run loads (profile files layer over the base config). The value is a string unit variant: on 0.156.1 only `\"disabled\"` parses (`false`, `on`, `off`, and `enabled` are all rejected with `expected string only in 'web_search'`).",
"fact": "In Codex CLI 0.156.1, web search must be disabled by setting the bare top-level key `web_search = \"disabled\"` as a string, because boolean or other string values like false, on, off, or enabled are rejected.",
"rubric": {
"must_include": [
"web_search = \"disabled\"",
"string"
],
"reference_answer": "Set the bare top-level key `web_search = \"disabled\"` as a string value (booleans like `false` or values like `off` are rejected)."
},
"excerpt_verbatim": true
}
],
"cost": 0.003798
},
"cursor_cli": {
"tasks": [
{
"question": "I am writing a script to automate Cursor CLI commands and want to pick up and continue the most recent session without having to parse and supply a specific chat or session ID. Which command-line flag should I use with `cursor agent`?",
"excerpt": "`--continue` resumes the most recent CLI session without naming an ID.",
"fact": "The `--continue` flag allows resuming the most recent Cursor CLI session without having to provide a session ID.",
"rubric": {
"must_include": [
"`--continue`"
],
"reference_answer": "You should use the `--continue` flag, which resumes the most recent Cursor CLI session without specifying a session ID."
},
"excerpt_verbatim": true
},
{
"question": "I tried running the Cursor agent non-interactively using the model ID `gemini-3.8-flash-high-fast`, but the command failed with a model error. Why was this model ID rejected, and how are effort levels specified for Gemini 3.8 Flash?",
"excerpt": "Gemini 3.8 Flash still has no `-fast` ID. Its effort levels are `-low` / `-medium` / `-high`. Requesting `gemini-3.8-flash-high-fast` fails with the available-model list.",
"fact": "Gemini 3.8 Flash does not support a `-fast` suffix; its variants are selected solely using `-low`, `-medium`, or `-high` effort suffixes.",
"rubric": {
"must_include": [
"Gemini 3.8 Flash does not have a `-fast` variant or ID",
"effort levels must be specified using `-low`, `-medium`, or `-high` (such as `gemini-3.8-flash-high`)"
],
"reference_answer": "Gemini 3.8 Flash does not support a `-fast` ID, and its effort levels are instead designated solely with `-low`, `-medium`, or `-high` (e.g., `gemini-3.8-flash-high`)."
},
"excerpt_verbatim": true
}
],
"cost": 0.010009500000000001
},
"grok_cli": {
"tasks": [
{
"question": "I am trying to authenticate the Grok CLI on a remote headless Linux server over SSH where no browser is available. What command-line flag should I pass to `grok login` to complete authentication?",
"excerpt": "For headless/SSH systems, use `grok login --device-auth` (alias `--device-code`). Do not combine with `--oauth` (they are mutually exclusive). `--oauth` requires a local browser capable of reaching `127.0.0.1`.",
"fact": "On headless or SSH systems, Grok CLI authentication requires the `--device-auth` (or `--device-code`) flag instead of `--oauth`.",
"rubric": {
"must_include": [
"--device-auth"
],
"reference_answer": "You should use `grok login --device-auth` (or its alias `--device-code`) to authenticate on headless or SSH systems."
},
"excerpt_verbatim": true
},
{
"question": "I logged into the Grok CLI with my subscription account, but my runs are still consuming API credits instead of using my subscription. Why is this happening, and how can I force the CLI to use my subscription?",
"excerpt": "If `XAI_API_KEY` is present in the environment, version 1.0.4 prioritizes the API key over stored subscription credentials in `~/.grok/auth.json`. Explicitly unset `XAI_API_KEY` to enforce subscription usage.",
"fact": "The Grok CLI prioritizes the `XAI_API_KEY` environment variable over subscription credentials, requiring it to be unset to enforce subscription usage.",
"rubric": {
"must_include": [
"XAI_API_KEY",
"unset"
],
"reference_answer": "The CLI prioritizes `XAI_API_KEY` in your environment over stored subscription credentials, so you must explicitly unset `XAI_API_KEY` to use your subscription."
},
"excerpt_verbatim": true
}
],
"cost": 0.009462
},
"opencode_cli": {
"tasks": [
{
"question": "I am configuring basic authentication for an OpenCode server by setting the OPENCODE_SERVER_PASSWORD environment variable. What default username will the server expect if none is explicitly specified?",
"excerpt": "Set environment variable `OPENCODE_SERVER_PASSWORD` to enable basic authentication (username defaults to `opencode`).",
"fact": "When basic authentication is enabled via OPENCODE_SERVER_PASSWORD, the default username is \"opencode\".",
"rubric": {
"must_include": [
"opencode"
],
"reference_answer": "The server defaults to using \"opencode\" as the username."
},
"excerpt_verbatim": true
},
{
"question": "Which command-line flag should I pass to the OpenCode CLI if I want to adjust the provider reasoning effort setting (such as high, max, or minimal)?",
"excerpt": "| `--variant` | Provider reasoning effort setting (`high`, `max`, `minimal`, etc.). |",
"fact": "The OpenCode CLI uses the `--variant` flag to configure provider reasoning effort levels.",
"rubric": {
"must_include": [
"--variant"
],
"reference_answer": "Use the `--variant` flag to specify provider reasoning effort settings."
},
"excerpt_verbatim": true
}
],
"cost": 0.007771499999999999
},
"skill_ai_agent_cli": {
"tasks": [
{
"question": "What is the non-interactive command invocation for running an Antigravity agent in an automated pipeline, and what command must specifically not be used?",
"excerpt": "Subcommands like `agy run`, `claude exec`, or `grok print` do not exist and are prohibited.",
"fact": "The CLI command for Antigravity is `agy --print`, and invented subcommands like `agy run` do not exist and are strictly prohibited.",
"rubric": {
"must_include": [
"agy --print",
"agy run"
],
"reference_answer": "Use `agy --print`; do not use `agy run`, as that subcommand does not exist."
},
"excerpt_verbatim": true
},
{
"question": "Before passing raw prompt strings to an external CLI agent subprocess in a pipeline wrapper, what specific sanitization step must be performed on the text?",
"excerpt": "Clean NULs out of prompt strings (`replace('\\0', '')`) before handing them to a subprocess.",
"fact": "NUL characters must be stripped from prompt strings using `replace('\\0', '')` before being passed to a subprocess.",
"rubric": {
"must_include": [
"NUL",
"replace('\\0', '')"
],
"reference_answer": "Clean NUL characters out of prompt strings by using `replace('\\0', '')` before passing them to a subprocess."
},
"excerpt_verbatim": true
}
],
"cost": 0.0024660000000000003
},
"ai_session_profanity_rate": {
"tasks": [
{
"question": "When preparing runs for profanity rate analysis on AI sessions, where should I point the `--output` parameter? What problem happens if I just let it use the default directory?",
"excerpt": "Always pass `--output` pointing under `tmp/` inside the workspace (e.g. `tmp/profanity_runs/-d`); the default `--home/runs/` location is outside the workspace tree and writing there triggers a user-approval prompt that blocks progress.",
"fact": "The `--output` flag must point to `tmp/` within the workspace because the default directory is outside the workspace tree and triggers a blocking user-approval prompt.",
"rubric": {
"must_include": [
"under `tmp/` inside the workspace",
"triggers a user-approval prompt that blocks progress"
],
"reference_answer": "You must set `--output` to a path under `tmp/` inside the workspace, because the default `--home/runs/` directory sits outside the workspace tree and triggers a user-approval prompt that halts progress."
},
"excerpt_verbatim": true
},
{
"question": "Which specific underlying model and configuration variant is the default `reasoning_gpt` batch classification sub-agent bound to?",
"excerpt": "It is explicitly bound to `openai/gpt-6-astra` with the low-effort variant; do not substitute the separate `glm` agent, which may use a different provider.",
"fact": "The default `reasoning_gpt` sub-agent is explicitly configured to use `openai/gpt-6-astra` with the low-effort variant.",
"rubric": {
"must_include": [
"openai/gpt-6-astra",
"low-effort"
],
"reference_answer": "The `reasoning_gpt` sub-agent is explicitly bound to `openai/gpt-6-astra` using the low-effort variant."
},
"excerpt_verbatim": true
}
],
"cost": 0.009570750000000001
},
"skill_ai_usage_dashboard": {
"tasks": [
{
"question": "When querying Grok's billing endpoint for weekly quota data, why might the credit_usage_percent field be missing completely from the response, and how should this case be handled?",
"excerpt": "Proto3 omits default float `0.0`, so a 0% week has no `credit_usage_percent` field; the parser treats that as 0% so the bar still renders.",
"fact": "Because Proto3 omits default float 0.0 values, an unused week omits the credit_usage_percent field, which must be parsed as 0%.",
"rubric": {
"must_include": [
"Proto3 omits default float values (or 0.0)",
"treated or parsed as 0%"
],
"reference_answer": "In Proto3, default float values of 0.0 are omitted from responses, so an absent credit_usage_percent represents a 0% week and should be parsed as 0% so the bar still renders."
},
"excerpt_verbatim": true
},
{
"question": "What is the minimum percentage-point movement required by default when using plan_usage to estimate plan capacity?",
"excerpt": "Use `--window` for a window kind or exact label, `--history-db` for a\ndatabase override, and `--min-change` for the minimum percentage-point movement\n(default 3).",
"fact": "The default minimum percentage-point movement for plan_usage is 3.",
"rubric": {
"must_include": [
"3"
],
"reference_answer": "The default minimum percentage-point movement is 3."
},
"excerpt_verbatim": true
}
],
"cost": 0.005166
},
"skill_antigravity_push": {
"tasks": [
{
"question": "I want to verify my Antigravity push settings and check how many entries are queued on my satellite machine without actually transmitting any data to the host. How can I do this?",
"excerpt": "To verify the configuration without POSTing:\n\n```bash\n.venv/bin/python scripts/push-antigravity --dry-run\n```",
"fact": "Passing the `--dry-run` flag to `.venv/bin/python scripts/push-antigravity` checks the configuration and entry count without sending data.",
"rubric": {
"must_include": [
"--dry-run",
"scripts/push-antigravity"
],
"reference_answer": "You can verify the configuration without sending data by running `.venv/bin/python scripts/push-antigravity --dry-run`."
},
"excerpt_verbatim": true
},
{
"question": "How does the dashboard host prevent duplicate usage records if the Antigravity push script is run multiple times from a satellite machine?",
"excerpt": "The host deduplicates by `response_id`, so pushing the same entries multiple times is safe and idempotent.",
"fact": "The dashboard host deduplicates pushed Antigravity usage entries using their `response_id`.",
"rubric": {
"must_include": [
"response_id"
],
"reference_answer": "The host deduplicates incoming entries by their response_id, making repeated pushes idempotent."
},
"excerpt_verbatim": true
}
],
"cost": 0.00226575
},
"apple_photos": {
"tasks": [
{
"question": "What is the maximum number of image pairs allowed in a single delete manifest when planning deletions with the Apple Photos tool?",
"excerpt": "Limit each delete manifest to 50 pairs. Split larger evidence sets into independent plans and require fresh review and authorization for each; never reuse a token or retry an uncertain batch.",
"fact": "Each delete manifest is limited to a maximum of 50 pairs.",
"rubric": {
"must_include": [
"50 pairs",
"50"
],
"reference_answer": "Each delete manifest is limited to a maximum of 50 pairs."
},
"excerpt_verbatim": true
},
{
"question": "What specific statistical threshold ceilings must be met for RGB MAE, luma MAE, and RGB P99 for a pair of still images to be eligible for deletion?",
"excerpt": "Require equal dimensions, RGB MAE <= 1%, luma MAE <= 1%, and RGB P99 <= 5%. Policies may be stricter, never weaker. Byte-identical duplicate deletion is unsupported.",
"fact": "Deletion eligibility requires RGB MAE <= 1%, luma MAE <= 1%, and RGB P99 <= 5%.",
"rubric": {
"must_include": [
"RGB MAE <= 1%",
"luma MAE <= 1%",
"RGB P99 <= 5%"
],
"reference_answer": "Deletion eligibility requires RGB MAE <= 1%, luma MAE <= 1%, and RGB P99 <= 5% (along with equal dimensions)."
},
"excerpt_verbatim": true
}
],
"cost": 0.003249
},
"asr": {
"tasks": [
{
"question": "I'm planning to process several long interview recordings through the local speech transcription pipeline. What is the maximum duration allowed for a single audio file before the system rejects it?",
"excerpt": "A single file longer than 7200 seconds is rejected before a model loads.",
"fact": "Any single audio file longer than 7200 seconds is rejected before a model is loaded.",
"rubric": {
"must_include": [
"7200 seconds (or 2 hours)"
],
"reference_answer": "A single audio file is rejected before any model loads if its duration exceeds 7200 seconds (2 hours)."
},
"excerpt_verbatim": true
},
{
"question": "When editing and validating transcript segments, how much time tolerance is permitted for row start and end timestamps to fall outside the original source interval boundaries?",
"excerpt": "Row start/end must be finite with start <= end, inside the envelope of known source intervals within 0.05 seconds.",
"fact": "Row start and end timestamps must fall within the envelope of known source intervals within a 0.05-second tolerance.",
"rubric": {
"must_include": [
"0.05 seconds"
],
"reference_answer": "Row start and end times must fall within the envelope of known source intervals within a 0.05-second tolerance."
},
"excerpt_verbatim": true
}
],
"cost": 0.011082000000000002
},
"skill_agentic_patrol": {
"tasks": [
{
"question": "When configuring a delayed command with Process Launcher via `POST /run`, what initial values are returned in the response for the process ID (`pid`) and the output file (`output_file`)?",
"excerpt": "Delayed runs initially return an immediate response with `pid: 0` and `output_file: null`.",
"fact": "Delayed runs on Process Launcher initially return an immediate response containing `pid: 0` and `output_file: null`.",
"rubric": {
"must_include": [
"0 for pid",
"null for output_file"
],
"reference_answer": "Delayed runs initially return an immediate response with pid set to 0 and output_file set to null."
},
"excerpt_verbatim": true
},
{
"question": "When checking Process Launcher's `GET /scheduled` endpoint to see if a patrol job is already pending, how is the `label` query parameter evaluated against existing jobs?",
"excerpt": "The `label` is a substring filter, not a uniqueness constraint; verify the command and target time before assuming a match.",
"fact": "Process Launcher evaluates the `label` parameter in `GET /scheduled` as a substring filter rather than a uniqueness constraint.",
"rubric": {
"must_include": [
"substring filter (or substring match)",
"not a uniqueness constraint (or not an exact match)"
],
"reference_answer": "The label parameter acts as a substring filter rather than a uniqueness constraint or exact match."
},
"excerpt_verbatim": true
}
],
"cost": 0.01418475
},
"skill_process_launcher": {
"tasks": [
{
"question": "What happens when I try to schedule a job via POST /run with a run_at timestamp that exactly matches an existing pending or running job?",
"excerpt": "Exact-time collision guard: when a scheduled `POST /run` (via `delay_seconds` or `run_at`) resolves to a `run_at` exactly equal to an existing pending or running job, the request returns HTTP 409 with `detail.error = \"schedule_conflict\"`, a message naming each conflicting job (id, label, command, status, run_at), and the full conflicting jobs in `detail.conflicts`. Nothing is created on 409.",
"fact": "Scheduling a job that matches the exact run_at timestamp of an existing pending or running job returns HTTP 409 with schedule_conflict details, creating nothing.",
"rubric": {
"must_include": [
"HTTP 409",
"schedule_conflict",
"nothing is created"
],
"reference_answer": "The request returns HTTP 409 with detail.error = \"schedule_conflict\" detailing the conflicting jobs, and nothing is created."
},
"excerpt_verbatim": true
},
{
"question": "What specific sections of config/launcher.yaml does the POST /periodic/reload endpoint update in the running process launcher?",
"excerpt": "`POST /periodic/reload` hot-reloads only the `periodic_jobs:` section of `config/launcher.yaml` into the running launcher; it does not touch declared services, logging, storage, or server settings, and a run already in flight keeps the configuration it started with.",
"fact": "POST /periodic/reload only hot-reloads the periodic_jobs: section of config/launcher.yaml, leaving other settings untouched.",
"rubric": {
"must_include": [
"periodic_jobs"
],
"reference_answer": "It hot-reloads only the periodic_jobs: section of config/launcher.yaml and does not touch declared services, logging, storage, or server settings."
},
"excerpt_verbatim": true
}
],
"cost": 0.0052605
},
"skill_circle_post": {
"tasks": [
{
"question": "I am preparing to remove an outdated post from our Circle community using the CLI tool. After verifying everything with a dry run, which command-line flag must be passed to the delete command to execute the live deletion?",
"excerpt": ".venv/bin/python circle_post.py delete --post-id YOUR_POST_ID --dry-run\n.venv/bin/python circle_post.py delete --post-id YOUR_POST_ID --confirm",
"fact": "Live post deletion requires passing the `--confirm` flag to the delete command.",
"rubric": {
"must_include": [
"--confirm"
],
"reference_answer": "You must supply the `--confirm` flag to execute the actual deletion."
},
"excerpt_verbatim": true
},
{
"question": "I am setting up our local environment file to allow publishing to Circle via the API. What exact environment variable name needs to be set for the API authentication token?",
"excerpt": "- `.env` configured with `CIRCLE_V2_TOKEN` for real Circle API calls",
"fact": "The environment variable required for real Circle API calls is CIRCLE_V2_TOKEN.",
"rubric": {
"must_include": [
"CIRCLE_V2_TOKEN"
],
"reference_answer": "The environment variable name must be CIRCLE_V2_TOKEN."
},
"excerpt_verbatim": true
}
],
"cost": 0.0092145
},
"ct_education": {
"tasks": [
{
"question": "In my CT education workspace, how does the pipeline encode anatomical structures inside labels.npy, and which specific numeric values are assigned to each structure?",
"excerpt": "| `labels.npy` | Reduced-grid uint8 bitfield, not mutually exclusive class IDs. Bit values: lungs 1, airways 2, vessels 4, bones 8. |",
"fact": "In labels.npy, anatomical structures are stored as a uint8 bitfield with values: lungs 1, airways 2, vessels 4, and bones 8.",
"rubric": {
"must_include": [
"uint8 bitfield",
"lungs 1, airways 2, vessels 4, bones 8"
],
"reference_answer": "The labels.npy file encodes structures as a uint8 bitfield where lungs are 1, airways are 2, vessels are 4, and bones are 8."
},
"excerpt_verbatim": true
},
{
"question": "When displaying CT slice rows in the 2D canvas viewer with a superior-up orientation, what specific mathematical condition triggers row reversal?",
"excerpt": "Display rows reverse when `affine_ras[2][2] > 0`; native PNG arrays and 3D texture UVs remain unchanged. This is not anatomical oblique resampling or full grid de-shearing.",
"fact": "Display rows in the 2D viewer canvas reverse when affine_ras[2][2] > 0.",
"rubric": {
"must_include": [
"affine_ras[2][2] > 0"
],
"reference_answer": "Display rows are reversed when the RAS affine matrix element affine_ras[2][2] is greater than zero."
},
"excerpt_verbatim": true
}
],
"cost": 0.01803675
},
"dataforseo": {
"tasks": [
{
"question": "When running keyword commands with the DataForSEO CLI, what flag should I pass if I need to output the complete API response instead of the default compact JSON rows?",
"excerpt": "Keyword commands return compact JSON rows by default. Use `--raw` to keep the full DataForSEO response.",
"fact": "Passing `--raw` causes DataForSEO keyword commands to output the full response rather than compact JSON rows.",
"rubric": {
"must_include": [
"`--raw`"
],
"reference_answer": "You should pass the `--raw` flag to keep the full DataForSEO response."
},
"excerpt_verbatim": true
},
{
"question": "I am able to check my account balance successfully with the DataForSEO CLI, but keyword queries fail with error code 40104 even against the sandbox. What causes this issue and how do I fix it?",
"excerpt": "DataForSEO may return `40104` when the account is authenticated but not verified. In that case, account/balance can still work, but keyword data endpoints, including sandbox, may be blocked until the user completes verification in the DataForSEO panel.",
"fact": "Error 40104 occurs when the DataForSEO account is authenticated but unverified, which blocks keyword endpoints until verified in the panel.",
"rubric": {
"must_include": [
"account is authenticated but not verified (or unverified)",
"complete verification in the DataForSEO panel"
],
"reference_answer": "Error 40104 indicates that your account is authenticated but not verified; you must complete verification in the DataForSEO panel to unblock keyword endpoints."
},
"excerpt_verbatim": true
}
],
"cost": 0.0024389999999999998
},
"design_critique": {
"tasks": [
{
"question": "When performing accessibility testing during a screen review, what proportion of total accessibility issues can an automated scan be expected to detect?",
"excerpt": "automated scan first (it catches roughly 30% of issues), then keyboard-only navigation, screen-reader testing (VoiceOver, NVDA), contrast verification, and zoom to 200% to confirm the layout does not break.",
"fact": "An automated scan is expected to catch roughly 30% of accessibility issues.",
"rubric": {
"must_include": [
"30%"
],
"reference_answer": "An automated accessibility scan catches roughly 30% of issues."
},
"excerpt_verbatim": true
},
{
"question": "When reviewing a UI screen's usability, what is the rule regarding the number of primary actions allowed on the screen, and what should be done with any competing actions?",
"excerpt": "Every screen should have exactly one primary action; competing actions should be visually demoted.",
"fact": "Each screen must have exactly one primary action, while competing actions must be visually demoted.",
"rubric": {
"must_include": [
"exactly one primary action",
"visually demote competing actions"
],
"reference_answer": "Every screen should have exactly one primary action, and any competing actions must be visually demoted."
},
"excerpt_verbatim": true
}
],
"cost": 0.01025625
},
"design_skill": {
"tasks": [
{
"question": "We are establishing copy guidelines for our international product to prevent UI truncation issues. How much text expansion should our English interface strings leave room for during translation?",
"excerpt": "English strings should stay short enough to allow 30% expansion in translation.",
"fact": "English interface copy must leave enough room to accommodate a 30% expansion when translated into other languages.",
"rubric": {
"must_include": [
"30%"
],
"reference_answer": "English strings should remain short enough to allow for a 30% expansion in translation."
},
"excerpt_verbatim": true
},
{
"question": "When assessing cognitive burden on a newly designed screen, how many rules or assumptions can a first-time user be required to infer before we must introduce onboarding copy, progressive disclosure, or reduce visible controls?",
"excerpt": "If a user must guess three or more rules before taking the primary action, the flow needs onboarding copy, stronger state labels, progressive disclosure, or fewer visible controls.",
"fact": "A flow must introduce onboarding copy, stronger state labels, progressive disclosure, or fewer visible controls if a user must guess three or more rules before taking the primary action.",
"rubric": {
"must_include": [
"three or more"
],
"reference_answer": "If a user must guess three or more rules before completing the primary action, the flow requires onboarding copy, stronger labels, progressive disclosure, or fewer visible controls."
},
"excerpt_verbatim": true
}
],
"cost": 0.011519249999999998
},
"design_system": {
"tasks": [
{
"question": "Our team wants to introduce a new shared component into our design system because they noticed similar-looking elements in upcoming designs. Before approving a new shared primitive, what two conditions must be confirmed, and what should be recommended if either is not met?",
"excerpt": "Before recommending a new shared component or token, verify both conditions:\n\n- **Reuse demand**: recurrence is observed across real product surfaces, not forecast from visual similarity\n- **Governance budget**: the team can document, version, test, migrate, support, and eventually deprecate the primitive\n\nIf either condition is missing, recommend a tracked one-off or snowflake path first.",
"fact": "A shared primitive requires verified reuse demand across real surfaces and an adequate governance budget, otherwise a tracked one-off or snowflake path must be recommended.",
"rubric": {
"must_include": [
"reuse demand based on observed recurrence across real product surfaces rather than visual similarity forecast",
"governance budget to document, version, test, migrate, support, and deprecate",
"recommend a tracked one-off or snowflake path"
],
"reference_answer": "You must verify reuse demand (observed recurrence across real surfaces, not forecasted from visual similarity) and governance budget (capacity to document, version, test, migrate, support, and deprecate); if either is missing, recommend a tracked one-off or snowflake path first."
},
"excerpt_verbatim": true
},
{
"question": "We are auditing our mobile app's iOS codebase to ensure it adheres to our design tokens. What specific implementation pattern should we look for to detect hardcoded color values?",
"excerpt": "For iOS, check for hardcoded Color literals instead of asset catalog references.",
"fact": "On iOS, hardcoded colors are detected by checking for Color literals rather than asset catalog references.",
"rubric": {
"must_include": [
"Color literals",
"asset catalog"
],
"reference_answer": "Check for hardcoded Color literals instead of asset catalog references."
},
"excerpt_verbatim": true
}
],
"cost": 0.00925275
},
"frontend_design": {
"tasks": [
{
"question": "Before writing any code for a new frontend direction, we need to establish the initial token system. How many named hex colors should this system define?",
"excerpt": "Capture the direction as a small token system before writing code: 4–6 named hex colors, the typefaces and their roles, a layout concept (one sentence plus a rough ASCII wireframe), and the principles that make this page specific.",
"fact": "The initial token system created before writing code should define 4 to 6 named hex colors.",
"rubric": {
"must_include": [
"4–6"
],
"reference_answer": "The initial token system should define 4 to 6 named hex colors."
},
"excerpt_verbatim": true
},
{
"question": "If an interface we are implementing fails to satisfy all the required production gate verification checks prior to handoff, what should the resulting artifact be labeled instead of production-grade frontend?",
"excerpt": "If these gates are not met, label the result as a prototype or directional draft, not production-grade frontend.",
"fact": "If the frontend production gates are not met, the result must be labeled as a prototype or directional draft.",
"rubric": {
"must_include": [
"prototype",
"directional draft"
],
"reference_answer": "The output must be labeled as a prototype or directional draft."
},
"excerpt_verbatim": true
}
],
"cost": 0.01053525
},
"dinov3_classifier_skill": {
"tasks": [
{
"question": "I am preparing the model disagreement cohort for a deployment review using `visualize-models`. What value must I supply to the `--count` parameter, and what does setting it achieve?",
"excerpt": "Setting `--count 0` is mandatory for deployment review. It records exact cohort identity and full disagreement counts in `workbench_private/model_cases.jsonl`.",
"fact": "For deployment review, setting `--count 0` is required to record the exact cohort identity and full disagreement counts.",
"rubric": {
"must_include": [
"0 (or --count 0)",
"records the exact cohort identity and full disagreement counts (or outputs to workbench_private/model_cases.jsonl)"
],
"reference_answer": "You must set `--count 0`, which is mandatory for deployment review to record the exact cohort identity and full disagreement counts in `workbench_private/model_cases.jsonl`."
},
"excerpt_verbatim": true
},
{
"question": "I decided to skip proposal model generation and want to build contact sheets for a blind visual audit directly. What should I provide for the `--labels` and `--expected` arguments when no proposal file exists?",
"excerpt": "If no proposal file exists, pass the candidate manifest as both `--labels` and `--expected`. In blind mode, missing labels are rendered as `?` and carry no evidentiary weight.",
"fact": "If no label proposal file exists, the candidate manifest must be passed to both `--labels` and `--expected` when generating contact sheets.",
"rubric": {
"must_include": [
"pass the candidate manifest as both --labels and --expected"
],
"reference_answer": "You should pass the candidate manifest file as both the `--labels` and `--expected` arguments."
},
"excerpt_verbatim": true
}
],
"cost": 0.01277925
},
"skill_firecrawl": {
"tasks": [
{
"question": "I want to perform a search that focuses only on certain sites while simultaneously excluding another specific site. How can I set this up using domain filtering in the CLI?",
"excerpt": "Include and exclude domain filters cannot be combined (Firecrawl constraint). To both target and exclude, use the `-site:` operator inside the query text.",
"fact": "Include and exclude domain flags cannot be used together; to combine targeting and exclusion, you must use the `-site:` operator directly within the search query text.",
"rubric": {
"must_include": [
"cannot combine --include-domain and --exclude-domain",
"-site: operator inside the query text"
],
"reference_answer": "You cannot combine `--include-domain` and `--exclude-domain` flags due to an API constraint; instead, specify the domain to exclude by adding the `-site:` operator directly into the query text."
},
"excerpt_verbatim": true
},
{
"question": "What happens if I pass the `--stdout` flag when running a Firecrawl `search` command to print the results directly to the terminal?",
"excerpt": "`--stdout` is rejected on `search` and `extract` (exit 2, before any request, no credits spent). `usage` still accepts it and prints the complete usage payload",
"fact": "`--stdout` is rejected on `search` commands and exits immediately with code 2 before making any request or spending credits.",
"rubric": {
"must_include": [
"rejected",
"exit code 2 (or exit 2)",
"no credits spent (or before any request)"
],
"reference_answer": "The CLI will reject the flag and exit with code 2 before sending any request, spending zero credits."
},
"excerpt_verbatim": true
}
],
"cost": 0.00693975
},
"firewalla": {
"tasks": [
{
"question": "I am trying to run device and alarm collection commands using the Firewalla CLI, but they are not actually querying my physical Firewalla device. What flag do I need to specify, and why is it needed?",
"excerpt": "Live collection is dry-run by default; add `--execute` only when you intend to query the user's Firewalla.",
"fact": "Live collection runs in dry-run mode by default and requires the `--execute` flag to actually query the Firewalla.",
"rubric": {
"must_include": [
"--execute",
"dry-run"
],
"reference_answer": "Live collection operates in dry-run mode by default, so you must pass the `--execute` flag to actually query the Firewalla."
},
"excerpt_verbatim": true
},
{
"question": "When analyzing Firewalla device attribution data, how should device names be reported if an `identity_conflict` flag is encountered?",
"excerpt": "If `identity_conflict` is present, report both the current name and stale aliases instead of silently choosing the alias.",
"fact": "When an identity_conflict occurs, both the current name and stale aliases must be reported rather than selecting an alias.",
"rubric": {
"must_include": [
"both",
"current name",
"aliases"
],
"reference_answer": "You must report both the current name and stale aliases rather than silently selecting an alias."
},
"excerpt_verbatim": true
}
],
"cost": 0.008328
},
"skill_genai_portrait": {
"tasks": [
{
"question": "When using the portrait editing tool for a multi-image transfer to apply lighting from a reference photo onto a subject, what explicit instruction must be included in the prompt regarding the first input image?",
"excerpt": "The first image must be declared in the prompt as the sole identity and geometry authority. Reference images may influence only the explicitly requested property.",
"fact": "In multi-image transfers, the prompt must explicitly declare the first image as the sole identity and geometry authority.",
"rubric": {
"must_include": [
"sole identity and geometry authority"
],
"reference_answer": "The prompt must state that the first image is the sole identity and geometry authority."
},
"excerpt_verbatim": true
},
{
"question": "When executing a complex portrait background adaptation or relighting workflow that requires a transparent output, at what stage should the mask or alpha channel be applied?",
"excerpt": "Apply a reviewed mask or alpha channel only after the RGB edit is accepted.",
"fact": "A reviewed mask or alpha channel should only be applied after the RGB edit has been accepted.",
"rubric": {
"must_include": [
"after the RGB edit is accepted"
],
"reference_answer": "A reviewed mask or alpha channel should only be applied after the RGB edit is accepted."
},
"excerpt_verbatim": true
}
],
"cost": 0.01278525
},
"godaddy": {
"tasks": [
{
"question": "I need to provision a new SOA record for one of our registered domains using our GoDaddy DNS management CLI. How can I configure the CLI to create this record?",
"excerpt": "Create is the only mutation. Supported types are `A`, `AAAA`, `CAA`, `CNAME`, `MX`, `NS`, `SRV`, and `TXT`; `SOA` is deliberately excluded.",
"fact": "SOA records cannot be created because SOA is deliberately excluded from the CLI's supported record types.",
"rubric": {
"must_include": [
"SOA is excluded",
"cannot be created"
],
"reference_answer": "You cannot create an SOA record because the tool deliberately excludes SOA from its supported DNS record creation types."
},
"excerpt_verbatim": true
},
{
"question": "We want to simplify our CI/CD runner environment by using a single token instead of maintaining separate read and write personal access tokens for GoDaddy domain changes. What environment variable will the execution step fall back to if the dedicated write PAT is missing, and what scopes are required on it?",
"excerpt": "Apply credentials prefer a separate `GODADDY_WRITE_PAT` with `domains.domain:read` and `domains.dns:update`. When `GODADDY_WRITE_PAT` is absent, `--execute` falls back to `GODADDY_PAT`, so a single token with both read and update scopes is sufficient.",
"fact": "When GODADDY_WRITE_PAT is absent, execution falls back to GODADDY_PAT, which must have both read and update scopes.",
"rubric": {
"must_include": [
"GODADDY_PAT",
"domains.domain:read",
"domains.dns:update"
],
"reference_answer": "If GODADDY_WRITE_PAT is absent, execution falls back to GODADDY_PAT, which must have both domains.domain:read and domains.dns:update scopes."
},
"excerpt_verbatim": true
}
],
"cost": 0.012086999999999999
},
"skill_google_maps_routing": {
"tasks": [
{
"question": "When invoking the CLI's route command to calculate driving times between two addresses, can I omit both timing flags or provide both of them at once? What is the rule for specifying departure and arrival times?",
"excerpt": "Calls Google Routes API using address-string waypoints. Exactly one of `--departure-time` or `--arrival-time` is required.",
"fact": "The route command requires exactly one of either `--departure-time` or `--arrival-time` to be provided.",
"rubric": {
"must_include": [
"exactly one",
"--departure-time or --arrival-time"
],
"reference_answer": "You cannot omit both or supply both; exactly one of `--departure-time` or `--arrival-time` must be provided."
},
"excerpt_verbatim": true
},
{
"question": "I need to run the live integration test suite against the real Google Maps endpoints instead of the default offline tests. What environment variable must be set to opt in and execute them?",
"excerpt": "Default tests are offline. Live tests require explicit opt-in:\n\n```bash\nRUN_LIVE_E2E=1 .venv/bin/python -m pytest -v -m live_integration\n```",
"fact": "Live integration tests require the `RUN_LIVE_E2E=1` environment variable to be set.",
"rubric": {
"must_include": [
"RUN_LIVE_E2E=1"
],
"reference_answer": "You must set the environment variable `RUN_LIVE_E2E=1` to run the live integration tests."
},
"excerpt_verbatim": true
}
],
"cost": 0.00740475
},
"blender_animation": {
"tasks": [
{
"question": "What exact FFmpeg command should be run to verify that an encoded deliverable named output.mp4 decodes cleanly without bitstream errors?",
"excerpt": "- **Stream Decode Verification**: Video encoded with FFmpeg decodes cleanly without bitstream errors:\n ```bash\n ffmpeg -v error -i output.mp4 -f null -\n ```",
"fact": "FFmpeg decodes video cleanly to the null muxer using `ffmpeg -v error -i output.mp4 -f null -` to verify there are no bitstream errors.",
"rubric": {
"must_include": [
"ffmpeg -v error -i output.mp4 -f null -"
],
"reference_answer": "Run `ffmpeg -v error -i output.mp4 -f null -`."
},
"excerpt_verbatim": true
},
{
"question": "When choreographing exploded-view breakdown animations for architectural models, what specific vector constraints should be applied to prevent foundation geometry from clipping beneath opaque ground planes?",
"excerpt": "- **Negative Explosion Subterranean Clipping**:\n - *Symptom*: Applying negative Z offsets during exploded-view animations buries foundation stones beneath opaque ground planes.\n - *Fix*: Constrain explosion vectors to positive vertical displacements and outward radial vectors.",
"fact": "To prevent subterranean clipping beneath opaque ground planes during exploded-view animations, explosion vectors must be constrained to positive vertical displacements and outward radial vectors.",
"rubric": {
"must_include": [
"positive vertical displacements",
"outward radial vectors"
],
"reference_answer": "Constrain explosion vectors to positive vertical displacements and outward radial vectors."
},
"excerpt_verbatim": true
}
],
"cost": 0.0112875
},
"blender_compositing_plate": {
"tasks": [
{
"question": "In our Blender compositing plate setup, we want regions where our lighting formula evaluates to zero to decode as pure black, but dimming the lamp intensity or base color still leaves specular reflections and indirect light behind. How should we set up the shader nodes in Blender to ensure that zero amplitude becomes exactly black?",
"excerpt": "Implement it in Blender as a world-position shader multiplier: compute the mask from Geometry Position and use it as the factor of a Mix Shader between black emission and the full BSDF. Multiplying the entire closure, reflections included, makes zero amplitude exactly black; attenuating only lamp energy or base colour leaves specular and indirect light behind.",
"fact": "To make zero-amplitude lighting decode as exactly black, use a position-derived mask as the factor in a Mix Shader between black emission and the full BSDF.",
"rubric": {
"must_include": [
"Mix Shader",
"black emission",
"Geometry Position"
],
"reference_answer": "Compute a mask from Geometry Position and use it as the factor of a Mix Shader between black emission and the full BSDF to multiply the entire closure."
},
"excerpt_verbatim": true
},
{
"question": "When configuring an orthographic camera in Blender to match a procedural rendering layer for a portrait aspect ratio, how should the camera's sensor fit and orthographic scale be set in terms of horizontal world extent width, frame height H, and frame width W?",
"excerpt": "For a portrait frame, set `sensor_fit` to `VERTICAL` and `ortho_scale = width * H / W`, where `width` is the horizontal world extent.",
"fact": "For portrait orthographic camera matching, set sensor_fit to VERTICAL and ortho_scale to width * H / W.",
"rubric": {
"must_include": [
"sensor_fit to VERTICAL",
"ortho_scale = width * H / W"
],
"reference_answer": "Set sensor_fit to VERTICAL and set ortho_scale to width * H / W, where width is the horizontal world extent."
},
"excerpt_verbatim": true
}
],
"cost": 0.010554000000000001
},
"blender_modeling": {
"tasks": [
{
"question": "I am writing a headless Python script to bundle my generated scene, but calling `bpy.ops.wm.pack_all()` results in an AttributeError saying the operator does not exist. Which operator should I call instead to pack all external textures and resources into the `.blend` file?",
"excerpt": "Calling `bpy.ops.wm.pack_all()` throws an `AttributeError` stating the operator does not exist.\n - *Fix*: Use `bpy.ops.file.pack_all()` to pack external textures and dependencies into the `.blend` file.",
"fact": "The operator to pack external textures and dependencies into a .blend file is bpy.ops.file.pack_all(), not bpy.ops.wm.pack_all().",
"rubric": {
"must_include": [
"bpy.ops.file.pack_all"
],
"reference_answer": "Use `bpy.ops.file.pack_all()` instead of `bpy.ops.wm.pack_all()` to pack external resources into the `.blend` file."
},
"excerpt_verbatim": true
},
{
"question": "When calculating the orientation quaternion for a Blender camera pointing toward a target vector using the `to_track_quat()` method, what specific track and up axis arguments should be provided?",
"excerpt": "Aim Blender cameras with `(target - camera.location).to_track_quat('-Z', 'Y')`; check the evaluated geometry at each view, then inspect the saved render.",
"fact": "Blender cameras should be oriented toward a target by calling to_track_quat('-Z', 'Y').",
"rubric": {
"must_include": [
"'-Z'",
"'Y'"
],
"reference_answer": "Provide `'-Z'` for the track axis and `'Y'` for the up axis, calling `to_track_quat('-Z', 'Y')`."
},
"excerpt_verbatim": true
}
],
"cost": 0.011379750000000001
},
"character_rigging_mocap": {
"tasks": [
{
"question": "In our browser-based character capture setup using a Web Worker for pose detection, how many video frames should we queue or have in flight simultaneously across acquisition, transfer, inference, and rendering?",
"excerpt": "Keep at most one frame in flight across acquisition, transfer, inference and consumption. Prefer fresh work over a queue of old frames, and release transferables on success, error and cancellation.",
"fact": "The capture and processing pipeline must maintain at most one frame in flight across acquisition, transfer, inference, and consumption.",
"rubric": {
"must_include": [
"at most one frame in flight",
"prefer fresh work over queuing old frames"
],
"reference_answer": "Maintain at most one frame in flight across acquisition, transfer, inference, and consumption, preferring fresh frames over queuing old ones."
},
"excerpt_verbatim": true
},
{
"question": "How should our character preview application handle UI controls and camera activation if the character asset fails to load?",
"excerpt": "Tracking failure leaves presets usable only if the character loaded; character failure disables dependent controls and offers model-only retry without starting a camera.",
"fact": "When a character model fails to load, dependent controls must be disabled and only a model retry should be offered without initializing the camera.",
"rubric": {
"must_include": [
"disable dependent controls",
"offer a model-only retry",
"do not start the camera"
],
"reference_answer": "If the character fails to load, disable dependent controls and offer a model-only retry without starting the camera."
},
"excerpt_verbatim": true
}
],
"cost": 0.010584
},
"hybrid_ai_video": {
"tasks": [
{
"question": "When fetching a completed video file from the CDN URL returned by the video generation task, what Bearer token or authorization header should our worker send on the initial download request?",
"excerpt": "Send no Bearer credentials on any CDN download, including the first request.",
"fact": "Bearer credentials must not be sent on any CDN download, including the initial request.",
"rubric": {
"must_include": [
"no Bearer credentials or tokens",
"including the first or initial request"
],
"reference_answer": "Do not send any Bearer credentials on any CDN download request, including the first request."
},
"excerpt_verbatim": true
},
{
"question": "We want to submit a locally rendered starting frame to the video generation API without putting it on a public web server. Can we specify the local file path directly in the request, and how should the image data be provided?",
"excerpt": "`image.url` can carry the approved JPEG as a data URI without public image hosting; a local filesystem path is not a usable URL.",
"fact": "Local filesystem paths are not usable URLs in the request, but the image can be passed directly as a data URI.",
"rubric": {
"must_include": [
"a local filesystem path cannot be used or is not a usable URL",
"as a data URI"
],
"reference_answer": "A local filesystem path is not a usable URL; the image must instead be passed as a data URI in `image.url`."
},
"excerpt_verbatim": true
}
],
"cost": 0.0173085
},
"skill_gpt_3d": {
"tasks": [
{
"question": "We are preparing texture maps for a deforming rigged character asset in our 3D pipeline. Which specific type of texture bake is explicitly considered invalid under arbitrary deformation?",
"excerpt": "Surface-color bakes can remain useful, but pose-dependent self-shadow bakes are not valid under arbitrary deformation.",
"fact": "Pose-dependent self-shadow bakes are not valid under arbitrary deformation on deforming character assets.",
"rubric": {
"must_include": [
"pose-dependent self-shadow"
],
"reference_answer": "Pose-dependent self-shadow bakes are not valid under arbitrary deformation."
},
"excerpt_verbatim": true
},
{
"question": "When preparing our 3D scene for export, we need to run polygon decimation, UV atlas unwrapping, and texture baking. On what target must these destructive operations be performed according to the source model preservation rules?",
"excerpt": "Geometry merging, UV unwrapping for atlases, texture baking, and polygon decimation must be executed on temporary export copies or non-destructive evaluation graphs.",
"fact": "Destructive operations like polygon decimation, UV unwrapping for atlases, and texture baking must be executed on temporary export copies or non-destructive evaluation graphs.",
"rubric": {
"must_include": [
"temporary export copies",
"non-destructive evaluation graphs"
],
"reference_answer": "They must be executed on temporary export copies or non-destructive evaluation graphs to preserve the authoritative master model."
},
"excerpt_verbatim": true
}
],
"cost": 0.01242975
},
"web_walkthrough": {
"tasks": [
{
"question": "When preparing multiple Blender architectural meshes with procedural materials for light baking, we want to preserve their procedural texture coordinates before joining them into a single mesh. Why is capturing raw local vertex positions into a custom attribute insufficient for replicating Blender's Generated texture coordinates?",
"excerpt": "`Generated` coordinates are normalized to the original local bounding box; storing raw vertex positions is not equivalent.",
"fact": "Blender's Generated coordinates are normalized relative to the original local bounding box, making raw vertex positions non-equivalent.",
"rubric": {
"must_include": [
"normalized to the (original local) bounding box",
"raw vertex positions are not equivalent"
],
"reference_answer": "Blender's Generated coordinates are normalized to the object's original local bounding box, meaning raw vertex positions are not equivalent and will distort procedural mappings."
},
"excerpt_verbatim": true
},
{
"question": "We are configuring a Three.js viewer for an architectural walkthrough using the display-referred baking workflow (KHR_materials_unlit). What specific settings must be applied to the Three.js renderer for output color space and tone mapping to maintain visual parity with Blender?",
"excerpt": "Match color management settings: if using Display-Referred baking, set renderer output color space to sRGB with no tone mapping.",
"fact": "When using display-referred baking, the Three.js renderer output color space must be set to sRGB and runtime tone mapping must be disabled.",
"rubric": {
"must_include": [
"output color space set to sRGB",
"tone mapping disabled (or no tone mapping)"
],
"reference_answer": "The Three.js renderer must have its output color space set to sRGB and runtime tone mapping completely disabled."
},
"excerpt_verbatim": true
}
],
"cost": 0.014175
},
"skill_image_generation": {
"tasks": [
{
"question": "What is the final delivery format rule when embedding generated charts or diagrams into external Markdown deliverables such as blog drafts or survey reports? Can a locally rendered matplotlib or Mermaid diagram be converted directly to PNG and embedded?",
"excerpt": "every embedded image in a survey report, blog draft, course material, or share-ready document must be a PNG, JPG, or WebP file produced by a GPT Image model (`gpt-image-2.5-sunburst` / `gpt-image-2.5-flare` / `gpt-image-2`) generation/redraw, or a compressed derivative of that output. SVG, Mermaid, matplotlib, Pillow, HTML/CSS screenshots, Graphviz, draw.io exports, Keynote/PowerPoint exports, and other local/generated visuals are allowed only as intermediate editable sources or structure drafts. They must not be embedded directly, even if converted to PNG/JPG/WebP locally.",
"fact": "All embedded visuals in external Markdown deliverables must be generated or redrawn by a GPT Image model (or its compressed derivative); local charts like matplotlib or Mermaid cannot be embedded directly even if converted to PNG/JPG/WebP locally.",
"rubric": {
"must_include": [
"must be produced by a GPT Image model generation or redraw (or a compressed derivative of it)",
"local visuals (e.g., matplotlib, Mermaid) cannot be embedded directly even if locally converted to PNG/JPG/WebP",
"local visuals are only allowed as intermediate drafts/sources"
],
"reference_answer": "No, local visuals like matplotlib or Mermaid cannot be embedded directly even if converted to PNG; all embedded images must be produced by a GPT Image model generation/redraw (or a compressed derivative of that output), with local charts serving only as intermediate drafts."
},
"excerpt_verbatim": true
},
{
"question": "When generating infographics with GPT Image 2.5, what workflow should be used instead of prompting the model from text alone, and why?",
"excerpt": "when building infographics with GPT Image 2.5, first produce a matplotlib structure draft and pass it as the `-i` input image. The model understands the draft layout, preserves title and annotation text, and improves the visual presentation. Asking it to generate an infographic from a text prompt alone (no input image) produces unreliable results.",
"fact": "When creating infographics with GPT Image 2.5, you must first create a matplotlib structure draft and pass it via the -i input image flag, because generating from a text prompt alone is unreliable.",
"rubric": {
"must_include": [
"first create a matplotlib structure draft",
"pass it as the input image (-i)",
"generating from text alone produces unreliable results"
],
"reference_answer": "You should first create a matplotlib structure draft and pass it as the input image (`-i`), because asking GPT Image 2.5 to generate an infographic from text alone produces unreliable results."
},
"excerpt_verbatim": true
}
],
"cost": 0.004287
},
"innovation_assistant": {
"tasks": [
{
"question": "We are trying to run structured innovation on an existing software product, but we also have broad, open-ended questions about what its future shape should be. Which pipeline setup should be used when both situations apply?",
"excerpt": "Both apply (an existing product AND an open question about its future) → run Think Bigger as the outer loop; use SIT templates as an additional tactic generator inside the search step.",
"fact": "When an existing product and an open question about its future both apply, run Think Bigger as the outer loop and use SIT templates as an additional tactic generator during the search step.",
"rubric": {
"must_include": [
"Think Bigger as the outer loop",
"SIT templates as a tactic generator inside the search step"
],
"reference_answer": "Run the Think Bigger pipeline as the outer loop, and use SIT templates as an additional tactic generator within the search step."
},
"excerpt_verbatim": false
},
{
"question": "In a Think Bigger innovation run, what should happen if an idea receives high scores on the 7-point originality and value scales but conflicts with the results of the Big Picture wants checkoff?",
"excerpt": "the Big Picture wants checkoff selects the winners (it encodes whose preferences the solution optimizes), while the 7-point scales rank the full candidate list and communicate idea quality to the reader. If the two disagree, the wants checkoff wins and the disagreement is worth a sentence in the report.",
"fact": "If the 7-point rating scales and the Big Picture wants checkoff disagree, the wants checkoff decides the winner and the disagreement must be noted in a sentence in the report.",
"rubric": {
"must_include": [
"the wants checkoff wins (or selects the winners)",
"the disagreement must be stated/documented in a sentence in the report"
],
"reference_answer": "The Big Picture wants checkoff wins and selects the winner, and the disagreement between the two instruments must be documented in a sentence in the report."
},
"excerpt_verbatim": false
}
],
"cost": 0.009543000000000001
},
"sit_pipeline": {
"tasks": [
{
"question": "When decomposing a product prior to applying innovation templates, what two conditions must both be met for a component-component link to exist?",
"excerpt": "a component-component link exists only if BOTH hold: (a) a change in one directly changes the other, and (b) that influence is an intentionally assigned function. (A seat buckling the legs is influence without assigned function — not a link.)",
"fact": "A component-component link requires both that a change in one directly changes the other and that the influence is an intentionally assigned function.",
"rubric": {
"must_include": [
"a change in one component directly changes the other",
"the influence is an intentionally assigned function"
],
"reference_answer": "A component-component link exists only if a change in one component directly changes the other and that influence represents an intentionally assigned function."
},
"excerpt_verbatim": true
},
{
"question": "When building the dependency matrix for the Attribute Dependency template, what specifically should be mapped to the rows versus the columns?",
"excerpt": "1. Build a matrix: columns = internal attributes; rows = internal + external attributes.",
"fact": "In the Attribute Dependency matrix procedure, columns represent internal attributes, whereas rows represent both internal and external attributes.",
"rubric": {
"must_include": [
"columns are internal attributes",
"rows are internal and external attributes"
],
"reference_answer": "The matrix columns should consist of internal attributes only, while the rows must consist of both internal and external attributes."
},
"excerpt_verbatim": true
}
],
"cost": 0.0068902500000000005
},
"think_bigger_pipeline": {
"tasks": [
{
"question": "When populating a row on my choice matrix, some of the precedents I found fall into gray areas (such as adapting a tech firm's internal management style to design a software tool). How should these borderline cases be classified and counted toward the requirement for external tactics?",
"excerpt": "mark these as \"out (edge)\" and do not let edge items alone satisfy a row's ≥3 out-of-domain quota: each row needs at least 2 unambiguously out-of-domain tactics.",
"fact": "Borderline precedents must be labeled \"out (edge)\" and cannot fulfill the out-of-domain quota by themselves, as each row requires at least two unambiguously out-of-domain tactics.",
"rubric": {
"must_include": [
"classified or marked as \"out (edge)\"",
"each row must include at least 2 unambiguously out-of-domain tactics (edge cases cannot satisfy the quota on their own)"
],
"reference_answer": "They must be marked as \"out (edge)\" and cannot fulfill the quota alone, because every row requires at least two tactics that are unambiguously out-of-domain."
},
"excerpt_verbatim": true
},
{
"question": "During the random combination phase of the pipeline, can the AI model simply select tactics at random on the fly? How exactly should these random selections be executed and documented?",
"excerpt": "Use a programmatic random number generator and record the seed in the report — an LLM \"choosing randomly\" in its head will pick what looks good and defeat the round's purpose.",
"fact": "The AI must not choose combinations in its head; it must use a programmatic random number generator and document the random seed in the report.",
"rubric": {
"must_include": [
"do not let the AI/LLM pick randomly in its head",
"use a programmatic random number generator",
"record the seed in the report"
],
"reference_answer": "The AI must not pick randomly in its head; instead, it must use a programmatic random number generator and document the seed in the report."
},
"excerpt_verbatim": true
}
],
"cost": 0.00862575
},
"skill_intake": {
"tasks": [
{
"question": "What is the exact column header required in the CSV file generated by the intake_skill ASR step, and what is stored in the speaker column?",
"excerpt": "The CSV header is exactly:\n\n```csv\nspeaker,content\n```\n\nThe `speaker` column is blank in current engines. This skill does not infer identity.",
"fact": "The ASR transcript CSV header must be exactly 'speaker,content', and the speaker column is left blank because identity is not inferred.",
"rubric": {
"must_include": [
"speaker,content",
"blank"
],
"reference_answer": "The CSV header must be exactly `speaker,content`, with the `speaker` column left blank as the current engines do not infer identity."
},
"excerpt_verbatim": true
},
{
"question": "What audio file extensions does the intake_skill sync process read from the source directory?",
"excerpt": "Sync reads only `.qta` and `.m4a` files. It writes normalized audio files under:\n\n```text\ndata/YYYYMMDD/YYYYMMDD_HHMM_watch.m4a\n```",
"fact": "The sync process only reads files with .qta and .m4a extensions.",
"rubric": {
"must_include": [
".qta",
".m4a"
],
"reference_answer": "Sync only reads `.qta` and `.m4a` files."
},
"excerpt_verbatim": true
}
],
"cost": 0.0046575
},
"app_store_release": {
"tasks": [
{
"question": "I set manageAppVersionAndBuildNumber to false in my ExportOptions.plist, but when uploading my archive with xcodebuild -exportArchive, the build number was still changed automatically. Why did the export plist setting fail to prevent this during the upload step?",
"excerpt": "In **both** the export and upload plist files, explicitly set `manageAppVersionAndBuildNumber` to ``. Modifying only the export options plist is ineffective because the upload step performs export again with its own options.",
"fact": "The upload step runs export again using its own upload options plist, so manageAppVersionAndBuildNumber must be set to false in both the export and upload plist files.",
"rubric": {
"must_include": [
"the upload step performs export again with its own options plist",
"manageAppVersionAndBuildNumber must be set to false in the upload plist as well"
],
"reference_answer": "Modifying only the export plist is ineffective because the upload command runs export again using its own upload options plist, so manageAppVersionAndBuildNumber must be set to false in both files."
},
"excerpt_verbatim": true
},
{
"question": "Can I use xcrun altool authenticated with an app-specific password to check whether an individual build has completed processing in App Store Connect? If not, what can be used instead?",
"excerpt": "An app-specific password cannot query the processing status of individual builds; that status requires an App Store Connect API key or the App Store Connect web interface.",
"fact": "An app-specific password cannot query individual build processing status in altool; an App Store Connect API key or the web interface is required.",
"rubric": {
"must_include": [
"an app-specific password cannot check individual build processing status",
"requires an App Store Connect API key or the App Store Connect web interface"
],
"reference_answer": "No, an app-specific password cannot query the processing status of individual builds; you must use an App Store Connect API key or the App Store Connect web interface."
},
"excerpt_verbatim": true
}
],
"cost": 0.014152499999999998
},
"ios_development": {
"tasks": [
{
"question": "When trying to speed up our CI pipeline, we attempted running multiple concurrent `xcodebuild` jobs against the same `-derivedDataPath` for our iOS project, but they crashed. What specific error message causes this failure?",
"excerpt": "Run `xcodebuild` sequentially against a single `-derivedDataPath` per project. Parallel builds against the same project or DerivedData fail with `build.db: database is locked`.",
"fact": "Running parallel builds against the same project or DerivedData fails with the error `build.db: database is locked`.",
"rubric": {
"must_include": [
"build.db: database is locked"
],
"reference_answer": "Parallel builds against the same project or DerivedData directory fail with the error `build.db: database is locked`."
},
"excerpt_verbatim": true
},
{
"question": "We are using `scripts/team_from_profile.sh` to extract our development team ID from a cached provisioning profile, but we need to target a specific profile instead of the default team wildcard. How do we pass that profile to the script?",
"excerpt": "The script prints only the team ID. Pass a profile name as the first argument to choose a profile other than the team wildcard.",
"fact": "To select a profile other than the default team wildcard in scripts/team_from_profile.sh, pass the profile name as the first argument.",
"rubric": {
"must_include": [
"first argument",
"profile name"
],
"reference_answer": "Pass the profile name as the first argument to the script to select a profile other than the team wildcard."
},
"excerpt_verbatim": true
}
],
"cost": 0.00319425
},
"real_device": {
"tasks": [
{
"question": "My iPhone and Mac are both connected to the same Tailscale tailnet, and I can successfully reach the phone with `tailscale ping`. However, `xcrun devicectl list devices` marks the device as unavailable. Why can't `devicectl` communicate with the device over the tailnet?",
"excerpt": "CoreDevice discovers devices over Bonjour, which the tailnet did not carry. Install, file copy, launch, and openURL were therefore unavailable until the phone was back on the same LAN; then it showed `available (paired)` again.",
"fact": "CoreDevice discovers paired physical devices via Bonjour, which is not carried over a Tailscale tailnet.",
"rubric": {
"must_include": [
"CoreDevice discovers devices using Bonjour",
"the tailnet does not carry Bonjour"
],
"reference_answer": "CoreDevice relies on Bonjour discovery to connect to paired devices, and the Tailscale tailnet does not carry Bonjour traffic."
},
"excerpt_verbatim": true
},
{
"question": "In our iOS application that only reads data from HealthKit, checking `HKHealthStore.authorizationStatus(for:)` returns `.sharingDenied`. Does this mean the user explicitly denied our app access to read that health metric?",
"excerpt": "`HKHealthStore.authorizationStatus(for:)` reflects **write** permission only. An authorization status of `.sharingDenied` in a read-only app does not establish that reading was refused.",
"fact": "`HKHealthStore.authorizationStatus(for:)` reflects write permissions only, so `.sharingDenied` does not indicate that read authorization was refused.",
"rubric": {
"must_include": [
"authorizationStatus only checks write (or sharing) permission",
".sharingDenied does not establish or prove that read access was denied"
],
"reference_answer": "No, because `HKHealthStore.authorizationStatus(for:)` reflects write permission only, so a status of `.sharingDenied` in a read-only app does not mean read access was denied."
},
"excerpt_verbatim": true
}
],
"cost": 0.014989500000000001
},
"simulator_testing": {
"tasks": [
{
"question": "In our XCUITest suite on iPhone simulators, we need to perform controlled scrolling without triggering fling momentum, but coordinate scroll APIs fail with pointer errors and drag APIs are missing. What specific coordinate method and duration should we call to achieve this?",
"excerpt": "For controlled scrolling without fling momentum, use:\n ```swift\n coordinate.press(forDuration: 0.1, thenDragTo: otherCoordinate)\n ```\n A duration of 0.1 s stays below the system long-press threshold, functioning as a controlled drag.",
"fact": "To achieve controlled scrolling without fling momentum when drag APIs are absent, use coordinate.press(forDuration: 0.1, thenDragTo: otherCoordinate).",
"rubric": {
"must_include": [
"press(forDuration:thenDragTo:)",
"0.1"
],
"reference_answer": "Call coordinate.press(forDuration: 0.1, thenDragTo: otherCoordinate), using a 0.1-second duration to stay under the system long-press threshold."
},
"excerpt_verbatim": true
},
{
"question": "Our simulator test run in xcodebuild exited with a generic TEST EXECUTE FAILED error without detailing which assertion failed. What exact xcresulttool command should we run to inspect the .xcresult bundle and extract the failing test cases, error messages, and source lines?",
"excerpt": "When `xcodebuild` logs generic failures such as `TEST EXECUTE FAILED`, inspect the `.xcresult` bundle path printed near the end of the log output:\n\n```bash\nxcrun xcresulttool get test-results tests --path \".xcresult\"\n```\n\nThis extracts the failing test case, error message, and source file line number directly, avoiding the need to sift through large build logs.",
"fact": "Failing test cases, error messages, and source line numbers can be extracted from an .xcresult bundle using xcrun xcresulttool get test-results tests --path \".xcresult\".",
"rubric": {
"must_include": [
"xcresulttool get test-results tests",
"--path"
],
"reference_answer": "Run xcrun xcresulttool get test-results tests --path \".xcresult\" to extract the failing test cases, error messages, and line numbers."
},
"excerpt_verbatim": true
}
],
"cost": 0.015644250000000002
},
"watch_development": {
"tasks": [
{
"question": "Our watchOS app crashes with an assertion failure in `CLLocationManager setAllowsBackgroundLocationUpdates:` as soon as a workout starts, even though we have `workout-processing` in `WKBackgroundModes`. How should location updates be configured during an active workout on watchOS to prevent this error?",
"excerpt": "A recording app gets background runtime from the active `HKWorkoutSession`; while the session\nruns, CoreLocation keeps delivering updates, so the app does not need — and must not set —\n`allowsBackgroundLocationUpdates` on watchOS",
"fact": "On watchOS, apps must not set allowsBackgroundLocationUpdates because background CoreLocation delivery is provided directly by the active HKWorkoutSession.",
"rubric": {
"must_include": [
"do not set allowsBackgroundLocationUpdates (or remove setting it to true)",
"background runtime/location delivery is provided by the active HKWorkoutSession"
],
"reference_answer": "Do not set allowsBackgroundLocationUpdates on watchOS, because location updates are automatically delivered in the background via the active HKWorkoutSession."
},
"excerpt_verbatim": true
},
{
"question": "I am trying to turn on Developer Mode on my Apple Watch, but the option is completely missing under Settings > Privacy & Security. What steps are required to make the Developer Mode option appear?",
"excerpt": "If *Settings > Privacy & Security* on the watch lacks a Developer Mode entry:\n1. Ensure the watch has been paired with the Mac in Device Hub at least once.\n2. **Restart the Apple Watch**.\n3. After rebooting, open *Settings > Privacy & Security*; the Developer Mode toggle will appear.",
"fact": "If Developer Mode is missing on the Apple Watch, the watch must be paired with the Mac in Device Hub at least once and then restarted.",
"rubric": {
"must_include": [
"pair the watch with the Mac in Device Hub at least once",
"restart (or reboot) the Apple Watch"
],
"reference_answer": "Pair the Apple Watch with your Mac via Device Hub at least once, and then restart the watch so the Developer Mode toggle appears in Settings > Privacy & Security."
},
"excerpt_verbatim": true
}
],
"cost": 0.01166325
},
"skill_kit": {
"tasks": [
{
"question": "I am viewing subscriber lists through the Kit CLI, but all the email addresses are hidden by default. What specific command-line option should I include to show the actual email addresses?",
"excerpt": "Subscriber list commands redact emails by default. Use `--show-emails` only when the terminal, log, and output file are private.",
"fact": "Passing the `--show-emails` flag disables the default redaction of email addresses in subscriber list commands.",
"rubric": {
"must_include": [
"--show-emails"
],
"reference_answer": "Pass the `--show-emails` flag to display unredacted subscriber email addresses."
},
"excerpt_verbatim": true
},
{
"question": "What exit status and error message does the Kit CLI return if I run the subscriber count command with a tag name that doesn't exist in the account?",
"excerpt": "An unknown tag name exits 1 with `Tag not found`.",
"fact": "Providing an unknown tag name to the subscriber count command exits with code 1 and displays the message 'Tag not found'.",
"rubric": {
"must_include": [
"1",
"Tag not found"
],
"reference_answer": "The command exits with status code 1 and outputs the error message `Tag not found`."
},
"excerpt_verbatim": true
}
],
"cost": 0.007611000000000001
},
"koyeb": {
"tasks": [
{
"question": "Which specific command-line flags does the Koyeb environment launcher script block from being passed to the CLI, and what exit code does it return when one is encountered?",
"excerpt": "The launcher explicitly rejects `--token`, `--debug-full`, and `--url` (exits with code `2`). Never pass tokens in `argv`.",
"fact": "The launcher script rejects the flags `--token`, `--debug-full`, and `--url` with exit code 2.",
"rubric": {
"must_include": [
"--token, --debug-full, and --url",
"code 2"
],
"reference_answer": "The launcher explicitly rejects `--token`, `--debug-full`, and `--url`, exiting with code 2."
},
"excerpt_verbatim": true
},
{
"question": "We are setting up automation to verify whether our Koyeb service has had zero traffic so it can safely scale down. Why shouldn't we rely on the standard CLI's metrics command for this check, and what script should be used instead?",
"excerpt": "*Rule*: Native CLI maps upstream nulls to 0. Always use `scripts/request_metrics.py` when evaluating scale-to-zero eligibility or confirming inactivity.",
"fact": "The native CLI maps upstream nulls to 0, requiring `scripts/request_metrics.py` to be used when evaluating inactivity or scale-to-zero eligibility.",
"rubric": {
"must_include": [
"maps nulls to 0",
"scripts/request_metrics.py"
],
"reference_answer": "The native CLI maps upstream nulls to 0 rather than preserving them, so `scripts/request_metrics.py` must be used instead to evaluate inactivity."
},
"excerpt_verbatim": true
}
],
"cost": 0.01303725
},
"logto_management_skill": {
"tasks": [
{
"question": "I am setting up an initial machine-to-machine application in the Logto Console to manage our tenant programmatically. What specific scope and built-in role need to be assigned to this M2M application for Logto Management API access?",
"excerpt": "The Management API has one all-or-nothing scope named `all`; the required built-in role is `Logto Management API access`.",
"fact": "Logto Management API requires a single all-or-nothing scope named `all` and the built-in role `Logto Management API access`.",
"rubric": {
"must_include": [
"all",
"Logto Management API access"
],
"reference_answer": "The application requires the scope named `all` and the built-in role named `Logto Management API access`."
},
"excerpt_verbatim": true
},
{
"question": "I need to configure field permissions for our Logto account center using the `account-center set-fields` command in the management CLI. What are the allowed permission values that can be assigned to a field?",
"excerpt": "Field permissions are `Off`, `ReadOnly`, or `Edit`. Invalid values fail locally before network access.",
"fact": "Account center field permissions in the management CLI only accept `Off`, `ReadOnly`, or `Edit`.",
"rubric": {
"must_include": [
"Off",
"ReadOnly",
"Edit"
],
"reference_answer": "The permitted values for account center fields are `Off`, `ReadOnly`, and `Edit`."
},
"excerpt_verbatim": true
}
],
"cost": 0.0122355
},
"narwal": {
"tasks": [
{
"question": "I tried starting a cleaning cycle using the `narwal-local start` command while the robot was undocked in another room, but the command failed. What state must the robot be in for a clean command to be accepted, and what specific response status or exit code does the robot return if it is not in that state?",
"excerpt": "Both use `clean/start_clean`, which **only applies while the robot is docked**; otherwise the robot answers `NOT_READY` (exit 20).",
"fact": "A clean command only applies when the robot is docked; otherwise the robot declines it with NOT_READY (exit code 20).",
"rubric": {
"must_include": [
"The robot must be docked",
"NOT_READY (or exit code 20)"
],
"reference_answer": "The robot must be docked for clean commands to be accepted, otherwise it will reject the command with NOT_READY and exit code 20."
},
"excerpt_verbatim": true
},
{
"question": "When inspecting the JSON output from `narwal-local snapshot`, both `is_docked` and `charging_state` show up as `null`. How should I determine whether the vacuum is actually docked and charging using the CLI?",
"excerpt": "To tell whether it is docked or charging, use the **battery trend** (rising across a `watch` window or between snapshots), not these fields.",
"fact": "To determine whether the robot is docked or charging, observe the battery trend (rising across snapshots or a watch window) rather than relying on the is_docked or charging_state fields.",
"rubric": {
"must_include": [
"battery trend",
"rising battery percentage across snapshots or a watch window"
],
"reference_answer": "To determine if the robot is docked or charging, observe the battery trend to see if the battery percentage is rising between snapshots or across a watch window."
},
"excerpt_verbatim": true
}
],
"cost": 0.012282
},
"bilingual_subtitles": {
"tasks": [
{
"question": "When segmenting English subtitles for bilingual tracks, what is the standard word limit for a segment, and what is the maximum length allowed if a phrase cannot be split?",
"excerpt": "Most English segments stay at or below 12 words; an inseparable phrase may reach 16.",
"fact": "English subtitle segments should generally be kept at or below 12 words, but an inseparable phrase can reach up to 16 words.",
"rubric": {
"must_include": [
"12 words",
"16 words"
],
"reference_answer": "Most English segments must stay at or below 12 words, with an inseparable phrase allowed to reach up to 16 words."
},
"excerpt_verbatim": true
},
{
"question": "When partitioning subtitle cues into adjacent batches for parallel processing, what criteria should be used to assign cues to avoid duplicating entries that span packet boundaries?",
"excerpt": "Selecting every cue that intersects adjacent time packets duplicates boundary-spanning cues. Packet assignment must use cue start time.",
"fact": "To prevent duplicate boundary-spanning cues across adjacent packets, cues must be assigned strictly by cue start time.",
"rubric": {
"must_include": [
"cue start time"
],
"reference_answer": "Cues must be assigned to packets based strictly on their cue start time rather than intersection."
},
"excerpt_verbatim": true
}
],
"cost": 0.009747
},
"download_and_transcribe": {
"tasks": [
{
"question": "When fetching media files directly from Bilibili CDN URLs during the curl fallback route, the CDN server rejects my download requests. Which specific HTTP header must be included alongside the browser User-Agent to prevent this rejection?",
"excerpt": "Download the returned CDN URL with the browser UA and a `Referer: https://www.bilibili.com/` header, otherwise the CDN rejects the request.",
"fact": "Downloading from Bilibili CDN URLs requires including a `Referer: https://www.bilibili.com/` header alongside the browser User-Agent.",
"rubric": {
"must_include": [
"Referer",
"https://www.bilibili.com/"
],
"reference_answer": "You must include the header `Referer: https://www.bilibili.com/` along with the browser user agent."
},
"excerpt_verbatim": true
},
{
"question": "What specific connection and request rate limits should be configured in streamrip before starting batch downloads from Tidal to avoid issues?",
"excerpt": "Before batch work, keep streamrip conservative: `max_connections = 1` or `2`, `requests_per_minute = 20` or lower, and no parallel agents using the same account.",
"fact": "Streamrip should be configured with max_connections set to 1 or 2 and requests_per_minute set to 20 or lower before batch Tidal downloads.",
"rubric": {
"must_include": [
"max_connections of 1 or 2",
"requests_per_minute of 20 or lower"
],
"reference_answer": "Configure streamrip with `max_connections = 1` or `2` and `requests_per_minute = 20` or lower, with no parallel agents using the account."
},
"excerpt_verbatim": true
}
],
"cost": 0.01145925
},
"media_metadata": {
"tasks": [
{
"question": "When downloading a medley video, who should be set as the downloaded audio file's artist tag, and what should be done with the original artists of the individual source songs?",
"excerpt": "This skill can use platform metadata and thumbnails to fill local tags. It should not decide the true original artist of a medley source song. For medleys, the downloaded file's artist is the uploader/performer; individual source-song artists belong in the medley identification output.",
"fact": "For medleys, the downloaded audio file's artist tag must be set to the uploader or performer, while the individual source-song artists are placed in the medley identification output.",
"rubric": {
"must_include": [
"uploader or performer as the file's artist",
"individual or source-song artists go into medley identification output"
],
"reference_answer": "The downloaded file's artist should be set to the uploader/performer, while the individual source-song artists belong in the medley identification output."
},
"excerpt_verbatim": true
},
{
"question": "What must be done to accepted Tidal FLAC tracks before they can be imported or synced to Apple Music or iOS, and what should happen to the original FLAC files?",
"excerpt": "Convert accepted FLAC sources to ALAC `.m4a`, then explicitly write MP4 tags and artwork after conversion. Keep the source FLAC files in the private runtime library unless the user asks to delete archive copies.",
"fact": "Accepted FLAC sources for Apple Music/iOS sync must be converted to ALAC .m4a with MP4 tags and artwork explicitly written after conversion, while the source FLAC files are retained in the private runtime library unless deletion is requested.",
"rubric": {
"must_include": [
"convert FLAC to ALAC (.m4a) and write MP4 tags/artwork",
"keep source FLAC files in the private runtime library unless deletion is requested"
],
"reference_answer": "Convert the FLAC files to ALAC .m4a and explicitly write MP4 tags and artwork after conversion, keeping the original FLAC files in the private runtime library unless the user explicitly requests their deletion."
},
"excerpt_verbatim": true
}
],
"cost": 0.00310875
},
"medley_source_identification": {
"tasks": [
{
"question": "During the post-identification deduplication step for medley songs, how does the system handle transition rows containing slash-separated song titles (such as `title A / title B`)?",
"fact": "Boundary rows containing slash-separated titles are split into separate entries only when the artist field contains matching slash-separated parts.",
"excerpt": "Boundary rows such as `title A / title B` are expanded into separate entries when the artist field has matching slash-separated parts.",
"rubric": {
"must_include": [
"expanded into separate entries (or split into separate rows)",
"artist field has matching slash-separated parts"
],
"reference_answer": "Boundary rows with slash-separated titles are expanded into separate entries provided that the artist field also has matching slash-separated parts."
},
"excerpt_verbatim": true
},
{
"question": "If I am identifying songs in a medley video that is branded around a specific theme year (such as an '80s hits compilation), should I populate the release_year column using that year? What is the required procedure?",
"fact": "Video theme years must not be used as release years, and release_year must be left blank unless separately sourced.",
"excerpt": "Video theme year is not the release year for each song. Leave `release_year` blank unless separately sourced.",
"rubric": {
"must_include": [
"leave release_year blank",
"separately sourced (or independently verified)"
],
"reference_answer": "Do not use the video theme year; leave the release_year field blank unless the release year for that specific song is separately sourced."
},
"excerpt_verbatim": true
}
],
"cost": 0.009321
},
"music_library_dedup": {
"tasks": [
{
"question": "When ranking audio files within an exact duplicate cluster to choose a canonical copy, what format priority order does the CLI follow for audio formats?",
"excerpt": "2. prefer higher-quality formats: FLAC > M4A > AAC > MP3 > WMA",
"fact": "The CLI prioritizes audio formats in the order FLAC > M4A > AAC > MP3 > WMA when selecting a canonical file.",
"rubric": {
"must_include": [
"FLAC > M4A > AAC > MP3 > WMA (or equivalent ordered listing)"
],
"reference_answer": "The CLI prefers higher-quality formats in the specific order FLAC > M4A > AAC > MP3 > WMA."
},
"excerpt_verbatim": true
},
{
"question": "What cluster type is assigned to an exact duplicate cluster if version markers like \"Live\", \"Cover\", \"伴奏\", or \"串烧\" are detected in the tracks?",
"excerpt": "If version markers such as `Live`, `Cover`, `伴奏`, or `串烧` appear, the cluster is marked `needs_review` even when title and artist match exactly.",
"fact": "If version markers appear, the duplicate cluster is marked with the cluster_type needs_review.",
"rubric": {
"must_include": [
"needs_review"
],
"reference_answer": "The cluster is marked as needs_review."
},
"excerpt_verbatim": true
}
],
"cost": 0.0024015
},
"online_media": {
"tasks": [
{
"question": "While trying to identify a track, the search engine's aggregate answer explicitly names a song, but none of the returned snippets actually corroborate or support that title. What confidence rating should be assigned to this match?",
"excerpt": "Search aggregate answers can name a song that returned snippets do not support. Treat this as `low` confidence.",
"fact": "A song named in search aggregate answers without support from the returned snippets must be treated as low confidence.",
"rubric": {
"must_include": [
"low"
],
"reference_answer": "Treat the match as having low confidence."
},
"excerpt_verbatim": true
},
{
"question": "In our audio transcription pipeline, an ASR transcript appears to have converted a proper song title or named entity into ordinary words. What should be used in search queries instead of relying on the suspect token alone?",
"excerpt": "ASR can turn named entities or song titles into plausible common words. Search nearby lyric anchors rather than the suspect token alone.",
"fact": "When ASR transcribes a named entity or song title as plausible common words, queries should search nearby lyric anchors instead of the suspect token alone.",
"rubric": {
"must_include": [
"nearby lyric anchors"
],
"reference_answer": "You should search for nearby lyric anchors rather than using the suspect token alone."
},
"excerpt_verbatim": true
}
],
"cost": 0.010916249999999999
},
"source_search": {
"tasks": [
{
"question": "When configuring yt-dlp to download YouTube candidates in our audio pipeline, what specific runtime and remote component options must be enabled, and what failure does this prevent?",
"excerpt": "For YouTube, run `yt-dlp` with a JavaScript runtime and the EJS remote component enabled; otherwise YouTube signature and n-challenge solving can silently miss formats.",
"fact": "yt-dlp must be run with a JavaScript runtime and the EJS remote component enabled to prevent signature and n-challenge solving from silently missing formats.",
"rubric": {
"must_include": [
"a JavaScript runtime (or deno) and the EJS remote component (ejs:github)",
"signature and n-challenge solving silently missing formats"
],
"reference_answer": "You must enable a JavaScript runtime and the EJS remote component so that YouTube signature and n-challenge solving do not silently miss available formats."
},
"excerpt_verbatim": true
},
{
"question": "During a batch run with streamrip on Tidal, search and audio downloads succeed, but requests to the lyrics endpoint fail with a 401 Unauthorized error. What should be done to handle this situation?",
"excerpt": "If streamrip fails on the Tidal lyrics endpoint with `401 Unauthorized` while track search and audio access work, skip lyrics sidecars and continue with audio verification.",
"fact": "A 401 Unauthorized error on the Tidal lyrics endpoint should be handled by skipping lyrics sidecars and continuing with audio verification.",
"rubric": {
"must_include": [
"skip lyrics sidecars (or skip fetching lyrics)",
"continue with audio verification (or proceed with audio downloading)"
],
"reference_answer": "Skip the lyrics sidecars and continue with audio verification rather than aborting the audio download."
},
"excerpt_verbatim": true
}
],
"cost": 0.009413999999999999
},
"skill_opencode_agent_to_agent": {
"tasks": [
{
"question": "When querying the `GET /experimental/session` endpoint, what sessions are returned if I set the query parameter `archived=true`? Does it return only sessions that have been archived?",
"excerpt": "`archived=true` returns both archived and non-archived sessions (it is not archived-only).",
"fact": "Passing `archived=true` to `GET /experimental/session` returns both archived and non-archived sessions rather than archived sessions only.",
"rubric": {
"must_include": [
"both archived and non-archived",
"not archived-only"
],
"reference_answer": "Setting `archived=true` returns both archived and non-archived sessions; it does not filter for archived sessions only."
},
"excerpt_verbatim": true
},
{
"question": "During pre-send verification of a peer session, how should an agent proceed if the parent-provided owner record specifies a project, but the session object returned from `GET /session/{id}` does not contain a comparable project field?",
"excerpt": "If the owner record has a project and the GET view has no equal project value, stop with `project_mismatch`. Do not report project as matched when the GET view has no comparable field.",
"fact": "If the owner record specifies a project but the GET response lacks an equal project value, the agent must stop with `project_mismatch`.",
"rubric": {
"must_include": [
"stop",
"project_mismatch"
],
"reference_answer": "The agent must stop immediately with `project_mismatch` rather than treating the missing field as a match."
},
"excerpt_verbatim": true
}
],
"cost": 0.014243249999999999
},
"skill_opencode_data": {
"tasks": [
{
"question": "When using the OpenCode CLI selectors to target sessions, what does the tool do with child or subagent sessions by default, and which flag should I use if I only want the exact matching sessions?",
"excerpt": "By default, CLI selectors expand descendants through the session `parent_id` chain. This keeps root sessions and subagent child sessions together. Use `--no-expand` only when the user explicitly wants the raw selector result.",
"fact": "Selectors automatically expand descendant sessions via parent_id by default, requiring --no-expand to restrict results to the raw match.",
"rubric": {
"must_include": [
"expands descendants",
"--no-expand"
],
"reference_answer": "By default, selectors expand descendants along the parent_id chain to include child sessions, and you must pass the `--no-expand` flag to get only the raw selector matches."
},
"excerpt_verbatim": true
},
{
"question": "When running an OpenCode export to markdown files, why might subagent fan-out sessions be missing from the output directory, and how can they be included?",
"excerpt": "Subagent fan-out sessions are skipped unless `--include-subagent` is set; `--since` accepts a relative window (e.g. `30d`) or an ISO date.",
"fact": "OpenCode export skips subagent fan-out sessions unless --include-subagent is explicitly passed.",
"rubric": {
"must_include": [
"skipped by default",
"--include-subagent"
],
"reference_answer": "Subagent fan-out sessions are skipped by default during export unless the `--include-subagent` flag is provided."
},
"excerpt_verbatim": true
}
],
"cost": 0.0028694999999999997
},
"skill_opencode_periodic_job": {
"tasks": [
{
"question": "I want to schedule an OpenCode prompt as a one-shot job to run later today rather than recurring continuously. Should I configure this using crontab with the OpenCode Skill CLI?",
"excerpt": "For one-shot delayed OpenCode jobs, prefer `process_launcher.md` durable scheduled jobs instead of cron. Still use this skill for recurring cron cadence.",
"fact": "For one-shot delayed OpenCode jobs, durable scheduled jobs via process_launcher.md should be preferred over cron.",
"rubric": {
"must_include": [
"process_launcher",
"cron is intended for recurring/periodic jobs, not one-shot delayed jobs"
],
"reference_answer": "No, for one-shot delayed OpenCode jobs you should prefer `process_launcher.md` durable scheduled jobs instead of cron."
},
"excerpt_verbatim": true
},
{
"question": "When testing a scheduled OpenCode periodic cron entry live, how far into the future should the temporary cron test be scheduled, and why?",
"excerpt": "Then schedule at least two minutes in the future. One-minute cron tests are flaky because the edit may land too close to the next minute boundary.",
"fact": "Temporary cron tests must be scheduled at least two minutes in the future because one-minute tests are flaky due to proximity to the next minute boundary.",
"rubric": {
"must_include": [
"at least two minutes",
"boundary"
],
"reference_answer": "It should be scheduled at least two minutes in the future because one-minute cron tests are flaky if the edit lands too close to the next minute boundary."
},
"excerpt_verbatim": true
}
],
"cost": 0.00323325
},
"skill_opencode_submission": {
"tasks": [
{
"question": "When passing `--model huggingface/example-org/example-model` to the OpenCode submission command, how does the CLI parse the provider and model components?",
"excerpt": "Model ids may contain slashes: in `--model provider/model` only the first segment is the provider, so `--model huggingface/example-org/example-model` parses as provider `huggingface` and model `example-org/example-model`.",
"fact": "When a model argument contains slashes, only the first segment is treated as the provider, with the remaining segments parsed as the model identifier.",
"rubric": {
"must_include": [
"provider is huggingface (or only the first segment)",
"model is example-org/example-model (or the remaining segment(s))"
],
"reference_answer": "Only the first segment is parsed as the provider (huggingface), while the entire remaining string is parsed as the model identifier (example-org/example-model)."
},
"excerpt_verbatim": true
},
{
"question": "When monitoring concurrent OpenCode jobs to determine whether they are busy or idle, which HTTP endpoint should be queried as the source of truth, and why shouldn't individual session endpoints be used?",
"excerpt": "When waiting for concurrent jobs, use the server's aggregate `GET /session/status` map as the source of truth for busy/idle state. `GET /session/{id}` may omit reliable running/status fields and can make a caller wait incorrectly or declare completion too early.",
"fact": "The aggregate GET /session/status endpoint is the source of truth for busy/idle state because GET /session/{id} can omit reliable running/status fields.",
"rubric": {
"must_include": [
"GET /session/status",
"GET /session/{id} may omit reliable running/status fields (or cause premature completion / incorrect waiting)"
],
"reference_answer": "You must use the aggregate GET /session/status endpoint because GET /session/{id} may omit reliable status fields, which could cause a caller to wait incorrectly or assume completion too early."
},
"excerpt_verbatim": true
}
],
"cost": 0.0127245
},
"skill_opencode_throughput": {
"tasks": [
{
"question": "When measuring OpenCode throughput, how is the generation duration calculated for assistant messages that concluded with tool calls?",
"excerpt": "When a message ended with `finish == \"tool-calls\"`, the duration is truncated at the earliest tool `state.time.start`, because the time the model spent waiting for a tool to run is not inference time.",
"fact": "For messages finishing with tool calls, generation duration is truncated at the earliest tool state.time.start.",
"rubric": {
"must_include": [
"truncated at the earliest tool state.time.start",
"tool execution time is excluded from inference time"
],
"reference_answer": "The generation duration is truncated at the earliest tool's `state.time.start` to exclude the time spent waiting for tool execution."
},
"excerpt_verbatim": true
},
{
"question": "What is the default minimum token count required by the OpenCode throughput command for an assistant reply to be included in the calculation?",
"excerpt": "Samples with fewer than `--min-output` output+reasoning tokens are dropped (tiny replies make the ratio noisy), as are samples whose duration falls outside a sane `[0.2s, 3600s]` band.",
"fact": "The default value for --min-output tokens is 50.",
"rubric": {
"must_include": [
"50",
"output and reasoning tokens"
],
"reference_answer": "The default minimum is 50 combined output and reasoning tokens."
},
"excerpt_verbatim": true
}
],
"cost": 0.0028575
},
"procedural-video-frames": {
"tasks": [
{
"question": "I'm muxing my rendered video frames, an audio track, and a soft subtitle track together using ffmpeg with the -shortest flag. However, the final video cuts off unexpectedly early before the visual and audio streams finish. What is causing this, and how should the subtitle track be added to prevent it?",
"excerpt": "Never add a soft subtitle track in the same command as `-shortest`: a track whose last cue ends early cuts the film there (`assemble` muxes subtitles in a second, stream-copy pass).",
"fact": "Muxing a soft subtitle track in the same command as -shortest truncates the video if the last subtitle cue ends before the other streams, so subtitles must be added in a separate stream-copy pass.",
"rubric": {
"must_include": [
"The video terminates early because the subtitle track's last cue ends early and -shortest cuts the output at the shortest stream",
"Subtitles must be added in a separate/second stream-copy pass"
],
"reference_answer": "The video cuts off prematurely because -shortest stops at the end of the subtitle stream if its last cue finishes early; subtitles should be added in a second, stream-copy pass instead."
},
"excerpt_verbatim": true
},
{
"question": "In our automated rendering pipeline, we set up a script that uses `pgrep -f` to wait until a background render pass finishes before starting the next pass. However, the check hangs indefinitely and never detects that the script ended. Why does `pgrep -f` fail to unblock in this environment?",
"excerpt": "Do not wait on a name with `pgrep -f`. It matches every process whose command line contains the name, and agent harnesses leave long-lived shells around whose command lines mention the script: the shell that launched a render with `&`, and other waiters.",
"fact": "Using `pgrep -f` to monitor a render script matches lingering launcher or waiter shells whose command lines contain the script name, causing the wait condition to loop indefinitely.",
"rubric": {
"must_include": [
"pgrep -f inspects the entire command line of processes",
"Lingering launcher shells or waiter shells still mention the script name in their arguments and continue matching"
],
"reference_answer": "`pgrep -f` matches against entire command lines, which continuously matches long-lived launcher shells or sibling waiter processes that mention the script name."
},
"excerpt_verbatim": false
}
],
"cost": 0.0149595
},
"video-scoring-audio": {
"tasks": [
{
"question": "When generating music using Google Lyria via the Gemini API, what kind of phrasing in the text prompt triggers the copyright filter and causes the call to fail with finishReason: 'OTHER'?",
"excerpt": "Naming an ethnic or genre tradition in the prompt gets the request blocked. A\nprompt containing \"in the East Asian tradition\" came back with `finishReason:\n\"OTHER\"` and a `finishMessage` saying the content may contain material\nresembling existing copyrighted works -- **no audio, but the call still counts\nagainst your quota.**",
"fact": "Naming an ethnic or genre tradition in a Google Lyria prompt triggers the copyright filter and blocks the request.",
"rubric": {
"must_include": [
"ethnic or genre tradition"
],
"reference_answer": "Naming an ethnic or genre tradition in the prompt triggers Lyria's copyright filter and blocks the request."
},
"excerpt_verbatim": true
},
{
"question": "Why should you avoid letting ffmpeg downmix stereo audio to mono with `-ac 1` before measuring the peak level?",
"excerpt": "Do not let ffmpeg downmix with `-ac 1` before measuring peak. For correlated\nstereo (a mono mix written to both channels, the usual case for a voice-over) it\nsums the channels at about +3 dB: a mix with a true peak of -1.5 dBFS measured\n1.19 and was reported as clipping.",
"fact": "Downmixing correlated stereo with ffmpeg's -ac 1 sums the channels by about +3 dB, causing false clipping reports.",
"rubric": {
"must_include": [
"sums the channels at about +3 dB",
"false clipping"
],
"reference_answer": "For correlated stereo, ffmpeg's `-ac 1` sums the channels by roughly +3 dB, which can falsely report a compliant audio mix as clipping."
},
"excerpt_verbatim": true
}
],
"cost": 0.0061335
},
"skill_outlook": {
"tasks": [
{
"question": "When updating a calendar event using the Outlook CLI tool, why is passing a body file via `--body-file` intentionally unsupported?",
"excerpt": "`--body-file` is intentionally not supported: Graph replaces the whole body, and dropping the meeting blob from an online meeting's body disables the meeting, so editing the body needs a read-modify-write that this command does not do.",
"fact": "The CLI does not support `--body-file` in `calendar update` because Microsoft Graph replaces the entire body, and stripping the online meeting blob disables the meeting.",
"rubric": {
"must_include": [
"Graph replaces the entire body",
"meeting blob"
],
"reference_answer": "It is unsupported because Microsoft Graph overwrites the entire body, which would drop the meeting blob from an online meeting and disable it."
},
"excerpt_verbatim": true
},
{
"question": "When creating or updating calendar events with `calendar invite` or `calendar update`, what is the default timezone used if no `--timezone` argument is provided?",
"excerpt": "The zone accepts IANA names (`America/Los_Angeles`) or case-insensitive aliases (`PT`/`PST`/`PDT`/`Pacific`/`US/Pacific` → `America/Los_Angeles`, `MT` → `America/Denver`, `CT` → `America/Chicago`, `ET` → `America/New_York`, plus the Windows name `Pacific Standard Time` → `America/Los_Angeles`); the default is `UTC`.",
"fact": "The default timezone for calendar CLI commands is UTC, and updating an event does not inherit the event's existing timezone.",
"rubric": {
"must_include": [
"UTC"
],
"reference_answer": "The default timezone is UTC."
},
"excerpt_verbatim": true
}
],
"cost": 0.004715250000000001
},
"skill_playwright_test": {
"tasks": [
{
"question": "In our E2E auth tests using Logto, after a guest completes the registration flow via the signup URL, our backend check finds that `lastSignInAt` remains null (or `has_logged_in=0`). Why does this happen, and what is required to verify that `has_logged_in=1`?",
"excerpt": "Register creates the user and sets a password, but doesn't constitute a sign-in. `lastSignInAt` only updates on actual sign-in.\n\n**Fix:** `has_logged_in=0` after register is expected behavior. To verify `has_logged_in=1`, the guest must complete a real sign-in (not register).",
"fact": "In Logto, the registration flow creates a user and password but does not count as a sign-in, leaving `lastSignInAt` null until a real sign-in occurs.",
"rubric": {
"must_include": [
"Registration creates the user/password but does not constitute an actual sign-in",
"The user must complete a real sign-in (not just registration) for `lastSignInAt` to update"
],
"reference_answer": "Registering only creates the user and sets their password rather than signing them in, so the user must complete a real sign-in for `lastSignInAt` to update."
},
"excerpt_verbatim": true
},
{
"question": "While debugging an SSO flow in a browser session, the page gets stuck oscillating continuously between 'loading' and 'signing in' states, and backend logs report continuous 401 errors. What is causing this infinite loop?",
"excerpt": "The CDP Chrome user-data-dir contains session tokens from a previous test. The SSO SDK detects the old token, uses it, gets 401, refreshes, gets 401 again — infinite loop.",
"fact": "Reusing a CDP Chrome user-data-dir with stale session tokens triggers an infinite redirect loop in the SSO SDK due to repeated 401 failures and token refreshes.",
"rubric": {
"must_include": [
"The Chrome user-data-dir contains stale session tokens from a previous run",
"The SSO SDK tries to use and refresh the expired/invalid token repeatedly after receiving 401s"
],
"reference_answer": "The Chrome user-data-dir contains stale session tokens from a previous test, causing the SSO SDK to repeatedly attempt using and refreshing the invalid token upon getting 401 errors."
},
"excerpt_verbatim": true
}
],
"cost": 0.011253
},
"skill_session_keeper": {
"tasks": [
{
"question": "We are setting up session-keeper to refresh credentials for an internal service, but the service doesn't have any safe, read-only endpoints we can probe. How should the tool behave when invoked with the `--apply` flag in this scenario?",
"excerpt": "If the site has no clean read-only endpoint, `--apply` must be refused — output the captured values for human confirmation instead. This is a hard constraint, not a suggestion.",
"fact": "If a site lacks a clean read-only endpoint, the `--apply` flag must be refused and captured credentials must be output for manual human confirmation instead.",
"rubric": {
"must_include": [
"The `--apply` flag must be refused or rejected",
"Captured values must be output for human confirmation or manual inspection"
],
"reference_answer": "If there is no clean read-only endpoint, the tool must refuse the `--apply` flag and output the captured credentials for human confirmation instead."
},
"excerpt_verbatim": true
},
{
"question": "While capturing headers from our web application, the intercepted API call included the authorization tokens but was missing the account identifier header. How should the tool handle this missing header?",
"excerpt": "The account/site identifier header may not appear on every request. Fall back to the existing value in the target file; do not treat its absence as a capture failure.",
"fact": "A missing account or site identifier header during capture should not cause a failure; the tool should fall back to the existing value in the target file.",
"rubric": {
"must_include": [
"Do not treat the missing header as a capture failure",
"Fall back to the existing value from the target file"
],
"reference_answer": "The tool should not treat the missing header as a capture failure; it should fall back to using the existing value stored in the target file."
},
"excerpt_verbatim": false
}
],
"cost": 0.0094935
},
"copy_workflow": {
"tasks": [
{
"question": "When budgeting spoken word counts for our deck script, what speaking rate should we plan for in terms of words per minute?",
"excerpt": "As for your word budget, plan for about 130 spoken words per minute.",
"fact": "The speech script word budget should be planned at approximately 130 spoken words per minute.",
"rubric": {
"must_include": [
"130 words per minute"
],
"reference_answer": "You should plan for approximately 130 spoken words per minute."
},
"excerpt_verbatim": true
},
{
"question": "When running the copy writer using the Antigravity CLI, the process returned a non-zero exit code and an error status. Why might this happen even if the generated output file is fully intact?",
"excerpt": "Keep in mind that seeing a non-zero exit code or an \"error\" status can happen simply because the writer's closing summary overflowed its output limit while the file itself is entirely complete, so always check the file itself, not only the status.",
"fact": "A non-zero exit code or error status can occur merely because the writer's closing summary exceeded its output limit while the target file is completely finished.",
"rubric": {
"must_include": [
"closing summary overflowed"
],
"reference_answer": "A non-zero exit code or error status can occur simply because the writer's closing summary overflowed its output limit even though the output file itself is completely generated."
},
"excerpt_verbatim": true
}
],
"cost": 0.00268875
},
"critic_review": {
"tasks": [
{
"question": "When setting up an automated sub-agent to evaluate revision rounds of a presentation deck, what materials is it permitted to inspect, and what must be strictly withheld from it?",
"excerpt": "It sees screenshots and the rubric only: no code, no reasoning.",
"fact": "The sub-agent is only allowed to see screenshots and the rubric, and must not see code or reasoning.",
"rubric": {
"must_include": [
"screenshots and the rubric",
"no code",
"no reasoning"
],
"reference_answer": "The sub-agent is permitted to see only screenshots and the rubric, while code and reasoning must be strictly withheld."
},
"excerpt_verbatim": true
},
{
"question": "When drafting the evaluation rubric for reviewing an animated presentation deck, how many criteria should be established?",
"excerpt": "Write `verification/critic/rubric.md` once per deck: lay out the target feel in the owner's words, the visual register, 6–8 criteria, and the genre's kitsch traps.",
"fact": "The deck rubric must specify between 6 and 8 criteria.",
"rubric": {
"must_include": [
"6–8"
],
"reference_answer": "The review rubric should lay out 6 to 8 criteria."
},
"excerpt_verbatim": true
}
],
"cost": 0.01665525
},
"generated_assets": {
"tasks": [
{
"question": "When preparing illustration plates for a presentation deck, what adjustment should you make if the background paper tone is inconsistent across different renders?",
"excerpt": "If the paper tone varies between renders, first divide each image by the median colour of its border so the paper becomes exactly white.",
"fact": "If the paper tone varies between renders, each image should be divided by the median color of its border to make the paper pure white.",
"rubric": {
"must_include": [
"divide each image",
"median color",
"border"
],
"reference_answer": "Divide each image by the median color of its border so that the background paper becomes pure white."
},
"excerpt_verbatim": true
},
{
"question": "Why is using CSS blend modes insufficient for integrating illustration plates into canvas decks, and what visual artifact occurs if an untreated plate is used instead of converting it to transparent ink?",
"excerpt": "That's because blend modes do not reach the page through a transformed world layer, so an untreated plate will show up as an unsightly white rectangle.",
"fact": "Blend modes do not pass through a transformed world layer to reach the page, causing untreated plates to render as unsightly white rectangles.",
"rubric": {
"must_include": [
"transformed world layer",
"white rectangle"
],
"reference_answer": "Blend modes cannot reach the page through a transformed world layer, causing an untreated plate to appear as an unsightly white rectangle."
},
"excerpt_verbatim": true
}
],
"cost": 0.008124
},
"html_decks": {
"tasks": [
{
"question": "I've added a custom clickable widget to one of my presentation frames, but tapping on it from an iPad advances the presentation instead of activating the widget. Standard links and buttons do not trigger navigation when tapped. How can I tell the engine not to trigger touch navigation on this custom element?",
"excerpt": "Links, buttons and form fields are skipped; mark any other interactive element `data-no-nav`.",
"fact": "Non-standard interactive elements must be marked with data-no-nav to prevent mobile touch navigation from capturing taps on them.",
"rubric": {
"must_include": [
"data-no-nav"
],
"reference_answer": "Add the attribute `data-no-nav` to the interactive element."
},
"excerpt_verbatim": true
},
{
"question": "On narrow screens such as portrait mobile phones, Reveal switches into scroll view, which bypasses the canvas presentation and breaks navigation. What specific configuration setting in Reveal.initialize prevents this?",
"excerpt": "Below 435 px wide, Reveal 5 switched to its scroll view and bypassed the canvas: `#/9` opened the title slide and navigation stopped working | Keep `scrollActivationWidth: null` in `Reveal.initialize` (the scaffold sets it)",
"fact": "Setting scrollActivationWidth to null in Reveal.initialize stops Reveal from switching to scroll view on narrow screens.",
"rubric": {
"must_include": [
"scrollActivationWidth",
"null"
],
"reference_answer": "Set `scrollActivationWidth: null` in `Reveal.initialize`."
},
"excerpt_verbatim": true
}
],
"cost": 0.01343325
},
"reference": {
"tasks": [
{
"question": "When using GPT Image 2.5 for slide generation with automatic variant selection, which specific model variant is chosen by default for 2K or 4K resolutions?",
"excerpt": "- `gpt-image-2.5-sunburst` — precision-first; default at 2K/4K.",
"fact": "The default model variant for 2K and 4K renders under GPT Image 2.5 is `gpt-image-2.5-sunburst`.",
"rubric": {
"must_include": [
"gpt-image-2.5-sunburst"
],
"reference_answer": "For 2K and 4K renders, the default model variant is `gpt-image-2.5-sunburst`."
},
"excerpt_verbatim": true
},
{
"question": "If I specify several image paths under the Asset section of a slide outline, how does the slide generation script handle and combine them before calling the image edit API?",
"excerpt": "When an outline slide lists multiple paths under `Asset`, `tools/generate_slides.py` passes them all to `client.images.edit` in list order. No stacking or compositing is performed.",
"fact": "The generator script passes all asset paths to client.images.edit in list order without performing any stacking or compositing.",
"rubric": {
"must_include": [
"in list order",
"no stacking or compositing"
],
"reference_answer": "The generation script passes all listed assets directly to `client.images.edit` in list order without performing any stacking or compositing."
},
"excerpt_verbatim": true
}
],
"cost": 0.010514250000000001
},
"skill_presentation": {
"tasks": [
{
"question": "When running the slide generator with the default auto variant setting, which GPT Image 2.5 model variant is selected for 1K drafts versus 2K or 4K renders?",
"excerpt": "The generator selects variants based on your target resolution: **1K → flare**, while **2K/4K → sunburst**. You can always override this by passing `--variant flare|sunburst` (leaving it as `auto` will follow the resolution rule).",
"fact": "When the variant parameter is set to auto, the slide generator chooses flare for 1K resolutions and sunburst for 2K or 4K resolutions.",
"rubric": {
"must_include": [
"flare for 1K",
"sunburst for 2K or 4K"
],
"reference_answer": "The generator automatically selects the flare variant for 1K resolution and the sunburst variant for 2K or 4K resolutions."
},
"excerpt_verbatim": true
},
{
"question": "A user wants a presentation created with no generative imagery at all. How should the rendering mode and asset settings be configured for this scaffold?",
"excerpt": "Keep in mind that your rendering mode and your asset policy are completely separate choices: an HTML deck typically takes advantage of generated illustration plates, so when a user asks for \"no image generation\", what you actually want is `html` configured with `assets: none`.",
"fact": "When a user requests no image generation, the deck should be set to HTML mode configured with assets: none.",
"rubric": {
"must_include": [
"html",
"assets: none"
],
"reference_answer": "The deck should be configured in html mode with assets set to none."
},
"excerpt_verbatim": true
}
],
"cost": 0.013253250000000001
},
"speaker_notes": {
"tasks": [
{
"question": "When drafting or editing the speaker notes for a presentation deck, what is the strict upper limit on the number of words allowed in an ordinary spoken sentence?",
"excerpt": "Keep ordinary sentences between 12-20 words. **Hard gate**: no ordinary spoken sentence may exceed 22 words.",
"fact": "An ordinary spoken sentence in the notes must never exceed 22 words.",
"rubric": {
"must_include": [
"22 words"
],
"reference_answer": "No ordinary spoken sentence may exceed 22 words."
},
"excerpt_verbatim": true
},
{
"question": "When calculating the time budget for our presentation script, what speaking pace in words per minute should be used, and what percentage of each segment needs to be reserved for demos, media, and pauses?",
"excerpt": "Plan at roughly 135 spoken words per minute, and reserve 10-15% of each segment for demos, clicks, video, and pauses.",
"fact": "Presentation timing should be budgeted at around 135 spoken words per minute, with 10-15% reserved for pauses, clicks, demos, and video.",
"rubric": {
"must_include": [
"135 spoken words per minute",
"10-15%"
],
"reference_answer": "Budget at roughly 135 spoken words per minute and reserve 10-15% of each segment for demos, clicks, video, and pauses."
},
"excerpt_verbatim": true
}
],
"cost": 0.0087075
},
"skill_resend_email": {
"tasks": [
{
"question": "I want to enable retries for sending emails via the CLI so it attempts to send up to 3 times on transient failures. What parameter is required if I set the maximum attempts above 1?",
"excerpt": "`--max-attempts` sets the total attempt count from 1 to 5 (default 1), counting the initial request. Setting `--max-attempts` above 1 requires `--idempotency-key`, including during dry runs.",
"fact": "Setting `--max-attempts` above 1 requires providing an `--idempotency-key`.",
"rubric": {
"must_include": [
"--idempotency-key",
"idempotency key"
],
"reference_answer": "Setting `--max-attempts` above 1 requires providing an `--idempotency-key`."
},
"excerpt_verbatim": true
},
{
"question": "I'm trying to retrieve received emails from a specific time window using `received list --after 2023-10-01T00:00:00Z`, but it returns a validation error. Why did this happen, and what does the `--after` flag actually expect?",
"excerpt": "`received list/export-all-md --after` and `--before` are Resend pagination cursors, not timestamp filters. Do not pass ISO timestamps to them; the API expects UUID cursor values and will return a validation error.",
"fact": "The `--after` and `--before` options in received email commands are pagination cursors expecting UUID values, not timestamps.",
"rubric": {
"must_include": [
"pagination cursor",
"UUID"
],
"reference_answer": "The `--after` flag is a pagination cursor that expects a UUID cursor value, not an ISO timestamp filter."
},
"excerpt_verbatim": true
}
],
"cost": 0.0035812500000000002
},
"skill_suppression_analysis": {
"tasks": [
{
"question": "When matching genuine bounce addresses against send-log recipients to detect typo-rescue pairs, what specific similarity ratio thresholds must be met using SequenceMatcher, and what status must the candidate rescued email have?",
"excerpt": "Compare bounce addresses against all send-log recipients using `difflib.SequenceMatcher` (same domain with local ratio > 0.8; or similar domain ratio > 0.92 with matched local). Confirm the close-spelling address has `last_event == \"delivered\"`.",
"fact": "Typo-rescue detection requires a local ratio > 0.8 for the same domain or domain ratio > 0.92 for matched local parts, and the rescued address must have last_event == 'delivered'.",
"rubric": {
"must_include": [
"local ratio > 0.8 (same domain)",
"domain ratio > 0.92 (matched local part)",
"last_event == \"delivered\""
],
"reference_answer": "The match requires either the same domain with a local part ratio greater than 0.8 or a matching local part with a domain ratio greater than 0.92, and the close-spelling address must have last_event == \"delivered\"."
},
"excerpt_verbatim": true
},
{
"question": "What are the three mandatory questions that must be answered in the conclusions section of our Resend suppression analysis report?",
"excerpt": "Every report must conclude by answering three mandatory questions:\n1. **Is there a systemic delivery problem?** (e.g., domain-wide blocks, authentication failures, broken DNS records).\n2. **Is anything urgent?** (e.g., new spam complaints, sudden volume surges, bounce rates exceeding thresholds).\n3. **Are there reset candidates?**",
"fact": "The conclusions section must answer whether there is a systemic delivery problem, whether anything is urgent, and whether there are reset candidates.",
"rubric": {
"must_include": [
"Is there a systemic delivery problem?",
"Is anything urgent?",
"Are there reset candidates?"
],
"reference_answer": "The three mandatory questions are: 'Is there a systemic delivery problem?', 'Is anything urgent?', and 'Are there reset candidates?'."
},
"excerpt_verbatim": true
}
],
"cost": 0.01382175
},
"skill_semantic_search": {
"tasks": [
{
"question": "I have a file list containing relative paths that I want to use with `semantic-search`. What command-line option can I use to define the base directory those paths should be resolved from, and what directory does the tool resolve them against if this option is omitted?",
"excerpt": "Relative entries resolve against `--source-root` when provided, otherwise against the CLI's current working directory — not the directory the file list was generated in.",
"fact": "Relative paths in a file list resolve against the CLI's current working directory unless `--source-root` is explicitly provided.",
"rubric": {
"must_include": [
"--source-root",
"current working directory"
],
"reference_answer": "Pass `--source-root` to specify the base directory; if omitted, relative paths resolve against the CLI's current working directory."
},
"excerpt_verbatim": true
},
{
"question": "When an agent creates an OpenCode deep link from semantic search results, where must the session ID be extracted from, and what prefix and character set must it have to be considered valid?",
"excerpt": "The `session_id` must come directly from the search result metadata/frontmatter. Never infer it from the filename, title, or body. If the ID is missing, does not start with `ses_`, or contains characters outside ASCII letters, digits, underscore, or hyphen, do not generate an action link.",
"fact": "A session ID must be retrieved directly from the result metadata or frontmatter, start with `ses_`, and consist only of ASCII letters, digits, underscores, or hyphens.",
"rubric": {
"must_include": [
"metadata or frontmatter",
"ses_",
"ASCII letters, digits, underscore, or hyphen"
],
"reference_answer": "The session ID must be extracted directly from the search result metadata/frontmatter, start with `ses_`, and contain only ASCII letters, digits, underscores, or hyphens."
},
"excerpt_verbatim": true
}
],
"cost": 0.01323525
},
"smart_home_skill": {
"tasks": [
{
"question": "I am writing an automation for our Samsung TV and want to inspect its current volume level and mute status. Can I retrieve these details using the smart home integration's status endpoint?",
"excerpt": "Volume and mute state are not readable (protocol limitation); `get_status` returns `PowerState` only.",
"fact": "Volume and mute states cannot be read from the Samsung TV due to protocol limitations, and get_status returns only PowerState.",
"rubric": {
"must_include": [
"volume and mute cannot be read (or are not readable)",
"only returns PowerState (or power state only)"
],
"reference_answer": "No, volume and mute state cannot be read due to protocol limitations; get_status returns PowerState only."
},
"excerpt_verbatim": true
},
{
"question": "When evaluating a camera verification run using the visual check tool, what specific details must be reported if the run returns a status of 'problem'?",
"excerpt": "If `status` is `problem` or an assertion failed, report the failed assertion and artifact path.",
"fact": "When a visual check results in a 'problem' status or failed assertion, the agent must report the failed assertion and the artifact path.",
"rubric": {
"must_include": [
"failed assertion",
"artifact path"
],
"reference_answer": "If the status is 'problem', you must report the failed assertion and the artifact path."
},
"excerpt_verbatim": true
}
],
"cost": 0.010922250000000001
},
"stripe_skill": {
"tasks": [
{
"question": "I want to run the live integration test suite against Stripe for this repository. Which environment variable do I need to set to enable them?",
"excerpt": "Live integration tests default to skip and require `STRIPE_ENABLE_LIVE_TESTS=1`.",
"fact": "Live integration tests default to skip and require setting the environment variable STRIPE_ENABLE_LIVE_TESTS=1.",
"rubric": {
"must_include": [
"STRIPE_ENABLE_LIVE_TESTS=1"
],
"reference_answer": "You must set `STRIPE_ENABLE_LIVE_TESTS=1` to enable live integration tests."
},
"excerpt_verbatim": true
},
{
"question": "When analyzing PaymentIntent sales to determine the settled USD amount, which exact field path on the PaymentIntent object should be checked?",
"excerpt": "For PaymentIntent sales analysis, use `latest_charge.balance_transaction.amount` for USD settled amount when available. Do not assume presentment currency equals settled currency.",
"fact": "For PaymentIntent sales analysis, the USD settled amount should be read from latest_charge.balance_transaction.amount when available.",
"rubric": {
"must_include": [
"latest_charge.balance_transaction.amount"
],
"reference_answer": "Use `latest_charge.balance_transaction.amount` when available for the USD settled amount."
},
"excerpt_verbatim": true
}
],
"cost": 0.002685
},
"skill_tavily": {
"tasks": [
{
"question": "When running the URL content extraction command, I want to restrict the output using the `--chunks-per-source` parameter. Which other command-line option must be provided for this parameter to work?",
"excerpt": "`--chunks-per-source` requires `--query` (the `extract` flag, not the `search` batch flag)",
"fact": "Using `--chunks-per-source` during extraction requires passing the `--query` flag.",
"rubric": {
"must_include": [
"--query"
],
"reference_answer": "The `--chunks-per-source` option requires passing the `--query` flag."
},
"excerpt_verbatim": true
},
{
"question": "When running a batch web search across multiple queries where only a subset of the queries fail while others succeed, what exit code does the command return?",
"excerpt": "Partial failure returns exit 0 with a stderr warning; if every query fails the exit code is 1. Usage errors exit 2.",
"fact": "In batch search mode, partial failures return an exit code of 0.",
"rubric": {
"must_include": [
"0"
],
"reference_answer": "The CLI exits with code 0 on partial failures."
},
"excerpt_verbatim": true
}
],
"cost": 0.012044250000000001
},
"tts_clone": {
"tasks": [
{
"question": "When sending requests to Gemini TTS for voice synthesis, how should the API request be configured to return audio, and is it valid to set AUDIO in the response_modalities parameter?",
"excerpt": "The `response_format` must be configured as `{\"type\": \"audio\"}`. Passing `AUDIO` in `response_modalities` will be rejected.",
"fact": "Gemini TTS requires response_format to be set to {\"type\": \"audio\"}, and passing AUDIO in response_modalities is rejected.",
"rubric": {
"must_include": [
"response_format",
"audio",
"rejected"
],
"reference_answer": "The request must configure response_format as {\"type\": \"audio\"}, because passing AUDIO in response_modalities will be rejected."
},
"excerpt_verbatim": true
},
{
"question": "If I enable server-side voice storage in Gemini voice replication by setting store=true, how long will the generated voice be retained, and what is the maximum number of voices permitted per project?",
"excerpt": "Specifying `store=true` returns a `voice_` id stored for 1 year. The cap is 200 voices per project, shared with prompted voices, and a stored voice can be deleted.",
"fact": "Setting store=true stores a voice ID for 1 year with a limit of 200 voices per project.",
"rubric": {
"must_include": [
"1 year",
"200"
],
"reference_answer": "Setting store=true retains the voice ID for 1 year, subject to a project cap of 200 voices."
},
"excerpt_verbatim": true
}
],
"cost": 0.013069500000000001
},
"skill_typefully_twitter": {
"tasks": [
{
"question": "If I run URL validation on a thread using the `post check-urls` command without passing the `--allowed-domain` option, what domain is permitted by default?",
"excerpt": "The default allowed domain is `example.com`; override with `--allowed-domain`.",
"fact": "The default allowed domain for `post check-urls` is `example.com` unless overridden by `--allowed-domain`.",
"rubric": {
"must_include": [
"example.com"
],
"reference_answer": "By default, `post check-urls` allows `example.com` unless overridden with `--allowed-domain`."
},
"excerpt_verbatim": true
},
{
"question": "Our environment has `TYPEFULLY_TWITTER_DEFAULT_LONG_POST=true` enabled, but I need to validate a tweet against regular character limits for a single command. What CLI flag should I include to enforce standard X length limits?",
"excerpt": "If `TYPEFULLY_TWITTER_DEFAULT_LONG_POST=true`, `post count`, `post draft`, and `post create` use long-post validation by default. Use `--no-long-post` only when the user explicitly wants normal X length limits for a command.",
"fact": "When default long-post validation is enabled, the `--no-long-post` flag must be passed to enforce normal X post length limits.",
"rubric": {
"must_include": [
"--no-long-post"
],
"reference_answer": "You should pass the `--no-long-post` flag to enforce normal X length limits when long-post validation is enabled by default."
},
"excerpt_verbatim": true
}
],
"cost": 0.011337000000000002
},
"skill_unifi": {
"tasks": [
{
"question": "I just adjusted my UniFi access point settings via the web interface. How long should I wait before capturing channel utilization and retry metrics to get an accurate baseline?",
"excerpt": "After any AP change made in the UI, CU/retries are distorted ~30 min\n (warm-up). Never baseline inside that window.",
"fact": "After an AP change in the UI, channel utilization and retries are distorted for approximately 30 minutes, during which baselining should not be done.",
"rubric": {
"must_include": [
"30 minutes (or ~30 min)",
"distorted or warm-up window"
],
"reference_answer": "You should wait approximately 30 minutes, as channel utilization and retries are distorted during this warm-up period after an AP change."
},
"excerpt_verbatim": true
},
{
"question": "When running `unifi-skill`, what specific exit code indicates an authentication failure caused by an expired cookie, and how should it be resolved?",
"excerpt": "Exit codes: **0** ok / **2** usage / **10** auth (cookie expired →\ntell the human to refresh, see docs/cookie_refresh.md; it is a human step)",
"fact": "Exit code 10 indicates authentication failure due to an expired cookie, which must be resolved by having the human refresh the cookie.",
"rubric": {
"must_include": [
"10",
"human refresh"
],
"reference_answer": "Exit code 10 indicates an expired session cookie, which requires a human to manually refresh it."
},
"excerpt_verbatim": true
}
],
"cost": 0.00217875
},
"bestpractice_external_prose": {
"tasks": [
{
"question": "When writing long-form technical articles, what is our guideline on how many specific numbers should be kept in a single paragraph, and how should any extra numbers be handled?",
"excerpt": "keep 1-2 numbers per paragraph (only ones whose deletion collapses the judgment), and handle the rest three ways — group them under one causal sentence, downgrade them to vague quantity (\"8xH100 for 8 hours\" → \"a few cards for one evening\"), or move them into a table/materials list.",
"fact": "Paragraphs should generally keep only 1 to 2 numbers, with remaining numbers either grouped under a causal sentence, downgraded to vague quantities, or moved to a table or materials list.",
"rubric": {
"must_include": [
"Limit numbers to 1-2 per paragraph (only keeping those whose deletion would collapse the judgment)",
"Handle excess numbers by grouping under a causal sentence, downgrading to a vague quantity, or moving to a table/materials list"
],
"reference_answer": "Retain only 1–2 critical numbers per paragraph and handle the remaining numbers by grouping them under a causal sentence, downgrading them to vague quantities, or moving them to a table or materials list."
},
"excerpt_verbatim": true
},
{
"question": "What are our guidelines for hyperlinking source URLs in an article, and should we include a dedicated sources or references section at the end?",
"excerpt": "link a source URL only at its first mention, and refer to it in words afterwards (\"the changelog\", \"the forum post\", \"the official docs\"); **do not create a separate references/sources section**—give every needed source inline.",
"fact": "A source URL should only be hyperlinked upon its first mention, referred to using descriptive words afterwards, and never compiled into a separate references or sources section.",
"rubric": {
"must_include": [
"Hyperlink the URL only at its first mention (and refer to it with words/text afterwards)",
"Do not create a separate references or sources section (provide sources inline)"
],
"reference_answer": "Link a source URL only on its first mention (referring to it in words thereafter), and do not create a separate references or sources section."
},
"excerpt_verbatim": true
}
],
"cost": 0.0177855
},
"external_prose_lint": {
"tasks": [
{
"question": "When scanning an article with the external prose linter, what specific condition triggers the `repeated_url` review finding?",
"excerpt": "| `repeated_url` | the same URL appears more than 2 times (a source gets an embedded link only at its first mention; no closing source list) | REVIEW |",
"fact": "The repeated_url review rule triggers when the same URL appears more than 2 times in the document.",
"rubric": {
"must_include": [
"more than 2 times",
"URL appears"
],
"reference_answer": "The `repeated_url` finding is triggered when the exact same URL appears more than 2 times in the draft."
},
"excerpt_verbatim": true
},
{
"question": "What are the exact paragraph threshold conditions that trigger a `number_density` warning when running the external prose linter?",
"excerpt": "Detection rule (WARNING level; does not block the exit code, but every hit must be resolved or justified):\n\n- a single prose paragraph containing >= 5 number tokens (Arabic numeral runs), or\n- >= 2 consecutive prose paragraphs each containing >= 3 number tokens.",
"fact": "A number_density warning is triggered if a single prose paragraph contains >= 5 number tokens, or if >= 2 consecutive prose paragraphs each contain >= 3 number tokens.",
"rubric": {
"must_include": [
"single prose paragraph containing >= 5 number tokens (or Arabic numeral runs)",
">= 2 consecutive prose paragraphs each containing >= 3 number tokens"
],
"reference_answer": "It triggers when a single prose paragraph contains 5 or more number tokens, or when 2 or more consecutive prose paragraphs each contain 3 or more number tokens."
},
"excerpt_verbatim": true
}
],
"cost": 0.0032812500000000003
},
"reference_writing_thesis_catalog": {
"tasks": [
{
"question": "When incorporating the L1-L8 analytical perspectives into a finished, public-facing article, what rule must be followed regarding their notation and naming?",
"excerpt": "Never expose L numbers or perspective labels in a reader-visible final article.",
"fact": "L numbers and perspective labels must never appear in reader-visible final articles.",
"rubric": {
"must_include": [
"do not expose or show L numbers",
"do not include perspective labels"
],
"reference_answer": "You must never include L numbers or perspective labels in a reader-visible final article."
},
"excerpt_verbatim": true
},
{
"question": "According to the thesis catalog, how many generations should you examine when tracing a technology's lineage to contextualize a new release?",
"excerpt": "Trace two to four generations, identifying the problem each generation solved and the limitation it left behind.",
"fact": "When tracing technology lineage, trace two to four generations.",
"rubric": {
"must_include": [
"two to four generations"
],
"reference_answer": "You should trace two to four generations, identifying what problem each generation resolved and what limitation it left behind."
},
"excerpt_verbatim": true
}
],
"cost": 0.00203625
},
"twitter_post_writing": {
"tasks": [
{
"question": "During the mechanical lint checks on our generated Twitter distribution post, how many consecutive single-sentence paragraphs are allowed before they are required to be merged?",
"excerpt": "| `quotes` / `bei_passive` / `single_sentence_paragraph` | REVIEW, each needs a keep-reason (≥4 consecutive single-sentence paragraphs get merged) |",
"fact": "Four or more consecutive single-sentence paragraphs must be merged during the mechanical review gate.",
"rubric": {
"must_include": [
"4 (or ≥4 / four or more) consecutive single-sentence paragraphs",
"merged"
],
"reference_answer": "When there are 4 or more consecutive single-sentence paragraphs, they must be merged."
},
"excerpt_verbatim": true
},
{
"question": "When a generated Twitter draft fails a review gate and needs to be rerun using the Antigravity CLI, can we resume the prior session using flags like `--continue` or `--conversation`?",
"excerpt": "Each generation and failure rerun starts a fresh Antigravity session (with `--new-project`), never `--continue` / `--conversation`.",
"fact": "Every generation and rerun must start a brand new Antigravity session with `--new-project`, never using `--continue` or `--conversation`.",
"rubric": {
"must_include": [
"must not use `--continue` or `--conversation`",
"must start a fresh session using `--new-project`"
],
"reference_answer": "No; every generation and failure rerun must start a completely fresh session with `--new-project`, and flags like `--continue` or `--conversation` are prohibited."
},
"excerpt_verbatim": true
}
],
"cost": 0.011686499999999999
},
"workflow_external_writing": {
"tasks": [
{
"question": "During the mechanical prose linting phase for our external analytical articles, what specific thresholds trigger a warning for numerical cognitive load?",
"excerpt": "numerical cognitive load (WARNING: triggered by ≥5 in a single paragraph or ≥3 in two consecutive paragraphs)",
"fact": "A numerical cognitive load warning is triggered when a single paragraph contains 5 or more numbers or two consecutive paragraphs contain 3 or more numbers.",
"rubric": {
"must_include": [
"5 or more (≥5) numbers in a single paragraph",
"3 or more (≥3) numbers in two consecutive paragraphs"
],
"reference_answer": "A numerical cognitive load warning is triggered by having 5 or more numbers in a single paragraph or 3 or more numbers in two consecutive paragraphs."
},
"excerpt_verbatim": true
},
{
"question": "We are preparing visual illustrations to accompany an analytical piece. What restrictions are placed on including text and sentences within the generated images?",
"excerpt": "compress text to the absolute minimum (2-6 word short labels or bare numbers; full sentences never enter images).",
"fact": "Text in illustrations must be compressed to 2–6 word short labels or bare numbers, and full sentences must never appear in images.",
"rubric": {
"must_include": [
"short labels of 2 to 6 words or bare numbers",
"full sentences must never appear in images"
],
"reference_answer": "Text inside images must be limited to 2–6 word short labels or bare numbers, and full sentences are strictly forbidden."
},
"excerpt_verbatim": true
}
],
"cost": 0.013890000000000001
},
"workflow_internal_writing": {
"tasks": [
{
"question": "I am working on an extensive internal memo and want to draft it in sections to keep my concept dependencies and ordering consistent. How many headings at most should I draft in a single batch?",
"excerpt": "For longer documents, write in batches by heading, with at most 4 headings per batch.",
"fact": "Long documents should be drafted in batches by heading with a maximum of 4 headings per batch.",
"rubric": {
"must_include": [
"at most 4 headings"
],
"reference_answer": "You should write in batches with at most 4 headings per batch."
},
"excerpt_verbatim": true
},
{
"question": "I'm adding an architectural diagram to an internal project document and was planning to embed it as an inline SVG so it scales well. Is this permitted, and if not, what is the reason and the preferred alternative?",
"excerpt": "prefer PNG/JPG/WebP images. **Do not use inline SVG** (to avoid mobile rendering incompatibilities).",
"fact": "Inline SVGs must not be used for diagrams to avoid mobile rendering incompatibilities, and PNG, JPG, or WebP images should be used instead.",
"rubric": {
"must_include": [
"inline SVGs must not be used (or are not permitted)",
"mobile rendering incompatibilities",
"PNG, JPG, or WebP"
],
"reference_answer": "Inline SVGs are not permitted because of mobile rendering incompatibilities; instead, PNG, JPG, or WebP formats are preferred."
},
"excerpt_verbatim": true
}
],
"cost": 0.0110325
},
"workflow_news_aggregation": {
"tasks": [
{
"question": "I need to turn a completed research item into an information packet for today's briefing, but looking in `tmp/`, there isn't a folder for today's date yet. Can you go ahead and create the folder so we can proceed?",
"excerpt": "Never create a folder. If no folder exists for the target date, stop and ask the user which folder to use. Creating one requires an explicit instruction.",
"fact": "An aggregation folder must never be created automatically; if none exists for the target date, the agent must stop and ask the user which folder to use.",
"rubric": {
"must_include": [
"do not create the folder automatically",
"stop and ask the user which folder to use"
],
"reference_answer": "No, you must not create the folder automatically; you must stop and ask the user which folder to use unless explicitly instructed to create one."
},
"excerpt_verbatim": true
},
{
"question": "While planning the outline for our multi-item briefing, we noticed there is a genuine common thread connecting all the items this week. How should we present this structure in the outline proposal?",
"excerpt": "When a real common thread exists, offer it as an option with the stacked structure as the default, not the reverse.",
"fact": "Even when a legitimate common thread exists across items, the outline must offer the stacked structure as the default and the themed thread only as an option.",
"rubric": {
"must_include": [
"stacked structure as the default",
"offer the common thread as an option"
],
"reference_answer": "Offer the common thread as an option while presenting the stacked structure as the default."
},
"excerpt_verbatim": true
}
],
"cost": 0.01105575
},
"writing_workflows": {
"tasks": [
{
"question": "When routing a writing task where it is not clear whether the audience shares project context or not, what question should be asked to the user?",
"excerpt": "ask the user one question: \"who is the reader, and do they already know this project?\" Do not guess.",
"fact": "When audience context is ambiguous, the agent must ask: 'who is the reader, and do they already know this project?' without guessing.",
"rubric": {
"must_include": [
"who is the reader",
"do they already know this project"
],
"reference_answer": "You should ask the user one specific question without guessing: \"who is the reader, and do they already know this project?\""
},
"excerpt_verbatim": true
},
{
"question": "What is the rule regarding the `bestpractice_external_prose.md` file when working with the writer context during external drafting?",
"excerpt": "Diagnosing why a candidate draft sounds like a textbook or is cognitively overloaded; writing the `voice_contract.md` | Main Agent only. Never paste into the writer context. Not a gate checklist.",
"fact": "`bestpractice_external_prose.md` is strictly for the Main Agent and must never be pasted into the writer context.",
"rubric": {
"must_include": [
"Main Agent only",
"never paste into the writer context"
],
"reference_answer": "`bestpractice_external_prose.md` is strictly for the Main Agent and must never be pasted into the writer context."
},
"excerpt_verbatim": true
}
],
"cost": 0.0031875000000000002
},
"zun_music": {
"tasks": [
{
"question": "I want to arrange a traditional folk tune that is originally in 3/4 time. Can I keep the arrangement in 3/4, or how does the system require non-duple meters to be handled?",
"excerpt": "4/4 only. `check()` rejects other meters, so convert triple-meter sources such as Greensleeves to 4/4 as part of the arrangement.",
"fact": "Only 4/4 time is supported and other meters are rejected by check(), requiring triple-meter source melodies to be converted to 4/4.",
"rubric": {
"must_include": [
"4/4 only",
"convert to 4/4"
],
"reference_answer": "The system only supports 4/4 and rejects other meters, so triple-meter sources must be converted to 4/4 as part of the arrangement."
},
"excerpt_verbatim": true
},
{
"question": "I'm customizing a melody line and need to deviate from the source notes on some of the strong beats. What is the limit on how many deliberate note deviations I can register in skeleton_allow without failing acceptance testing?",
"excerpt": "Every deliberate departure is listed in `skeleton_allow`, capped at one per four bars; normally it is only the final cadence.",
"fact": "Departures from the source melody in skeleton_allow are capped at one per four bars.",
"rubric": {
"must_include": [
"one per four bars"
],
"reference_answer": "Deliberate departures listed in skeleton_allow are strictly capped at one per four bars."
},
"excerpt_verbatim": true
}
],
"cost": 0.010794
},
"send_email": {
"tasks": [
{
"question": "When executing an actual email delivery via the Resend CLI rather than a simulation, what specific command-line flag must be passed to execute the real send?",
"excerpt": "Real sends require explicit user intent and the Resend CLI `--confirm-send` flag. Always do a dry run first unless the current user message already includes clear authorization to send for real.",
"fact": "Executing a real email send with the Resend CLI requires specifying the --confirm-send flag.",
"rubric": {
"must_include": [
"--confirm-send"
],
"reference_answer": "The Resend CLI requires the `--confirm-send` flag to perform an actual email send."
},
"excerpt_verbatim": true
},
{
"question": "If our workspace is set up to route email notifications to Pushover, should automated pipeline result emails and test emails also CC Pushover?",
"excerpt": "Unified Email / unified client outputs are an explicit exception to Pushover notification behavior. Self-thread notes, test emails, and local pipeline result emails should go only to the configured mailbox recipient and should not CC Pushover.",
"fact": "Pipeline results and test emails are an exception to Pushover notification behavior and must not CC Pushover.",
"rubric": {
"must_include": [
"not CC Pushover"
],
"reference_answer": "No, test emails and local pipeline result emails must not CC Pushover and should go only to the configured mailbox recipient."
},
"excerpt_verbatim": true
}
],
"cost": 0.0096855
},
"workflow_presentation_slides": {
"tasks": [
{
"question": "I'm defining the visual guidelines for our AI-generated slide deck and planning where our character mascot should appear. On which specific slides is the mascot allowed to be placed?",
"excerpt": "- Mascot: (title slide and closing slide only)",
"fact": "The mascot is only permitted to appear on the title slide and the closing slide.",
"rubric": {
"must_include": [
"title slide",
"closing slide"
],
"reference_answer": "The mascot is only allowed on the title slide and the closing slide."
},
"excerpt_verbatim": true
},
{
"question": "We are designing slides intended to function as both a live presentation and a standalone handout. What is the recommended balance ratio between visual illustrations and readable text?",
"excerpt": "- **Target ratio**: ~40% illustration / ~60% readable text",
"fact": "The recommended layout balance is approximately 40% illustration and 60% readable text.",
"rubric": {
"must_include": [
"40% illustration",
"60% readable text"
],
"reference_answer": "The target ratio is roughly 40% illustration to 60% readable text."
},
"excerpt_verbatim": true
}
],
"cost": 0.0087075
},
"apple_calendar": {
"tasks": [
{
"question": "I want to query my Mac calendar events by directly copying Calendar.sqlitedb from my Library folder, but my script returns empty results every time. Which path should I actually copy from, and why did the file under ~/Library/Calendars fail?",
"excerpt": "Live store: `~/Library/Group Containers/group.com.apple.calendar/Calendar.sqlitedb`\n (EventKit store). `-wal` / `-shm` sidecars hold uncheckpointed writes (P4).\n- **DECOY:** `~/Library/Calendars/Calendar.sqlitedb` is a 0-byte file on\n current macOS (P2). Do not use it.",
"fact": "The file at ~/Library/Calendars/Calendar.sqlitedb is a 0-byte decoy on current macOS, whereas the real live database is located at ~/Library/Group Containers/group.com.apple.calendar/Calendar.sqlitedb.",
"rubric": {
"must_include": [
"~/Library/Group Containers/group.com.apple.calendar/Calendar.sqlitedb",
"0-byte"
],
"reference_answer": "You must use `~/Library/Group Containers/group.com.apple.calendar/Calendar.sqlitedb` because `~/Library/Calendars/Calendar.sqlitedb` is merely a 0-byte decoy file on current macOS."
},
"excerpt_verbatim": true
},
{
"question": "When querying calendar events directly from the macOS SQLite store, how are the timestamps in start_date and end_date encoded, and what epoch do they measure from?",
"excerpt": "`start_date` / `end_date` are **absolute instants**: seconds since\n `2001-01-01 00:00:00 UTC` (the NSDate reference). Not wall-clock values.",
"fact": "In the macOS Calendar SQLite store, start_date and end_date are absolute instants measured in seconds since 2001-01-01 00:00:00 UTC (the NSDate reference).",
"rubric": {
"must_include": [
"seconds",
"2001-01-01 00:00:00 UTC"
],
"reference_answer": "They are absolute instants represented as seconds elapsed since 2001-01-01 00:00:00 UTC (the NSDate reference epoch)."
},
"excerpt_verbatim": true
}
],
"cost": 0.00439125
},
"colab": {
"tasks": [
{
"question": "I accidentally passed a typo in the accelerator name when invoking the Google Colab CLI (version 0.7.4). Instead of aborting with an invalid argument error, what does the CLI actually do with an unrecognized `--gpu` value?",
"excerpt": "On 0.7.4 an unrecognized `--gpu` value is sent as A100, not rejected.",
"fact": "In Google Colab CLI version 0.7.4, passing an unrecognized `--gpu` value does not reject the request but sends it as A100.",
"rubric": {
"must_include": [
"sent as A100",
"not rejected"
],
"reference_answer": "On CLI version 0.7.4, an unrecognized `--gpu` parameter is not rejected and is instead sent as A100."
},
"excerpt_verbatim": true
},
{
"question": "If I execute `colab stop` targeting a session name that does not exist locally, what does the CLI output and what exit status or result does it return?",
"excerpt": "`colab stop` with a missing local name prints \"not found\" and returns success; that is not a release.",
"fact": "Running `colab stop` with a missing local session name prints \"not found\" and returns success.",
"rubric": {
"must_include": [
"prints \"not found\"",
"returns success (or zero exit status)"
],
"reference_answer": "The command prints \"not found\" and returns success."
},
"excerpt_verbatim": true
}
],
"cost": 0.015539250000000001
},
"ai_session_export": {
"tasks": [
{
"question": "I am recovering missing cloud transcript messages for a truncated Claude Code Remote session to backfill into my exported archives. When saving the extracted messages into the backfill folder as a JSONL file, should the filename be based on the local session ID or the cloud session ID?",
"excerpt": "One JSON object per line, named by the LOCAL\n session id: `/.jsonl`.",
"fact": "Claude Code backfill JSONL files must be named using the local session ID rather than the cloud session ID.",
"rubric": {
"must_include": [
"local session id"
],
"reference_answer": "The backfill JSONL file must be named using the local session ID."
},
"excerpt_verbatim": true
},
{
"question": "I am setting up my environment to export coding sessions from DeepSeek Harness. Are there any external system command-line tools or binaries required on my PATH beyond the standard Python library?",
"excerpt": "The DeepSeek Harness source needs the `zstd` binary on PATH for compressed session logs",
"fact": "DeepSeek Harness requires the zstd binary on PATH to handle compressed session logs.",
"rubric": {
"must_include": [
"zstd"
],
"reference_answer": "The DeepSeek Harness source requires the zstd binary on your PATH to handle compressed session logs."
},
"excerpt_verbatim": true
}
],
"cost": 0.01289175
},
"docx": {
"tasks": [
{
"question": "When filling in form values such as full names or phone numbers on an underline-based Word form, what kind of space should be used between words or numbers to prevent unwanted line wrapping?",
"excerpt": "Use non-breaking spaces inside values such as names, phone numbers, or relationship labels if a line break would be unacceptable.",
"fact": "Non-breaking spaces should be used inside values such as names or phone numbers to prevent unacceptable line breaks.",
"rubric": {
"must_include": [
"non-breaking space"
],
"reference_answer": "Use non-breaking spaces inside values like names and phone numbers to ensure they do not split across a line break."
},
"excerpt_verbatim": true
},
{
"question": "When completing a form-filling edit on a Word document, what specific verification artifact path are you expected to report back to confirm visual layout and rendering?",
"excerpt": "When returning a completed edit, report:\n\n- Output DOCX path.\n- Verification artifact path, usually a rendered PDF.\n- Which paragraphs/tables/runs were modified.\n- Whether the edit was layout-verified by rendering.\n- Any residual risks, such as LibreOffice/Word rendering differences.",
"fact": "Agents must report a verification artifact path, typically a rendered PDF, when returning a completed edit.",
"rubric": {
"must_include": [
"rendered PDF"
],
"reference_answer": "You should report the path to a verification artifact, typically a rendered PDF of the document."
},
"excerpt_verbatim": true
}
],
"cost": 0.009675
},
"skill_google_docs": {
"tasks": [
{
"question": "I am trying to export some of my locally cached Gmail messages into Markdown, but I want to save them to an external directory like `/tmp/my_emails`. The CLI seems to be rejecting this path. What specific flag do I need to pass to allow exporting outside of the default mail directory?",
"excerpt": "`export-md --output-dir` refuses paths outside `data/mail/` unless `--unsafe-output-dir` is set. Treat exported Markdown as private email data.",
"fact": "The export-md command refuses output directories outside data/mail/ unless the --unsafe-output-dir flag is passed.",
"rubric": {
"must_include": [
"--unsafe-output-dir"
],
"reference_answer": "You need to pass the `--unsafe-output-dir` flag to allow exporting outside of `data/mail/`."
},
"excerpt_verbatim": true
},
{
"question": "When downloading a folder from Google Drive, how does the CLI resolve file name collisions if two items end up with the same case-insensitive name?",
"excerpt": "Path separators and invalid characters are sanitized;\n collisions are resolved case-insensitively by appending `__`.",
"fact": "Name collisions during Drive downloads are resolved case-insensitively by appending `__` to the file or directory name.",
"rubric": {
"must_include": [
"__",
"appending"
],
"reference_answer": "Name collisions are resolved case-insensitively by appending `__` to the name."
},
"excerpt_verbatim": true
}
],
"cost": 0.006533250000000001
},
"tiff_icc_profile": {
"tasks": [
{
"question": "I exported a still from DaVinci Resolve with the project output color space set to P3-D65 ST.2084, but it was mistakenly tagged with a BT.2100 (Rec.2020) profile. What visual artifact will this create on skin tones, and what is the technical reason for it?",
"excerpt": "Common mistake: tagging **BT.2100 (Rec.2020)** on a **P3-D65 ST.2084** export causes red / sunburn skin tones. These share PQ transfer function but **different primaries**.",
"fact": "Tagging a P3-D65 ST.2084 image with a BT.2100 profile causes red or sunburned skin tones because the profiles share a PQ transfer function but have different primaries.",
"rubric": {
"must_include": [
"red or sunburn skin tones",
"different primaries (despite sharing the PQ transfer function)"
],
"reference_answer": "It causes red or sunburned skin tones because although both spaces share the PQ transfer function, they utilize different color primaries."
},
"excerpt_verbatim": true
},
{
"question": "When exporting a graded still from DaVinci Resolve by right-clicking in the Gallery and choosing Export TIFF, what bit depth is produced in practice, and why is this inadequate for an HDR master?",
"excerpt": "- Gallery still export is **8-bit** in practice (~100 MB for ~37 MP RGB vs ~220 MB for 16-bit Deliver).\n- Pixels are still encoded for the project output space, but the container loses bit depth and often lacks a usable ICC tag.",
"fact": "Gallery Grab Still TIFF exports are 8-bit in practice, losing bit depth and often lacking a usable ICC profile.",
"rubric": {
"must_include": [
"8-bit",
"loses bit depth or lacks a usable ICC tag"
],
"reference_answer": "In practice, exporting a still from the Gallery yields only an 8-bit file, meaning the container loses bit depth and often lacks a usable ICC profile."
},
"excerpt_verbatim": true
}
],
"cost": 0.01838175
},
"skill_smtp": {
"tasks": [
{
"question": "When sending an email from a custom domain via iCloud, the SMTP server responds with '250 queued' and no bounce error occurs, but the message never reaches the recipient. What causes this silent drop, and how should the envelope address and From header be configured to resolve it?",
"excerpt": "1. **Custom-domain envelope drop** \n Symptom: SMTP returns `250 queued`, no bounce, message never arrives. \n Cause: `MAIL FROM` used the custom-domain alias. \n Rule: authenticate and envelope as the primary iCloud mailbox; custom domain only in the From header. Library does this for `SMTP_PROVIDER=icloud`.",
"fact": "On iCloud, using a custom-domain alias in MAIL FROM causes messages to be silently dropped despite a 250 queued response; the envelope sender must be the primary iCloud mailbox while the custom domain belongs only in the From header.",
"rubric": {
"must_include": [
"The drop occurs because the envelope sender (MAIL FROM) was set to the custom-domain alias",
"The envelope sender and authentication must use the primary iCloud mailbox (@icloud.com, @me.com, or @mac.com)",
"The custom-domain address must only be placed in the visible From header"
],
"reference_answer": "The message is silently dropped because the custom-domain alias was used as the envelope sender (MAIL FROM); to fix it, authenticate and set the envelope sender to the primary iCloud mailbox while placing the custom-domain address only in the From header."
},
"excerpt_verbatim": true
},
{
"question": "When using the CLI to create a draft message on the IMAP server instead of performing an offline dry-run preview, what exact command-line flag is required to authorize creating the draft?",
"excerpt": "4. Any real draft used `draft create --confirm` after user authorization.",
"fact": "Creating a real draft on the IMAP server via the CLI requires the `--confirm` flag.",
"rubric": {
"must_include": [
"--confirm"
],
"reference_answer": "The `--confirm` flag must be passed to `draft create` to create a real draft on the server."
},
"excerpt_verbatim": true
}
],
"cost": 0.020838
},
"superlinear-brand": {
"tasks": [
{
"question": "I am styling some green text in a Superlinear Academy layout that is smaller than 24 px. Normal brand green is #238343, but what specific hex code should be used for small text to satisfy contrast requirements?",
"excerpt": "Green text under 24 px uses `#22683b` (contrast).",
"fact": "Brand green text smaller than 24 px must use #22683b for contrast instead of standard brand green.",
"rubric": {
"must_include": [
"#22683b"
],
"reference_answer": "Green text under 24 px must use the hex code `#22683b`."
},
"excerpt_verbatim": true
},
{
"question": "When running the render and audit script for a Superlinear Academy presentation deck prior to shipping, which command-line flag should be passed so the audit fails if any asset placeholders remain?",
"excerpt": "(add `--final` for the version you ship: it fails on placeholders).",
"fact": "Passing the `--final` flag causes the render script to fail if any placeholders are present.",
"rubric": {
"must_include": [
"--final"
],
"reference_answer": "Pass the `--final` flag when running the render script to fail if any placeholders are still present."
},
"excerpt_verbatim": true
}
],
"cost": 0.01479
},
"track17": {
"tasks": [
{
"question": "When fetching tracking details for a new package, should I rely on the `get` command's `--auto-register` flag to handle unregistered numbers, or is there a different recommended practice?",
"excerpt": "`--auto-register` is intended to auto-register unregistered numbers, but in a live run (2026-09-25) the API still returned `-18019902`; prefer an explicit `register` first",
"fact": "The `--auto-register` flag on the `get` command can still result in error `-18019902`, so running an explicit `register` command first is preferred.",
"rubric": {
"must_include": [
"prefer an explicit register command first",
"returns error code -18019902"
],
"reference_answer": "You should run an explicit `register` command first because using `--auto-register` can still fail with upstream API error `-18019902`."
},
"excerpt_verbatim": true
},
{
"question": "If our automated workflow accidentally submits a registration request for a shipment that was already registered earlier, will it consume our tracking quota or trigger an error exit code?",
"excerpt": "Calling `register` on a new tracking number consumes 1 quota unit. Re-registering an already-registered number returns API code `-18019901` and consumes zero quota (handled as exit code `0`).",
"fact": "Re-registering an existing tracking number consumes zero quota and exits with code 0.",
"rubric": {
"must_include": [
"consumes zero quota (or does not consume quota)",
"exit code 0 (or treated as success / no-op)"
],
"reference_answer": "It will consume zero quota and complete successfully with an exit code of 0 as an idempotent no-op."
},
"excerpt_verbatim": true
}
],
"cost": 0.01163175
},
"chinese_visa_photo": {
"tasks": [
{
"question": "When running the Chinese visa photo CLI, which operating mode should I select if I want the command to generate all output artifacts and exit successfully even if active compliance checks fail?",
"excerpt": "Use `diagnostic` mode for review loops; it writes the same artifacts and exits successfully even when active checks fail, so an AI agent can inspect the JSON and diagnostic image before rerunning with targeted adjustments.",
"fact": "diagnostic mode writes all output artifacts and exits with success even when active validation checks fail.",
"rubric": {
"must_include": [
"diagnostic"
],
"reference_answer": "Use `diagnostic` mode (via `--mode diagnostic`), which writes all output artifacts and exits successfully even when active checks fail."
},
"excerpt_verbatim": true
},
{
"question": "What is the minimum allowed chin-to-bottom distance requirement verified by the active checks in the Chinese visa photo CLI?",
"excerpt": "Chin-to-bottom is an active check and must be at least the pixel equivalent of 7mm.",
"fact": "The chin-to-bottom measurement must be at least the pixel equivalent of 7mm.",
"rubric": {
"must_include": [
"7mm",
"pixel equivalent of 7mm"
],
"reference_answer": "The chin-to-bottom distance must be at least the pixel equivalent of 7mm."
},
"excerpt_verbatim": true
}
],
"cost": 0.002499
},
"gpt-live": {
"tasks": [
{
"question": "In our GPT-Live client-delegation integration, our headless automated test client stopped sending audio input after the user's speech finished, but now the backend result acknowledgment (`session.commentary.appended`) is hanging indefinitely. What is causing this append acknowledgment to stay pending?",
"excerpt": "The acknowledgment waits until the session's frame timeline reaches the estimated end of context injection. If no audio frames are flowing (e.g., a headless client stopped sending input after the user's utterance), the timeline freezes and the ack stays pending indefinitely.",
"fact": "Append acknowledgments freeze and hang indefinitely if incoming audio frames stop flowing because frame progress is required to advance the session's frame timeline to the end of context injection.",
"rubric": {
"must_include": [
"timeline freezes or stops advancing because no audio frames are flowing",
"session frame progress"
],
"reference_answer": "Append acknowledgments require audio frame progress to advance the session timeline to the end of context injection, so stopping audio input freezes the timeline and leaves the acknowledgment pending indefinitely."
},
"excerpt_verbatim": true
},
{
"question": "We are implementing Responses delegation mode in GPT-Live. When our application handles a function call, we submit the tool output via `response.item.create`, but the backend never finishes or speaks the final answer. What specific step must be executed immediately after submitting the output?",
"excerpt": "After executing the authorized operation:\n 1. `response.item.create` with an item `{type: \"function_call_output\", call_id, output: \"\"}`.\n 2. Explicitly `response.create` to continue the backend response.\n- Submitting a result does not continue the response.",
"fact": "In Responses delegation mode, submitting a tool result via `response.item.create` does not continue the backend response; an explicit `response.create` must follow.",
"rubric": {
"must_include": [
"`response.create`",
"continue the response"
],
"reference_answer": "You must explicitly send a `response.create` message immediately after `response.item.create`, as submitting output does not automatically continue the backend response."
},
"excerpt_verbatim": true
}
],
"cost": 0.0056932499999999995
}
}