{
  "title": "Instrument assistant: captured software exercise and study notes",
  "date": "2026-10-24",
  "scope": "Project-authored exercise guides and public software code. No manufacturer manual, physical instrument or customer incident was tested.",
  "method": {
    "primary": "Three development questions, then twelve frozen questions: eight model-supported tasks and four deterministic routing checks. One attempt per model; no expected error names supplied to retrieval.",
    "retrieval": "Expected passage present for 7 of 8 supported questions with lexical retrieval, and 8 of 8 with BGE-small hybrid retrieval. The primary generation comparison retained the preselected lexical packets.",
    "assessment": "Unblinded rubric review of task completion, separate from structural validation. No independent domain-expert review or field accuracy estimate.",
    "primary_results": {
      "local_full_tasks": 3,
      "hosted_full_tasks": 7,
      "supported_task_count": 8,
      "excerpt_baseline_tasks_with_required_evidence": 7,
      "structurally_valid_outputs_per_model": 8,
      "backend_routing_checks": 4,
      "backend_checks_in_model_denominator": false
    },
    "lookup": "A separately frozen, post-hoc narrower task on the same tiny corpus: six new answerable questions and two absent-answer questions, hybrid retrieval, concise answer or clarification plus exact quotation. Different questions and contract prevent a causal comparison with primary task scores.",
    "lookup_results": {
      "local_supported_answers": 6,
      "hosted_supported_answers": 6,
      "answerable_count": 6,
      "local_appropriate_clarifications": 1,
      "hosted_appropriate_clarifications": 2,
      "absent_answer_count": 2,
      "inference_calls_per_model": 8,
      "structurally_valid_outputs_per_model": 8
    },
    "models": {
      "local": "IBM Granite 4.2 3B, Ollama granite4.2:3b, Q4_K_M, CPU-local",
      "hosted": "Claude Opus 5.5 through CLI"
    },
    "baseline": "Exact excerpts already contained all six answerable lookup facts. Neither a need for generation nor time savings was demonstrated."
  },
  "captured_examples": [
    {
      "question": "Give the guide's exact sentence defining what the volume ledger records.",
      "local_answer": "The volume ledger records a software estimate, not a measurement of liquid in a vessel.",
      "exact_quote": "The volume ledger records a software estimate, not a measurement of liquid in a vessel.",
      "source": {
        "kind": "QPillars-authored exercise guide",
        "revision": "2.0",
        "section": "Tracked liquid and preflight"
      },
      "checks": {
        "quote_in_source": true,
        "applicable_revision": true,
        "answer_supported_in_rubric_review": true
      }
    },
    {
      "question": "Which API call verifies that a physical tip is securely seated after pickup?",
      "local_answer": "The rollback method verifies that a physical tip is securely seated after pickup.",
      "exact_quote": "assert not self.is_disabled, \"Tip tracker is disabled. Call `enable()`.\"",
      "source": {
        "kind": "Public software code",
        "package": "PyLabRobot 0.2.2",
        "function": "TipTracker.rollback",
        "distribution": "https://pypi.org/project/pylabrobot/0.2.2/"
      },
      "checks": {
        "quote_in_source": true,
        "applicable_revision": true,
        "answer_supported_in_rubric_review": false
      },
      "assessment": "The quoted software assertion does not verify physical seating. The model should have requested missing evidence. A correct quote and valid source reference did not establish entailment."
    }
  ],
  "limits": [
    "Small authored corpus and selected tasks; no unseen-instrument generalization.",
    "Guide revisions 1.0 and 2.0 are authored exercise revisions, not a manufacturer release claim.",
    "Recorded software outcomes are supplied by application code, not established by a model.",
    "A selected one-question hybrid retrieval follow-up is not a new held-out accuracy score.",
    "No hardware recovery, support-time savings or production readiness was established."
  ]
}