{
  "schema": "grex-public-experiment-summary/v1",
  "slug": "coding-team-comparison",
  "updated": "2026-11-04",
  "experimentDate": "2026-10-15",
  "status": "Completed · diagnostic case uncertain",
  "scope": "Sanitized aggregate transcription of retained source records, not a raw-data release or independent reproduction. Private operational identifiers, source bodies and traces are excluded.",
  "question": "Does adding a planning agent before the same coding worker improve small, well-specified code changes?",
  "hypothesis": {
    "statement": "A separate planner might improve correctness enough to justify its added execution time.",
    "provenance": "Retrospective directional interpretation. The pre-run pair plan explicitly changed planner presence while fixing the worker, reviewer and task contract; no numerical improvement prediction is claimed."
  },
  "sample": "Three task pairs, six submissions: HTTP retry-delay parsing, per-key request coalescing and exact-output diagnostic refactoring. One pair per task.",
  "method": "Create two missions for each isolated code fixture: a coding worker alone, and a planner followed by that same worker. Keep the reviewer and acceptance contract fixed. Reassess exact retained submissions with an independent deterministic evaluator.",
  "comparison": {
    "columns": [
      "Measure",
      "Worker only",
      "Planner + same worker"
    ],
    "rows": [
      [
        "Retry-delay parsing",
        "10 / 10 cases",
        "10 / 10 cases"
      ],
      [
        "Request coalescing",
        "7 / 7 cases",
        "7 / 7 cases"
      ],
      [
        "Diagnostic refactor, selected cases",
        "5 / 5 cases",
        "5 / 5 cases"
      ],
      [
        "Retry-delay elapsed time",
        "40.288 seconds",
        "48.948 seconds"
      ],
      [
        "Request-coalescing elapsed time",
        "29.671 seconds",
        "38.835 seconds"
      ],
      [
        "Diagnostic elapsed time",
        "40.517 seconds",
        "49.136 seconds"
      ],
      [
        "Model review acceptance",
        "3 / 3",
        "3 / 3"
      ]
    ]
  },
  "findings": [
    {
      "title": "No measured benefit on the selected cases",
      "body": "Both arms passed ten retry-delay cases, seven request-coalescing cases and five selected diagnostic cases. The planner added 8.660, 9.164 and 8.619 seconds respectively."
    },
    {
      "title": "Both gates missed an edge case",
      "body": "The diagnostic outputs inherited a lookup behavior that treated certain object-prototype keys as known severity levels. The broad requirement called for unknown levels to fall back. Neither the selected cases nor the model reviewer caught this."
    },
    {
      "title": "Review acceptance did not settle correctness",
      "body": "All six model reviews accepted. The later diagnostic finding left operator assessments uncertain. Passing a bounded evaluator and receiving an approving review did not establish the whole requirement."
    },
    {
      "title": "Setup failures were retained",
      "body": "The retry-delay baseline required a corrected setup retry. Final evaluator reassessment also encountered a missing executable path and a report-path collision. Earlier failed evidence stayed intact; no experimental code output was integrated."
    }
  ],
  "limitations": [
    "One small pair per task cannot show that planning generally helps or does not help.",
    "Selected-case scores are not exhaustive correctness scores; diagnostic correctness remains unresolved in this record.",
    "Cost is unknown for every arm. Available token observations do not establish a complete cost comparison.",
    "The historical evaluator used an externally supplied compiler, differing from the workspace-local command declared in the manifests.",
    "This isolates planner presence within a fixed model arrangement, not the merits of different models or larger teams."
  ],
  "decision": "Record no measured correctness improvement from adding the planner on these tasks. Preserve the diagnostic uncertainty and improve coverage before treating review acceptance as sufficient.",
  "next": "Use a planner when a concrete task warrants it, then test that decision with task-appropriate evidence. This study did not establish a general team-selection rule.",
  "controls": [
    "The same worker and reviewer revisions, ordered criteria, paths and 15-minute deadline were pinned within each pair.",
    "The worker and planner used the same Codex model with different role settings; Claude supplied the independent engineering review. This was a role-structure comparison, not a model-vendor bake-off.",
    "Pair order varied: two tasks ran worker-only first and one ran the planner arm first.",
    "Evaluator cases and reference material were excluded from worker and planner inputs."
  ]
}