{
  "schema": "grex-public-experiment-summary/v1",
  "slug": "direct-execution-comparison",
  "updated": "2026-11-04",
  "experimentDate": "2026-10-01",
  "status": "Completed · historical architecture",
  "scope": "Sanitized aggregate transcription of retained source records, not a raw-data release or independent reproduction. Private operational identifiers, source bodies and traces are excluded.",
  "question": "For a single bounded task, does the earlier Grex execution chain improve the output beyond one direct solver invocation?",
  "hypothesis": {
    "statement": "One direct invocation with frozen inputs and independent verification might perform as well as the longer execution chain.",
    "provenance": "The adopted architecture decision records this as the experiment question. This directional wording is a retrospective summary, not a separately verified preregistered numerical prediction."
  },
  "sample": "Five pairs across three task types and two solver models. Repeated biostatistics pairs are not independent task samples. Scores count rubric items, not fully completed tasks.",
  "method": "Run the same frozen task, model and prompt once directly and once through the earlier Grex invocation chain. Compare the retained deliverables using local rubric judging. A later pair used a previously uninspected task.",
  "comparison": {
    "columns": [
      "Measure",
      "Direct invocation",
      "Earlier Grex chain"
    ],
    "rows": [
      [
        "Biostatistics · Sonnet",
        "4 / 8 items",
        "4 / 8 items"
      ],
      [
        "Biostatistics · MiniMax",
        "4 / 8 items",
        "2 / 8 items"
      ],
      [
        "Biostatistics · Sonnet, reversed order",
        "4 / 8 items",
        "5 / 8 items"
      ],
      [
        "Administrative task · MiniMax",
        "3 / 7 items",
        "4 / 7 items"
      ],
      [
        "Bookkeeping holdout · MiniMax",
        "2 / 10 items",
        "2 / 10 items"
      ],
      [
        "Pooled rubric items",
        "17 / 41",
        "17 / 41"
      ]
    ]
  },
  "findings": [
    {
      "title": "The aggregate was tied",
      "body": "Both paths passed 17 of 41 judged items. The largest pair-level gap was two items. This found no measured overall output advantage for the extra chain on these tasks."
    },
    {
      "title": "The holdout was difficult for both",
      "body": "Both bookkeeping attempts produced the three expected files but timed out and returned no final text. Each passed two of ten judged items. Producing files was not equivalent to completing the task correctly."
    },
    {
      "title": "The experiment exposed comparison defects",
      "body": "Effective environments were not fully equal, and some chain attempts were marked failed because of provenance-label mismatch despite retained artifacts passing integrity checks. Those failures and mismatches remain part of the record."
    },
    {
      "title": "The decision had a narrow scope",
      "body": "The resulting direction used one bounded solver invocation for one task. Scheduling, persistence, authorization, recovery and multi-agent coordination were not tested by this comparison and were not shown unnecessary."
    }
  ],
  "limitations": [
    "This concerns the earlier Grex architecture, not a present-day Grex 2 benchmark score.",
    "Five pairs across three task types are too few for a general equivalence or superiority claim.",
    "Local rubric judging had known validity defects and was not official JobBench evaluation.",
    "Two task types had been inspected during development. Repeated tasks, model variation and environment differences limit causal inference.",
    "No general speed advantage or complete all-in cost comparison was established."
  ],
  "decision": "Simplify the execution path for one bounded task while retaining external verification and mission-level controls. Do not extend this finding to all orchestration or cooperating agents.",
  "next": "The later coding-team study isolates planner presence on three small code tasks. Its result is a separate experiment on Grex 2.",
  "controls": [
    "Both arms in a pair used the same frozen task, model and prompt.",
    "One Sonnet pair reversed execution order; that pair also changed a Grex source pin, preventing clean order attribution.",
    "The bookkeeping holdout was selected by a fixed rule before its task card, rubric and references were inspected.",
    "Execution and artifact integrity were recorded separately from rubric judgments."
  ]
}