{
  "schema": "grex-public-experiment-summary/v1",
  "slug": "legacy-benchmark-attempts",
  "updated": "2026-11-04",
  "experimentDate": "2026-08-31",
  "status": "Historical · mixed completion",
  "scope": "Sanitized aggregate transcription of retained source records, not a raw-data release or independent reproduction. Private operational identifiers, source bodies and traces are excluded.",
  "question": "Could the earlier Grex framework carry different kinds of benchmark work through the same execution and evidence process?",
  "hypothesis": {
    "statement": "A shared framework might support research, transactional, tool-use and terminal tasks without a separate execution system for each.",
    "provenance": "Retrospective interpretation of the original standing-suite objective. These runs were not a registered claim that Grex would beat a direct agent or a particular model."
  },
  "sample": "The August 31 suite planned 75 tasks: 30 GAIA, 20 retail tasks, 13 MCP-Atlas and 12 adapted terminal tasks. Earlier subsets differed, especially MCP-Atlas.",
  "method": "Small pinned subsets were assigned to the earlier Grex benchmark runner. A solving process produced answers or work artifacts; a separate scoring process used the benchmark-specific checks. The later run changed both tool access and model allocation.",
  "comparison": {
    "columns": [
      "Measure",
      "August 30 run",
      "August 31 run"
    ],
    "rows": [
      [
        "GAIA Level 1",
        "21 / 27 scored; 3 not attempted",
        "24 / 30"
      ],
      [
        "Retail transactional tasks",
        "16 / 20",
        "16 / 20"
      ],
      [
        "MCP-Atlas",
        "0 / 3; mean partial 0.4",
        "3 / 10 scored; 3 unavailable"
      ],
      [
        "Adapted terminal tasks",
        "Not run in this suite",
        "6 pass; 5 fail; 1 unscored"
      ]
    ]
  },
  "findings": [
    {
      "title": "The denominator changes the story",
      "body": "The first GAIA headline counted 27 scored tasks out of 30 selected. Three were not attempted because of a safeguards refusal or timeouts. Reporting only the scored denominator would hide those outcomes."
    },
    {
      "title": "Tool availability was part of the failure",
      "body": "The tool-use runs included missing server support and later admission failures. The August 31 record lists three unavailable tasks separately from the ten scored tasks."
    },
    {
      "title": "Terminal work did not fully settle",
      "body": "Six adapted terminal tasks passed and five failed. One remained unscored because of a scorer-settlement defect. The attempted environment differed from the official benchmark setup."
    },
    {
      "title": "The later score does not isolate the model",
      "body": "Tools, harness behavior and model allocation changed together. The recorded gains cannot show that one model was better or that orchestration caused the difference."
    }
  ],
  "limitations": [
    "Historical Grex architecture, not a score for the current Grex 2 platform.",
    "Small subsets and changed conditions prevent causal comparisons or official leaderboard claims.",
    "Missing and unscored tasks remain visible; the table does not pool incompatible denominators.",
    "These figures are transcribed from the archived program record. This publication does not reproduce the raw benchmark runs.",
    "No complete, reconciled all-in cost comparison is established here."
  ],
  "decision": "Keep these attempts as engineering evidence about task coverage and failure handling. Do not present them as general autonomy scores or proof that the platform improves its models.",
  "next": "Later experiments asked narrower questions with explicit direct-runner comparisons. The direct-execution study and AutomationBench records show how the evaluation changed.",
  "controls": [
    "Task identities were retained for the 30 GAIA and 20 retail tasks across the two runs.",
    "Solving and scoring were separate processes, with benchmark-specific task environments where required.",
    "The terminal subset was adapted to the machine operating system rather than the official container environment; it was explicitly not leaderboard-comparable.",
    "The second run was budget-model-first but still used a frontier reviewer. It was not a single-model comparison."
  ]
}