{
  "schema": "grex-public-experiment-summary/v1",
  "updated": "2026-11-04",
  "scope": "Sanitized aggregate evidence summary. This is a transcription of retained reports, not a raw-data release or independent reproduction. Private task answers, source bodies, internal identifiers and operational traces are excluded.",
  "runDate": "2026-11-03",
  "status": "completed-diagnostic",
  "sourceRecord": {
    "report": "AutomationBench pilot independent evaluation report, including Addendum A",
    "reportDate": "2026-11-03",
    "evaluation": "Independently authored and recorded by the study lead",
    "laterDecision": "Study closed on 2026-11-04 before the unseen-task test"
  },
  "benchmark": {
    "name": "AutomationBench",
    "upstreamRevision": "4a8e1061254004d9dac807054eed33fad7d1ff14",
    "officialLeaderboardResult": false,
    "environment": "simulated business tools; no live customer accounts"
  },
  "design": {
    "tasks": 30,
    "domains": 6,
    "tasksPerDomain": 5,
    "matchedPairs": 30,
    "primaryEpisodes": 60,
    "sealedAttempts": 90,
    "workerModel": "MiniMax-M3",
    "grexReviewer": "Claude Opus 5.5",
    "episodeToolCallCeiling": 80,
    "episodeMinutes": 30,
    "directAttemptMinutes": 30,
    "grexAttemptMinutes": 7,
    "maxCorrectionsPerGrexEpisode": 1,
    "armOrder": "15 direct-first, 15 Grex-first"
  },
  "results": {
    "direct": {
      "strictCorrect": 13,
      "timelyUnassistedStrictCorrect": 13,
      "totalEpisodeMinutes": 68.7,
      "medianEpisodeSeconds": 120,
      "duplicateEffects": 2,
      "episodesWithDuplicates": 2
    },
    "grex": {
      "strictCorrect": 14,
      "timelyUnassistedStrictCorrect": 13,
      "totalEpisodeMinutes": 144.7,
      "medianEpisodeSeconds": 258,
      "reviews": 58,
      "reviewAcceptances": 0,
      "correctionAttempts": 30,
      "correctionsConvertingStrictFailToPass": 0,
      "duplicateEffects": 34,
      "episodesWithDuplicates": 11,
      "duplicateEffectsInCorrections": 32,
      "reviewTokenBoundBreaches": 33
    },
    "pairedTimelyCorrectnessGrexVsDirect": {
      "wins": 2,
      "losses": 2,
      "ties": 26
    },
    "reviewerRejectionsOfOraclePassingWorlds": {
      "total": 27,
      "justifiedOperational": 15,
      "incompleteEvidence": 6,
      "unsupportedOrImmaterial": 6
    }
  },
  "protocolFailures": [
    "All 30 corrections received unchanged initial model requests, without reviewer feedback or the world-persistence notice.",
    "33 of 58 reviews exceeded their token reserve and the driver did not halt.",
    "33 review packets omitted calls; 25 omitted at least one write call.",
    "No blind operator dispositions were recorded; they cannot be recovered after oracle scores are revealed."
  ],
  "correctionNotice": "The full Addendum A audit replaces the initial Grex duplicate-effect count of 19 with 34. Upstream scores are unchanged.",
  "cost": {
    "reportedIncrementalCashUSD": 0,
    "basis": "Existing MiniMax Token Plan and Claude Max subscriptions",
    "invoiceReconciled": false,
    "allocatedSubscriptionCostUSD": null,
    "allInCostUSD": null,
    "zeroMeansFree": false
  },
  "limits": [
    "Diagnostic, not protocol-conformant. Intended feedback-driven correction was not tested.",
    "One worker model, one run per task, 30 matched tasks.",
    "Known upstream scoring defects remain; one task was unreachable with its offered tools.",
    "World correctness, operational appropriateness, reviewer acceptance and operator usability are different measures."
  ]
}