{
  "title": "Grex AutomationBench first pilot",
  "asOf": "2026-11-03",
  "status": "diagnostic pilot completed; no tuning results reported",
  "provenance": "Public summary derived from the separately authored Grex pilot evaluation report and its November 3 Addendum A. This summary does not contain raw trajectories or task answers and is not an independently reproducible evidence package.",
  "benchmark": "AutomationBench",
  "upstream": "https://github.com/zapier/AutomationBench/tree/4a8e1061254004d9dac807054eed33fad7d1ff14",
  "scope": "Simulated business tools; no real customer messages or accounts",
  "tasksPerArm": 30,
  "primaryEpisodes": 60,
  "totalAttemptsIncludingCorrections": 90,
  "workerModel": "MiniMax-M3",
  "reviewerModel": "Claude Opus 5.5",
  "direct": {
    "timelyUnassistedStrictCorrect": 13,
    "strictPass": 13,
    "elapsedExecutionMinutes": 68.7,
    "toolCalls": 1270
  },
  "grex": {
    "timelyUnassistedStrictCorrect": 13,
    "strictPass": 14,
    "elapsedExecutionMinutes": 144.7,
    "toolCalls": 1865
  },
  "pairedTimelyCorrect": {
    "grexWins": 2,
    "grexLosses": 2,
    "ties": 26
  },
  "review": {
    "completed": 58,
    "accepted": 0,
    "rejectedOracleCorrect": 27,
    "rejectedOracleIncorrect": 31,
    "rationalesCitingTruncatedEvidence": 20,
    "oracleCorrectRejectionClassification": {
      "operationallyJustified": 15,
      "incompleteDecisiveEvidence": 6,
      "unsupportedOrImmaterial": 6
    },
    "packetsWithOmittedCalls": 33,
    "packetsWithOmittedWrites": 25
  },
  "corrections": {
    "episodesRescuedFromStrictFailToPass": 0,
    "episodesWithNoCallsRemaining": 8,
    "duplicateEffectsFromCorrections": 32
  },
  "deviations": {
    "reviewReserveBreaches": 33,
    "driverHaltedOnBreaches": false,
    "blindOperatorDispositionsRecorded": false
  },
  "cost": {
    "observedMeteredFallback": false,
    "funding": "Existing subscriptions",
    "invoiceReconciled": false,
    "subscriptionHardwareAndPreparationFullyAllocated": false
  },
  "limits": [
    "One worker model; one run per task; small public sample",
    "Reviewer, evaluator and acceptance roles used the same model family",
    "24 tasks flagged as potentially affected by registered grader issues; unresolved false-pass risk remains",
    "A strict pass is not reviewer acceptance or demonstrated operational safety",
    "Grex had shorter attempt limits inside the shared episode deadline",
    "Not a leaderboard submission, production reliability result or savings claim"
  ],
  "operationalEffects": {
    "definition": "A second successful outward write conveying the same item to the same target after the first succeeded. Distinct items sent to the same recipient are excluded. Simulated actions only.",
    "direct": {
      "episodesWithDuplicates": 2,
      "duplicateEffects": 2,
      "messages": 2,
      "records": 0
    },
    "grex": {
      "episodesWithDuplicates": 11,
      "duplicateEffects": 34,
      "messages": 19,
      "records": 15
    },
    "revisionNote": "Addendum A full action-log audit replaces the initial 19 duplicate sends or creates in 10 Grex episodes. Upstream scores unchanged."
  }
}