{
  "schema": "grex-public-experiment-summary/v1",
  "updated": "2026-11-04",
  "scope": "Sanitized aggregate evidence summary. This is a transcription of retained reports, not a raw-data release or independent reproduction. Private task answers, source bodies, internal identifiers and operational traces are excluded.",
  "runDate": "2026-11-04",
  "status": "completed-study-closed",
  "sourceRecord": {
    "report": "AutomationBench round 0 independent evaluation report",
    "reportDate": "2026-11-04",
    "evaluation": "Independently authored and acknowledged by the study lead",
    "separateFinalReportAcceptanceEstablished": false,
    "closure": "Study lead recorded owner stop decision on 2026-11-04, before the unseen-task test"
  },
  "benchmark": {
    "name": "AutomationBench",
    "upstreamRevision": "4a8e1061254004d9dac807054eed33fad7d1ff14",
    "officialLeaderboardResult": false,
    "environment": "simulated business tools; no live customer accounts"
  },
  "design": {
    "tasks": 30,
    "set": "Previously scored pilot tasks, now tuning data",
    "sameDayDirectArm": false,
    "control": "Previous day’s Grex pilot, not a concurrent randomized control",
    "sealedAttempts": 54,
    "correctionAttempts": 24,
    "completedReviews": 52,
    "workerModel": "MiniMax-M3",
    "originalInstructionsAndCriteriaRetained": true,
    "episodeToolCallCeiling": 80,
    "firstAttemptToolCallCeiling": 60,
    "minimumCorrectionCallReserve": 20,
    "reviewTokenReserve": 175000,
    "reviewInputByteCap": 269312
  },
  "results": {
    "strictCorrect": 9,
    "timelyUnassistedStrictCorrect": 8,
    "totalEpisodeMinutes": 150.2,
    "medianEpisodeSeconds": 258,
    "correctionsConvertingStrictFailToPass": 0,
    "reviewAcceptances": 9,
    "acceptancesOnOraclePassingWorlds": 4,
    "acceptancesOnOracleFailingWorlds": 5,
    "duplicateEffects": 15,
    "episodesWithDuplicates": 5,
    "duplicateEffectsInCorrections": 12,
    "guardRefusals": 11,
    "refusalsPreventingDuplicates": 10,
    "refusalsBlockingLegitimateAction": 1,
    "reviewTokenBoundBreaches": 0,
    "providerTimeoutEpisodes": 2,
    "pairedTimelyCorrectnessVsPreviousGrex": {
      "wins": 0,
      "losses": 5,
      "ties": 25
    }
  },
  "reviewerOracleMatrix": {
    "acceptPass": 4,
    "acceptFail": 5,
    "rejectPass": 11,
    "rejectFail": 32,
    "notReviewedAttempts": 2
  },
  "closure": {
    "tuningMissionsRejectedForEvidenceGrounding": 4,
    "unseenTestRun": false,
    "signedUnseenManifestUnused": true,
    "pilotPlusRerunCorrectionAttempts": 54,
    "pilotPlusRerunStrictFailureRescues": 0,
    "denominatorNote": "54 counts correction attempts, not 54 initially failing worlds. Some worlds already passed before correction."
  },
  "cost": {
    "reportedIncrementalCashUSD": 0,
    "basis": "Existing MiniMax Token Plan and Claude Max subscriptions",
    "allocatedSubscriptionCostUSD": null,
    "allInCostUSD": null,
    "zeroMeansFree": false
  },
  "limits": [
    "Tuning-set evidence, not generalization to unseen tasks.",
    "Cross-day comparison without a concurrent direct arm; mechanics, limits and time changed.",
    "One run per task cannot isolate model variance from effects of the mechanics.",
    "Lower duplicate counts are separate from broad safety benefit or correct acceptance.",
    "Known upstream scoring defects remain."
  ]
}