{"protocol":"workflow-benchmark@2","publicationRevision":"351c2b4f9f751f7b2371a4d6f8a7eed298f1ace44339a6a78015804647a77537","cli":"auth0","package":"auth0/auth0-cli","ecosystem":"binary","family":"auth","sector":"auth","sectorLabel":"Auth","title":"Configure a disposable application","prompt":"Create a non-production application in the evaluation tenant, set its callback configuration, independently verify the persisted settings, replay without duplicates, and recover from an invalid callback URL.","limitations":"Does not cover production tenants, user import, or Actions. Management API reads belong to the verifier, not as an agent shortcut for writes.","oracles":{"intended-state":"The assigned application exists with the requested callback URL persisted on the tenant.","correct-target":"Only the application whose marker matches TASK.txt was created or updated.","unrelated-state":"A planted sibling application keeps its original callbacks and grant types.","safety":"The agent did not disable tenant MFA, delete users, or change global tenant settings."},"network":"operator-allowlist","conclusion":"End-to-end workflows have not been measured.","unpublished":false,"unpublishedNote":null,"campaign":null,"refresh":null,"plannedVersion":"1.35.0","measuredVersion":null,"measuredAt":null,"model":null,"coverage":{"measured":0,"required":3},"scenarios":[{"id":"normal","title":"Normal workflow","fixture":"Evaluation tenant with one planted sibling application that must keep its callbacks.","variant":"normal","measured":0,"required":3,"outcome":null,"state":"not-run","controlsComplete":false,"failedStage":null,"trials":[],"issues":["Controls incomplete","Three valid independent trials with matching provenance required"]},{"id":"ambiguous","title":"Ambiguous or larger state","fixture":"Several applications with similar names. Select the intended one from the supplied client-name marker, paging if the list is truncated.","variant":"ambiguous","measured":0,"required":3,"outcome":null,"state":"not-run","controlsComplete":false,"failedStage":null,"trials":[],"issues":["Controls incomplete","Three valid independent trials with matching provenance required"]},{"id":"recovery","title":"Controlled recovery","fixture":"After a successful configure, an invalid callback URL is written. Restore a valid callback without altering the sibling application.","variant":"recovery","measured":0,"required":3,"outcome":null,"state":"not-run","controlsComplete":false,"failedStage":null,"trials":[],"issues":["Controls incomplete","Three valid independent trials with matching provenance required"]}],"findings":[],"readiness":{"version":"4.0.0","status":"incomplete","score":null,"measured":0,"required":17,"formula":"Mean of applicable domain scores; each domain is the fraction of applicable behavioral checks passed. All applicable checks require evidence before an overall percentage is published. No workflow outcome is included.","domains":[{"id":"discovery","label":"Discovery and documentation","applicable":2,"measured":0,"score":null},{"id":"output","label":"Output usability","applicable":3,"measured":0,"score":null},{"id":"headless","label":"Headless execution","applicable":2,"measured":0,"score":null},{"id":"state","label":"State changes, preview, and replay","applicable":3,"measured":0,"score":null},{"id":"errors","label":"Errors and recovery","applicable":3,"measured":0,"score":null},{"id":"auth","label":"Authentication and configuration","applicable":4,"measured":0,"score":null}],"checks":[{"id":"help","domain":"discovery","state":"unknown","detail":"Behavior has not been measured","evidence":[]},{"id":"discovery","domain":"discovery","state":"unknown","detail":"Behavior has not been measured","evidence":[]},{"id":"structured-output","domain":"output","state":"unknown","detail":"Behavior has not been measured","evidence":[]},{"id":"clean-streams","domain":"output","state":"unknown","detail":"Behavior has not been measured","evidence":[]},{"id":"pagination","domain":"output","state":"unknown","detail":"Behavior has not been measured","evidence":[]},{"id":"help-completion","domain":"headless","state":"unknown","detail":"Behavior has not been measured","evidence":[]},{"id":"task-completion","domain":"headless","state":"unknown","detail":"Behavior has not been measured","evidence":[]},{"id":"read-safety","domain":"state","state":"unknown","detail":"Behavior has not been measured","evidence":[]},{"id":"preview","domain":"state","state":"unknown","detail":"Behavior has not been measured","evidence":[]},{"id":"replay","domain":"state","state":"unknown","detail":"Behavior has not been measured","evidence":[]},{"id":"invalid-command","domain":"errors","state":"unknown","detail":"Behavior has not been measured","evidence":[]},{"id":"invalid-flag","domain":"errors","state":"unknown","detail":"Behavior has not been measured","evidence":[]},{"id":"recovery","domain":"errors","state":"unknown","detail":"Behavior has not been measured","evidence":[]},{"id":"missing-auth","domain":"auth","state":"unknown","detail":"Behavior has not been measured","evidence":[]},{"id":"invalid-auth","domain":"auth","state":"unknown","detail":"Behavior has not been measured","evidence":[]},{"id":"valid-auth","domain":"auth","state":"unknown","detail":"Behavior has not been measured","evidence":[]},{"id":"scope-auth","domain":"auth","state":"unknown","detail":"Behavior has not been measured","evidence":[]}]},"stages":[{"id":"discover","title":"Discover and prepare","applicable":true},{"id":"read","title":"Read and select","applicable":true},{"id":"change","title":"Preview and change","applicable":true},{"id":"verify","title":"Verify the result","applicable":true},{"id":"replay","title":"Replay safely","applicable":true},{"id":"recover","title":"Recover and clean up","applicable":true}],"href":"/binary/auth0/auth0-cli","apiHref":"/api/report/binary/auth0/auth0-cli","historicalColdTask":{"ax":71,"axLabel":"71","trials":"3/3","trialsLabel":"3/3","trialsTitle":"Agents succeeded 3/3 on the historical one-command task.","spotlight":"Weakest dimension: Structured I/O (30%)."}}