Replays a structure change over past cases with feedback: it spends model calls and stores a run. Not a dry run of evaluate. Replays a local structure and its base over the decision’s reviewed and imported cases in the key’s environment (a test key previews against test cases only), and returns fixes, breaks, the action mix, the loss_report (base and candidate judged by the decision’s objective, or balanced losses when none is set, with loss per case, macro-F1, where the cost sits, the costliest mistakes, and whether the candidate is accepted) and an evidence label: training_cases when a scored case is a training case (its deciding feedback is not an import), else imported_examples (every scored case is an imported example); either way the cases are ones a suggestion may have learned from. The result must decide outcomes (bands, rules or match): a value result is 400 OUTCOMES_INVALID. Each case reuses the answers its evaluation stored for questions whose hash is unchanged; changed and new questions are asked the way evaluate asks them (each question’s reads, grouped into System One calls), so a preview costs what evaluating the changed questions would. Writes an eval run only: never an evaluation, never traffic, never acceptance. A preview does not register the candidate: before evaluating by its structure hash, register it with POST /v1/decisions (or send the structure inline on evaluate once). The base, in order: the base of suggestion_id when it is sent (then base is ignored), else base, else the version serving primary traffic in the key’s environment, else the newest one examples were imported under there, else none. With suggestion_id, hashes_match says whether the structure is that suggestion’s candidate; another environment’s suggestion is 404. The evidence replays only cases with an outcome from the owner, reviewers or imported examples; the action mix replays a sample of the environment’s recent primary traffic. For questions the candidate changes, before is a fresh re-run of the base question (the fair baseline), so model nondeterminism can move it between previews. noise_rate measures outcome flips from that re-run against the stored answers, not value drift. A test key’s preview neither reads nor fills the shared cache of model answers. A decision nothing was registered, evaluated or imported under is 404.
POST
/
v1
/
decisions
/
{decision_id}
/
previews
Preview a structure change against past feedback
curl --request POST \ --url https://api.moonbounce.ai/v1/decisions/{decision_id}/previews \ --header 'Authorization: Bearer <token>' \ --header 'Content-Type: application/json' \ --data '{ "structure": { "questions": { "item_used": { "type": "noul", "instructions": "Has the customer used the item beyond trying it on or setting it up once?", "reads": [ "state.message" ] }, "defective": { "type": "noul", "instructions": "Does the customer describe a defect, damage on arrival, or the item failing under normal use?", "reads": [ "state.message" ] }, "shipping_failure": { "type": "noul", "instructions": "Did the order arrive later than promised, or get lost in transit?", "reads": [ "state.message" ] } }, "attributes": { "days_since_delivery": { "type": "number", "minimum": 0 } }, "aggregations": { "grant_score": { "type": "weighted_mean", "inputs": [ { "name": "questions.item_used", "invert": true }, { "name": "questions.defective" }, { "name": "questions.shipping_failure" } ] }, "handling": { "type": "rules", "rules": [ { "when": { "name": "attributes.days_since_delivery", "gt": 30 }, "outcome": "escalate" }, { "when": { "name": "aggregations.grant_score", "gte": 0.72 }, "outcome": "grant" }, { "when": { "name": "aggregations.grant_score", "lt": 0.38 }, "outcome": "deny" } ], "else": "escalate" } }, "result": "aggregations.handling" }}'
import requests
url = "https://api.moonbounce.ai/v1/decisions/{decision_id}/previews"
payload = { "structure": {
"questions": {
"item_used": {
"type": "noul",
"instructions": "Has the customer used the item beyond trying it on or setting it up once?",
"reads": ["state.message"]
},
"defective": {
"type": "noul",
"instructions": "Does the customer describe a defect, damage on arrival, or the item failing under normal use?",
"reads": ["state.message"]
},
"shipping_failure": {
"type": "noul",
"instructions": "Did the order arrive later than promised, or get lost in transit?",
"reads": ["state.message"]
}
},
"attributes": { "days_since_delivery": {
"type": "number",
"minimum": 0
} },
"aggregations": {
"grant_score": {
"type": "weighted_mean",
"inputs": [{
"name": "questions.item_used",
"invert": True
}, { "name": "questions.defective" }, { "name": "questions.shipping_failure" }]
},
"handling": {
"type": "rules",
"rules": [
{
"when": {
"name": "attributes.days_since_delivery",
"gt": 30
},
"outcome": "escalate"
},
{
"when": {
"name": "aggregations.grant_score",
"gte": 0.72
},
"outcome": "grant"
},
{
"when": {
"name": "aggregations.grant_score",
"lt": 0.38
},
"outcome": "deny"
}
],
"else": "escalate"
}
},
"result": "aggregations.handling"
} }
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)
const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
structure: {
questions: {
item_used: {
type: 'noul',
instructions: 'Has the customer used the item beyond trying it on or setting it up once?',
reads: ['state.message']
},
defective: {
type: 'noul',
instructions: 'Does the customer describe a defect, damage on arrival, or the item failing under normal use?',
reads: ['state.message']
},
shipping_failure: {
type: 'noul',
instructions: 'Did the order arrive later than promised, or get lost in transit?',
reads: ['state.message']
}
},
attributes: {days_since_delivery: {type: 'number', minimum: 0}},
aggregations: {
grant_score: {
type: 'weighted_mean',
inputs: [
{name: 'questions.item_used', invert: true},
{name: 'questions.defective'},
{name: 'questions.shipping_failure'}
]
},
handling: {
type: 'rules',
rules: [
{when: {name: 'attributes.days_since_delivery', gt: 30}, outcome: 'escalate'},
{when: {name: 'aggregations.grant_score', gte: 0.72}, outcome: 'grant'},
{when: {name: 'aggregations.grant_score', lt: 0.38}, outcome: 'deny'}
],
else: 'escalate'
}
},
result: 'aggregations.handling'
}
})
};
fetch('https://api.moonbounce.ai/v1/decisions/{decision_id}/previews', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));
An API key, sent as Authorization: Bearer <key>. Each operation lists the scope its key needs: decisions:evaluate (evaluate and read evaluations), decisions:feedback (your app's feedback) or decisions:develop (everything else, including owner feedback and the review queue). An operation listing no scope takes any scoped key. A key with no scope, or without the operation's, gets 403 INSUFFICIENT_SCOPE.
Makes a retry safe. For 24 hours, the same key with the same request returns the first stored result (with Idempotent-Replayed: true); the same key with a different request is a 422 IDEMPOTENCY_KEY_REUSED. Keys are scoped to the organization and environment across every operation, so use a fresh key per request. A request that failed before its result was stored runs again when retried with its key.
A name you choose for this decision, such as refund-exception: a lowercase letter, then up to 127 of a-z 0-9 _ . -. Unique in your organization and shared by its test and production keys. The first registration or evaluation under a new one creates the decision.
A name you choose for this decision, such as refund-exception: a lowercase letter, then up to 127 of a-z 0-9 _ . -. Unique in your organization and shared by its test and production keys. The first registration or evaluation under a new one creates the decision.
The environment of the key that made the call: production or test. Evaluations, feedback, reviews and suggestions are separate per environment; decisions and their versions belong to the whole organization.
The decision's stored objective revision this preview was scored with, or null when none is set (balanced losses). Absent only on a replay of a response stored before it was recorded.
Base and candidate judged by the decision's objective, on the same scored cases (each mistake's loss is its severity, every mistake normal without an objective, weighted so each expected outcome counts equally), and whether the candidate is accepted: lower loss per case and no new severe mistake on a human-labeled case. Absent only on a replay of a response stored before previews were judged.
Show child attributes
{"docsConfig":{"theme":"mint","$schema":"https://mintlify.com/docs.json","name":"Moonbounce Decisions API","colors":{"primary":"#1D5AB5","light":"#74A8F0","dark":"#478EF0"},"logo":{"light":"https://mintcdn.com/moonbounce/mWuXGW-nUJSEdABp/logo/lockup-light.svg?fit=max\u0026auto=format\u0026n=mWuXGW-nUJSEdABp\u0026q=85\u0026s=367a5dedb5799a16c5737d136af68dc5","dark":"https://mintcdn.com/moonbounce/mWuXGW-nUJSEdABp/logo/lockup-dark.svg?fit=max\u0026auto=format\u0026n=mWuXGW-nUJSEdABp\u0026q=85\u0026s=3b3e891bc9f1e5b26bee272f878ecd4f","href":"https://moonbounce.ai"},"favicon":{"light":"https://mintcdn.com/moonbounce/XgvwJtU0Z2TC5k2H/logo/symbol-light.svg?fit=max\u0026auto=format\u0026n=XgvwJtU0Z2TC5k2H\u0026q=85\u0026s=cd361cb434cc5cabaa2240471ebc780a","dark":"https://mintcdn.com/moonbounce/XgvwJtU0Z2TC5k2H/logo/symbol-dark.svg?fit=max\u0026auto=format\u0026n=XgvwJtU0Z2TC5k2H\u0026q=85\u0026s=30d3353b4d3a5756ecfadd153e271d89"},"background":{"color":{"light":"#FFFEEE","dark":"#000000"}},"navbar":{"primary":{"type":"button","label":"Request access","href":"https://app.moonbounce.ai"}},"banner":{"content":"Looking for the Clavata policy and labeling API? Those docs are at [docs.clavata.ai](https://docs.clavata.ai). You are on the docs for the Moonbounce decisions platform.","dismissible":true,"type":"info","color":{"light":"#1D5AB5","dark":"#1D5AB5"}},"styling":{"latex":false},"navigation":{"pages":[]}},"deployment":{"deploymentMetadataValue":{"subdomain":"moonbounce","hasAudioNarration":false,"actualSubdomain":"moonbounce","customDomains":["docs.moonbounce.ai"],"gitSource":{"type":"github","owner":"clavataai","repo":"decisions-platform","deployBranch":"main","contentDirectory":"docs/site","isPrivate":true,"mountPath":"","label":"","_id":"6ae8f7a905e268646b31d443"},"entitlements":{},"buildId":"6aeeab2dcf919c3dc1b0bf30","clientVersion":"0.0.4076","searchFilterCounts":[{"language":"default","tag":"API reference","count":32},{"language":"default","tag":"Guides","count":15},{"language":"default","tag":"Decisions","count":6},{"language":"default","tag":"Feedback","count":5},{"language":"default","tag":"Suggestions","count":5},{"language":"default","tag":"Objective","count":4},{"language":"default","tag":"Get started","count":3},{"language":"default","tag":"Build a decision","count":3},{"language":"default","tag":"Run a decision","count":3},{"language":"default","tag":"Improve a decision","count":3}]},"assistantConfig":{"enableAskAiButton":false,"enableStarterQuestions":false,"enableContextualStarterQuestions":false,"enableAskAiSearchBar":true},"basePath":"","hasChatPermissions":false,"hasWebMcpPermissions":false,"toggles":[{"name":"kill-isr","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-mintlify-slack","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-integrations-settings","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"rbac-expansion","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-pr-grouping","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"custom-prefix","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"analytics-v7","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"user-flows-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"no-publish-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-onboarding","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"llms-txt-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving-live","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"stripTemperatureMiddleware","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"support-platform-pylon-zendesk","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"instant-rollback","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications-server","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-fable","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fixed-ai-action-pricing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automation-resolve-merge-conflict","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"scoped-publishing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"mintql","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"page-level-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-version-history-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"billing-page-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"full-page-search","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workspace-projects","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"assistant-v5","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fork-pr-previews","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"revalidation-jwt","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-sso-scim","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"hobby-abuse-scan","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"dashboard-inbox","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-signup","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"turbopuffer-dual-write","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-enabled","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"approve-review-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-metadata","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"sso-setup-deadline-banner","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"turbopuffer-serve","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"live-agents","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"comment-mode","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false}],"googleFontsInlined":false},"customCss":[{"filePath":"brand.css","content":":root{--mb-font-sans: \"Helvetica Neue\", Helvetica, Arial, sans-serif;--mb-font-mono: ui-monospace, \"SF Mono\", Menlo, Consolas, monospace;--mb-rule: rgb(225, 230, 240);--mb-rule-strong: rgb(134, 134, 134)}.dark{--mb-rule: rgb(68, 69, 74);--mb-rule-strong: rgb(134, 134, 134)}body{font-family:var(--mb-font-sans)}h1,h2,h3,h4,h5,h6{font-family:var(--mb-font-sans);font-weight:600;letter-spacing:-0.011em}#page-title{letter-spacing:-0.03em;line-height:1.05}code,pre,kbd,samp{font-family:var(--mb-font-mono)}#navbar a[target=\"_blank\"][href=\"https://app.moonbounce.ai\"]>span.absolute{background:rgb(0, 0, 0);border-radius:999px}#navbar a[target=\"_blank\"][href=\"https://app.moonbounce.ai\"] .text-white,#navbar a[target=\"_blank\"][href=\"https://app.moonbounce.ai\"] svg{color:rgb(255, 254, 238)}.dark #navbar a[target=\"_blank\"][href=\"https://app.moonbounce.ai\"]>span.absolute{background:rgb(255, 254, 238)}.dark #navbar a[target=\"_blank\"][href=\"https://app.moonbounce.ai\"] .text-white,.dark #navbar a[target=\"_blank\"][href=\"https://app.moonbounce.ai\"] svg{color:rgb(0, 0, 0)}.card,.callout,.code-block,.accordion,table{border-radius:0;box-shadow:none}.card{border:1px solid var(--mb-rule)}.card:hover{border-color:var(--mb-rule-strong)}"}],"page":{"pagePath":"api-reference/previews-\u0026-tests/preview-a-structure-change-against-past-feedback","slug":"/api-reference/previews-\u0026-tests/preview-a-structure-change-against-past-feedback","pageMetadata":{"title":"Preview a structure change against past feedback","description":"Replays a structure change over past cases with feedback: it spends model calls and stores a run. Not a dry run of evaluate. Replays a local structure and its base over the decision's reviewed and imported cases in the key's environment (a test key previews against test cases only), and returns fixes, breaks, the action mix, the `loss_report` (base and candidate judged by the decision's objective, or balanced losses when none is set, with loss per case, macro-F1, where the cost sits, the costliest mistakes, and whether the candidate is `accepted`) and an evidence label: `training_cases` when a scored case is a training case (its deciding feedback is not an import), else `imported_examples` (every scored case is an imported example); either way the cases are ones a suggestion may have learned from. The result must decide outcomes (bands, rules or match): a value result is 400 OUTCOMES_INVALID. Each case reuses the answers its evaluation stored for questions whose hash is unchanged; changed and new questions are asked the way evaluate asks them (each question's `reads`, grouped into System One calls), so a preview costs what evaluating the changed questions would. Writes an eval run only: never an evaluation, never traffic, never acceptance. A preview does not register the candidate: before evaluating by its structure hash, register it with POST /v1/decisions (or send the structure inline on evaluate once). The base, in order: the base of `suggestion_id` when it is sent (then `base` is ignored), else `base`, else the version serving primary traffic in the key's environment, else the newest one examples were imported under there, else none. With `suggestion_id`, `hashes_match` says whether the structure is that suggestion's candidate; another environment's suggestion is 404. The evidence replays only cases with an outcome from the owner, reviewers or imported examples; the action mix replays a sample of the environment's recent primary traffic. For questions the candidate changes, `before` is a fresh re-run of the base question (the fair baseline), so model nondeterminism can move it between previews. `noise_rate` measures outcome flips from that re-run against the stored answers, not value drift. A test key's preview neither reads nor fills the shared cache of model answers. A decision nothing was registered, evaluated or imported under is 404.","deprecated":null,"version":null,"openapi":"/openapi.json post /v1/decisions/{decision_id}/previews","href":"/api-reference/previews-\u0026-tests/preview-a-structure-change-against-past-feedback","autogeneratedByOpenApi":true},"description":{"compiledSource":"\"use strict\";\nconst {jsx: _jsx, jsxs: _jsxs} = arguments[0];\nconst {useMDXComponents: _provideComponents} = arguments[0];\nfunction _createMdxContent(props) {\n const _components = {\n code: \"code\",\n p: \"p\",\n ..._provideComponents(),\n ...props.components\n };\n return _jsxs(_components.p, {\n children: [\"Replays a structure change over past cases with feedback: it spends model calls and stores a run. Not a dry run of evaluate. Replays a local structure and its base over the decision’s reviewed and imported cases in the key’s environment (a test key previews against test cases only), and returns fixes, breaks, the action mix, the \", _jsx(_components.code, {\n children: \"loss_report\"\n }), \" (base and candidate judged by the decision’s objective, or balanced losses when none is set, with loss per case, macro-F1, where the cost sits, the costliest mistakes, and whether the candidate is \", _jsx(_components.code, {\n children: \"accepted\"\n }), \") and an evidence label: \", _jsx(_components.code, {\n children: \"training_cases\"\n }), \" when a scored case is a training case (its deciding feedback is not an import), else \", _jsx(_components.code, {\n children: \"imported_examples\"\n }), \" (every scored case is an imported example); either way the cases are ones a suggestion may have learned from. The result must decide outcomes (bands, rules or match): a value result is 400 OUTCOMES_INVALID. Each case reuses the answers its evaluation stored for questions whose hash is unchanged; changed and new questions are asked the way evaluate asks them (each question’s \", _jsx(_components.code, {\n children: \"reads\"\n }), \", grouped into System One calls), so a preview costs what evaluating the changed questions would. Writes an eval run only: never an evaluation, never traffic, never acceptance. A preview does not register the candidate: before evaluating by its structure hash, register it with POST /v1/decisions (or send the structure inline on evaluate once). The base, in order: the base of \", _jsx(_components.code, {\n children: \"suggestion_id\"\n }), \" when it is sent (then \", _jsx(_components.code, {\n children: \"base\"\n }), \" is ignored), else \", _jsx(_components.code, {\n children: \"base\"\n }), \", else the version serving primary traffic in the key’s environment, else the newest one examples were imported under there, else none. With \", _jsx(_components.code, {\n children: \"suggestion_id\"\n }), \", \", _jsx(_components.code, {\n children: \"hashes_match\"\n }), \" says whether the structure is that suggestion’s candidate; another environment’s suggestion is 404. The evidence replays only cases with an outcome from the owner, reviewers or imported examples; the action mix replays a sample of the environment’s recent primary traffic. For questions the candidate changes, \", _jsx(_components.code, {\n children: \"before\"\n }), \" is a fresh re-run of the base question (the fair baseline), so model nondeterminism can move it between previews. \", _jsx(_components.code, {\n children: \"noise_rate\"\n }), \" measures outcome flips from that re-run against the stored answers, not value drift. A test key’s preview neither reads nor fills the shared cache of model answers. A decision nothing was registered, evaluated or imported under is 404.\"]\n });\n}\nfunction MDXContent(props = {}) {\n const {wrapper: MDXLayout} = {\n ..._provideComponents(),\n ...props.components\n };\n return MDXLayout ? _jsx(MDXLayout, {\n ...props,\n children: _jsx(_createMdxContent, {\n ...props\n })\n }) : _createMdxContent(props);\n}\nreturn {\n default: MDXContent\n};\n","frontmatter":{},"scope":{}},"pageType":"default","mdx":{"mdxSource":{"compiledSource":"\"use strict\";const{Fragment:_Fragment,jsx:_jsx}=arguments[0];const{useMDXComponents:_provideComponents}=arguments[0];function _createMdxContent(props){return _jsx(_Fragment,{})}function MDXContent(props={}){const{wrapper:MDXLayout}={..._provideComponents(),...props.components};return MDXLayout?_jsx(MDXLayout,{...props,children:_jsx(_createMdxContent,{...props})}):_createMdxContent(props)}return{default:MDXContent}","frontmatter":{},"scope":{"config":{},"pageMetadata":{"title":"Preview a structure change against past feedback","description":"Replays a structure change over past cases with feedback: it spends model calls and stores a run. Not a dry run of evaluate. Replays a local structure and its base over the decision's reviewed and imported cases in the key's environment (a test key previews against test cases only), and returns fixes, breaks, the action mix, the `loss_report` (base and candidate judged by the decision's objective, or balanced losses when none is set, with loss per case, macro-F1, where the cost sits, the costliest mistakes, and whether the candidate is `accepted`) and an evidence label: `training_cases` when a scored case is a training case (its deciding feedback is not an import), else `imported_examples` (every scored case is an imported example); either way the cases are ones a suggestion may have learned from. The result must decide outcomes (bands, rules or match): a value result is 400 OUTCOMES_INVALID. Each case reuses the answers its evaluation stored for questions whose hash is unchanged; changed and new questions are asked the way evaluate asks them (each question's `reads`, grouped into System One calls), so a preview costs what evaluating the changed questions would. Writes an eval run only: never an evaluation, never traffic, never acceptance. A preview does not register the candidate: before evaluating by its structure hash, register it with POST /v1/decisions (or send the structure inline on evaluate once). The base, in order: the base of `suggestion_id` when it is sent (then `base` is ignored), else `base`, else the version serving primary traffic in the key's environment, else the newest one examples were imported under there, else none. With `suggestion_id`, `hashes_match` says whether the structure is that suggestion's candidate; another environment's suggestion is 404. The evidence replays only cases with an outcome from the owner, reviewers or imported examples; the action mix replays a sample of the environment's recent primary traffic. For questions the candidate changes, `before` is a fresh re-run of the base question (the fair baseline), so model nondeterminism can move it between previews. `noise_rate` measures outcome flips from that re-run against the stored answers, not value drift. A test key's preview neither reads nor fills the shared cache of model answers. A decision nothing was registered, evaluated or imported under is 404.","deprecated":null,"version":null,"openapi":"/openapi.json post /v1/decisions/{decision_id}/previews","href":"/api-reference/previews-\u0026-tests/preview-a-structure-change-against-past-feedback","autogeneratedByOpenApi":true}}},"mdxSourceWithNoJs":null},"mdxExtracts":{"tableOfContents":[],"codeExamples":{}},"jsLinks":[],"openApiReferenceData":{"operation":{"operationId":"preview","x-mcp-annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false,"openWorldHint":false},"tags":["Previews \u0026 tests"],"security":["cedf8060-3208-4cb4-ae4c-4d9c08a5df3c"],"summary":"Preview a structure change against past feedback","description":"Replays a structure change over past cases with feedback: it spends model calls and stores a run. Not a dry run of evaluate. Replays a local structure and its base over the decision's reviewed and imported cases in the key's environment (a test key previews against test cases only), and returns fixes, breaks, the action mix, the `loss_report` (base and candidate judged by the decision's objective, or balanced losses when none is set, with loss per case, macro-F1, where the cost sits, the costliest mistakes, and whether the candidate is `accepted`) and an evidence label: `training_cases` when a scored case is a training case (its deciding feedback is not an import), else `imported_examples` (every scored case is an imported example); either way the cases are ones a suggestion may have learned from. The result must decide outcomes (bands, rules or match): a value result is 400 OUTCOMES_INVALID. Each case reuses the answers its evaluation stored for questions whose hash is unchanged; changed and new questions are asked the way evaluate asks them (each question's `reads`, grouped into System One calls), so a preview costs what evaluating the changed questions would. Writes an eval run only: never an evaluation, never traffic, never acceptance. A preview does not register the candidate: before evaluating by its structure hash, register it with POST /v1/decisions (or send the structure inline on evaluate once). The base, in order: the base of `suggestion_id` when it is sent (then `base` is ignored), else `base`, else the version serving primary traffic in the key's environment, else the newest one examples were imported under there, else none. With `suggestion_id`, `hashes_match` says whether the structure is that suggestion's candidate; another environment's suggestion is 404. The evidence replays only cases with an outcome from the owner, reviewers or imported examples; the action mix replays a sample of the environment's recent primary traffic. For questions the candidate changes, `before` is a fresh re-run of the base question (the fair baseline), so model nondeterminism can move it between previews. `noise_rate` measures outcome flips from that re-run against the stored answers, not value drift. A test key's preview neither reads nor fills the shared cache of model answers. A decision nothing was registered, evaluated or imported under is 404.","parameters":["a6e7e494-af3f-4770-a9b0-988847180957","8f202223-5bd6-4ee0-918c-a57a02ef6e58"],"requestBody":"1762e27f-0d39-4a22-82ef-1069e5a50bee","responses":{"200":"6e969e10-0c4c-4d36-b1e1-6bdf5128a1a8","400":"d92c086e-b85e-4d96-9081-0d576e9fa7ce","401":"966254c0-16c3-4273-9dad-e090d29da618","403":"436544e6-8438-45d0-a099-8bd404440201","404":"e4d2bb6c-d819-40b3-9ab7-13c366029600","409":"1930a3e4-d6ee-4aaf-b62f-57a172f11d39","410":"57c6da5a-3a4e-4691-832e-a8a3a4834739","413":"72572673-c7e8-4d26-80b7-8be239249c1c","422":"4c13e08f-6ad0-49c0-add0-a2a6daeaeb33","429":"f387853d-bba6-4e79-accb-261a8d5dd01c","502":"109e198e-46f0-48e9-9c58-7f6a7905ad05","503":"8bd3be84-bfbb-4492-a3d1-8a6e0b9b6861"},"method":"post","type":"path","servers":["48eee345-ee42-4a1e-aa8f-c5d5e9868495"],"path":"/v1/decisions/{decision_id}/previews","baseUrlOptions":["https://api.moonbounce.ai"],"title":"Preview a structure change against past feedback","codeSamples":[],"requiredOnlyExamples":false,"requestExampleType":"generated","responseExampleType":"generated"},"dependencies":{"parameters":{"a6e7e494-af3f-4770-a9b0-988847180957":{"schema":{"type":"string","pattern":"^[a-z][a-z0-9_.-]{0,127}$","description":"A name you choose for this decision, such as `refund-exception`: a lowercase letter, then up to 127 of `a-z 0-9 _ . -`. Unique in your organization and shared by its test and production keys. The first registration or evaluation under a new one creates the decision.","isRequired":true,"uniqueKey":"","typeLabel":"string"},"required":true,"description":"A name you choose for this decision, such as `refund-exception`: a lowercase letter, then up to 127 of `a-z 0-9 _ . -`. Unique in your organization and shared by its test and production keys. The first registration or evaluation under a new one creates the decision.","name":"decision_id","in":"path"},"8f202223-5bd6-4ee0-918c-a57a02ef6e58":{"schema":{"type":"string","minLength":1,"maxLength":255,"pattern":"^[\\x20-\\x7e]+$","description":"Makes a retry safe. For 24 hours, the same key with the same request returns the first stored result (with Idempotent-Replayed: true); the same key with a different request is a 422 IDEMPOTENCY_KEY_REUSED. Keys are scoped to the organization and environment across every operation, so use a fresh key per request. A request that failed before its result was stored runs again when retried with its key.","isRequired":false,"uniqueKey":"","typeLabel":"string"},"required":false,"description":"Makes a retry safe. For 24 hours, the same key with the same request returns the first stored result (with Idempotent-Replayed: true); the same key with a different request is a 422 IDEMPOTENCY_KEY_REUSED. Keys are scoped to the organization and environment across every operation, so use a fresh key per request. A request that failed before its result was stored runs again when retried with its key.","name":"idempotency-key","in":"header"}},"requestBody":{"required":true,"content":{"application/json":{"examples":{"new_question":{"title":"new_question","value":{"structure":{"questions":{"item_used":{"type":"noul","instructions":"Has the customer used the item beyond trying it on or setting it up once?","reads":["state.message"]},"defective":{"type":"noul","instructions":"Does the customer describe a defect, damage on arrival, or the item failing under normal use?","reads":["state.message"]},"shipping_failure":{"type":"noul","instructions":"Did the order arrive later than promised, or get lost in transit?","reads":["state.message"]}},"attributes":{"days_since_delivery":{"type":"number","minimum":0}},"aggregations":{"grant_score":{"type":"weighted_mean","inputs":[{"name":"questions.item_used","invert":true},{"name":"questions.defective"},{"name":"questions.shipping_failure"}]},"handling":{"type":"rules","rules":[{"when":{"name":"attributes.days_since_delivery","gt":30},"outcome":"escalate"},{"when":{"name":"aggregations.grant_score","gte":0.72},"outcome":"grant"},{"when":{"name":"aggregations.grant_score","lt":0.38},"outcome":"deny"}],"else":"escalate"}},"result":"aggregations.handling"}}}},"schema":{"type":"object","properties":{"structure":{"type":"object","properties":{"questions":{"type":"object","additionalProperties":{"type":"object","description":"P(true) for a statement about the state.","required":["type","instructions"],"properties":{"type":{"type":"string","enum":["noul"],"isRequired":true,"uniqueKey":"structure.questions.additionalProperties.type","typeLabel":"enum\u003cstring>"},"instructions":{"oneOf":[{"type":"string","maxLength":4000,"minLength":1,"isRequired":true,"uniqueKey":"structure.questions.additionalProperties.instructions.oneOf.0","typeLabel":"string"},{"type":"object","additionalProperties":{"oneOf":[{}],"uniqueKey":"structure.questions.additionalProperties.instructions.oneOf.1.additionalProperties","typeLabel":"any"},"isRequired":true,"uniqueKey":"structure.questions.additionalProperties.instructions.oneOf.1","typeLabel":"object"},{"type":"array","items":{"oneOf":[{}],"uniqueKey":"structure.questions.additionalProperties.instructions.oneOf.2.items.[INDEX]","typeLabel":"any"},"minItems":1,"isRequired":true,"uniqueKey":"structure.questions.additionalProperties.instructions.oneOf.2","typeLabel":"any[]"}],"isRequired":true,"uniqueKey":"structure.questions.additionalProperties.instructions","typeLabel":"string | object | any[]"},"criteria":{"type":"object","properties":{"true":{"type":"string","minLength":1,"maxLength":4000,"uniqueKey":"structure.questions.additionalProperties.criteria.true","typeLabel":"string"},"false":{"type":"string","minLength":1,"maxLength":4000,"uniqueKey":"structure.questions.additionalProperties.criteria.false","typeLabel":"string"}},"additionalProperties":false,"uniqueKey":"structure.questions.additionalProperties.criteria","typeLabel":"object"},"reads":{"type":"array","items":{"type":"string","minLength":1,"maxLength":256,"description":"`state.\u003ckey>` or a nested object path such as `state.order.customer.tier`: keys of an object `state`. A key has no `.`, `[` or `]`. Array indexes are not supported yet.","examples":["state.message","state.order.order_total_usd"],"uniqueKey":"structure.questions.additionalProperties.reads.items.[INDEX]","typeLabel":"string"},"minItems":1,"maxItems":64,"description":"Paths into an object `state` this question reads, such as `state.message` or `state.order.customer.tier`. The question sees just those values, nested as in the state. Without it the question sees the whole state, as in System One. Every path must exist in the call's `state`; no entry may repeat or contain another.","uniqueKey":"structure.questions.additionalProperties.reads","typeLabel":"string[]"}},"additionalProperties":false,"uniqueKey":"structure.questions.additionalProperties","typeLabel":"object"},"description":"System One questions, keyed by name.","isRequired":true,"uniqueKey":"structure.questions","typeLabel":"object"},"result":{"type":"string","minLength":1,"maxLength":80,"description":"The final feature's path, such as `aggregations.band`.","examples":["questions.asks_for_refund","aggregations.band"],"isRequired":true,"uniqueKey":"structure.result","typeLabel":"string"},"attributes":{"type":"object","additionalProperties":{"oneOf":[{"type":"object","required":["type","enum"],"properties":{"type":{"type":"string","enum":["string"],"isRequired":true,"uniqueKey":"structure.attributes.additionalProperties.oneOf.0.type","typeLabel":"enum\u003cstring>"},"enum":{"type":"array","items":{"type":"string","minLength":1,"maxLength":200,"uniqueKey":"structure.attributes.additionalProperties.oneOf.0.enum.items.[INDEX]","typeLabel":"string"},"minItems":1,"maxItems":255,"isRequired":true,"uniqueKey":"structure.attributes.additionalProperties.oneOf.0.enum","typeLabel":"string[]"},"default":{"type":"string","uniqueKey":"structure.attributes.additionalProperties.oneOf.0.default","typeLabel":"string"},"source":{"type":"string","minLength":1,"maxLength":200,"description":"The model and version behind this attribute, such as `toxicity-classifier-v3`. Hashed.","uniqueKey":"structure.attributes.additionalProperties.oneOf.0.source","typeLabel":"string"},"from":{"type":"string","minLength":1,"maxLength":256,"description":"A state path (as in `reads`) evaluate fills this attribute from, in place of `attribute_values`: the value at that path, validated by this declaration with no coercion. A missing or null path takes `default`, else it is a 400. A value for it in `attribute_values` is a 400. Hashed as written.","examples":["state.message","state.order.order_total_usd"],"uniqueKey":"structure.attributes.additionalProperties.oneOf.0.from","typeLabel":"string"}},"additionalProperties":false,"uniqueKey":"structure.attributes.additionalProperties.oneOf.0","typeLabel":"object"},{"type":"object","required":["type"],"properties":{"type":{"type":"string","enum":["number"],"isRequired":true,"uniqueKey":"structure.attributes.additionalProperties.oneOf.1.type","typeLabel":"enum\u003cstring>"},"minimum":{"type":"number","uniqueKey":"structure.attributes.additionalProperties.oneOf.1.minimum","typeLabel":"number"},"maximum":{"type":"number","uniqueKey":"structure.attributes.additionalProperties.oneOf.1.maximum","typeLabel":"number"},"default":{"type":"number","uniqueKey":"structure.attributes.additionalProperties.oneOf.1.default","typeLabel":"number"},"source":{"type":"string","minLength":1,"maxLength":200,"description":"The model and version behind this attribute, such as `toxicity-classifier-v3`. Hashed.","uniqueKey":"structure.attributes.additionalProperties.oneOf.1.source","typeLabel":"string"},"from":{"type":"string","minLength":1,"maxLength":256,"description":"A state path (as in `reads`) evaluate fills this attribute from, in place of `attribute_values`: the value at that path, validated by this declaration with no coercion. A missing or null path takes `default`, else it is a 400. A value for it in `attribute_values` is a 400. Hashed as written.","examples":["state.message","state.order.order_total_usd"],"uniqueKey":"structure.attributes.additionalProperties.oneOf.1.from","typeLabel":"string"}},"additionalProperties":false,"uniqueKey":"structure.attributes.additionalProperties.oneOf.1","typeLabel":"object"},{"type":"object","required":["type"],"properties":{"type":{"type":"string","enum":["boolean"],"isRequired":true,"uniqueKey":"structure.attributes.additionalProperties.oneOf.2.type","typeLabel":"enum\u003cstring>"},"default":{"type":"boolean","uniqueKey":"structure.attributes.additionalProperties.oneOf.2.default","typeLabel":"boolean"},"source":{"type":"string","minLength":1,"maxLength":200,"description":"The model and version behind this attribute, such as `toxicity-classifier-v3`. Hashed.","uniqueKey":"structure.attributes.additionalProperties.oneOf.2.source","typeLabel":"string"},"from":{"type":"string","minLength":1,"maxLength":256,"description":"A state path (as in `reads`) evaluate fills this attribute from, in place of `attribute_values`: the value at that path, validated by this declaration with no coercion. A missing or null path takes `default`, else it is a 400. A value for it in `attribute_values` is a 400. Hashed as written.","examples":["state.message","state.order.order_total_usd"],"uniqueKey":"structure.attributes.additionalProperties.oneOf.2.from","typeLabel":"string"}},"additionalProperties":false,"uniqueKey":"structure.attributes.additionalProperties.oneOf.2","typeLabel":"object"}],"uniqueKey":"structure.attributes.additionalProperties","typeLabel":"object"},"description":"Values your app supplies with each call in `attribute_values`. The model never sees them.","uniqueKey":"structure.attributes","typeLabel":"object"},"aggregations":{"type":"object","additionalProperties":{"oneOf":[{"type":"object","description":"Σ wᵢ·xᵢ / Σ wᵢ: a weighted average from 0 to 1, not a probability.","required":["type","inputs"],"properties":{"type":{"type":"string","enum":["weighted_mean"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.0.type","typeLabel":"enum\u003cstring>"},"inputs":{"type":"array","items":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":80,"description":"A feature's path: `questions.\u003cname>`, `attributes.\u003cname>` or `aggregations.\u003cname>`.","examples":["questions.asks_for_refund","aggregations.band"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.0.inputs.items.[INDEX].name","typeLabel":"string"},"weight":{"type":"number","exclusiveMinimum":0,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.0.inputs.items.[INDEX].weight","typeLabel":"number"},"invert":{"type":"boolean","uniqueKey":"structure.aggregations.additionalProperties.oneOf.0.inputs.items.[INDEX].invert","typeLabel":"boolean"}},"required":["name"],"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.0.inputs.items.[INDEX]","typeLabel":"object"},"minItems":1,"maxItems":64,"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.0.inputs","typeLabel":"object[]"}},"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.0","typeLabel":"object"},{"type":"object","description":"σ(bias + Σ wᵢ·xᵢ), with signed weights on raw input values.","required":["type","bias","inputs"],"properties":{"type":{"type":"string","enum":["logistic"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.1.type","typeLabel":"enum\u003cstring>"},"bias":{"type":"number","isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.1.bias","typeLabel":"number"},"inputs":{"type":"array","items":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":80,"description":"A feature's path: `questions.\u003cname>`, `attributes.\u003cname>` or `aggregations.\u003cname>`.","examples":["questions.asks_for_refund","aggregations.band"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.1.inputs.items.[INDEX].name","typeLabel":"string"},"weight":{"type":"number","isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.1.inputs.items.[INDEX].weight","typeLabel":"number"}},"required":["name","weight"],"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.1.inputs.items.[INDEX]","typeLabel":"object"},"minItems":1,"maxItems":64,"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.1.inputs","typeLabel":"object[]"}},"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.1","typeLabel":"object"},{"type":"object","description":"min xᵢ: all of these, when they overlap.","required":["type","inputs"],"properties":{"type":{"type":"string","enum":["min"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.2.type","typeLabel":"enum\u003cstring>"},"inputs":{"type":"array","items":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":80,"description":"A feature's path: `questions.\u003cname>`, `attributes.\u003cname>` or `aggregations.\u003cname>`.","examples":["questions.asks_for_refund","aggregations.band"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.2.inputs.items.[INDEX].name","typeLabel":"string"},"invert":{"type":"boolean","uniqueKey":"structure.aggregations.additionalProperties.oneOf.2.inputs.items.[INDEX].invert","typeLabel":"boolean"}},"required":["name"],"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.2.inputs.items.[INDEX]","typeLabel":"object"},"minItems":1,"maxItems":64,"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.2.inputs","typeLabel":"object[]"}},"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.2","typeLabel":"object"},{"type":"object","description":"max xᵢ: any of these, when they overlap.","required":["type","inputs"],"properties":{"type":{"type":"string","enum":["max"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.3.type","typeLabel":"enum\u003cstring>"},"inputs":{"type":"array","items":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":80,"description":"A feature's path: `questions.\u003cname>`, `attributes.\u003cname>` or `aggregations.\u003cname>`.","examples":["questions.asks_for_refund","aggregations.band"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.3.inputs.items.[INDEX].name","typeLabel":"string"},"invert":{"type":"boolean","uniqueKey":"structure.aggregations.additionalProperties.oneOf.3.inputs.items.[INDEX].invert","typeLabel":"boolean"}},"required":["name"],"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.3.inputs.items.[INDEX]","typeLabel":"object"},"minItems":1,"maxItems":64,"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.3.inputs","typeLabel":"object[]"}},"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.3","typeLabel":"object"},{"type":"object","description":"Π xᵢ: all of these, when they are independent.","required":["type","inputs"],"properties":{"type":{"type":"string","enum":["product"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.4.type","typeLabel":"enum\u003cstring>"},"inputs":{"type":"array","items":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":80,"description":"A feature's path: `questions.\u003cname>`, `attributes.\u003cname>` or `aggregations.\u003cname>`.","examples":["questions.asks_for_refund","aggregations.band"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.4.inputs.items.[INDEX].name","typeLabel":"string"},"invert":{"type":"boolean","uniqueKey":"structure.aggregations.additionalProperties.oneOf.4.inputs.items.[INDEX].invert","typeLabel":"boolean"}},"required":["name"],"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.4.inputs.items.[INDEX]","typeLabel":"object"},"minItems":1,"maxItems":64,"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.4.inputs","typeLabel":"object[]"}},"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.4","typeLabel":"object"},{"type":"object","description":"1 − Π(1 − xᵢ): any of these, when they are independent.","required":["type","inputs"],"properties":{"type":{"type":"string","enum":["noisy_or"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.5.type","typeLabel":"enum\u003cstring>"},"inputs":{"type":"array","items":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":80,"description":"A feature's path: `questions.\u003cname>`, `attributes.\u003cname>` or `aggregations.\u003cname>`.","examples":["questions.asks_for_refund","aggregations.band"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.5.inputs.items.[INDEX].name","typeLabel":"string"},"invert":{"type":"boolean","uniqueKey":"structure.aggregations.additionalProperties.oneOf.5.inputs.items.[INDEX].invert","typeLabel":"boolean"}},"required":["name"],"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.5.inputs.items.[INDEX]","typeLabel":"object"},"minItems":1,"maxItems":64,"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.5.inputs","typeLabel":"object[]"}},"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.5","typeLabel":"object"},{"type":"object","description":"1 when every input passes its threshold (default ≥ 0.5), else 0.","required":["type","inputs"],"properties":{"type":{"type":"string","enum":["all"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.6.type","typeLabel":"enum\u003cstring>"},"inputs":{"type":"array","items":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":80,"description":"A feature's path: `questions.\u003cname>`, `attributes.\u003cname>` or `aggregations.\u003cname>`.","examples":["questions.asks_for_refund","aggregations.band"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.6.inputs.items.[INDEX].name","typeLabel":"string"},"gte":{"type":"number","uniqueKey":"structure.aggregations.additionalProperties.oneOf.6.inputs.items.[INDEX].gte","typeLabel":"number"},"gt":{"type":"number","uniqueKey":"structure.aggregations.additionalProperties.oneOf.6.inputs.items.[INDEX].gt","typeLabel":"number"},"lte":{"type":"number","uniqueKey":"structure.aggregations.additionalProperties.oneOf.6.inputs.items.[INDEX].lte","typeLabel":"number"},"lt":{"type":"number","uniqueKey":"structure.aggregations.additionalProperties.oneOf.6.inputs.items.[INDEX].lt","typeLabel":"number"}},"required":["name"],"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.6.inputs.items.[INDEX]","typeLabel":"object"},"minItems":1,"maxItems":64,"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.6.inputs","typeLabel":"object[]"}},"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.6","typeLabel":"object"},{"type":"object","description":"1 when any input passes its threshold (default ≥ 0.5), else 0.","required":["type","inputs"],"properties":{"type":{"type":"string","enum":["any"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.7.type","typeLabel":"enum\u003cstring>"},"inputs":{"type":"array","items":{"type":"object","properties":{"name":{"type":"string","minLength":1,"maxLength":80,"description":"A feature's path: `questions.\u003cname>`, `attributes.\u003cname>` or `aggregations.\u003cname>`.","examples":["questions.asks_for_refund","aggregations.band"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.7.inputs.items.[INDEX].name","typeLabel":"string"},"gte":{"type":"number","uniqueKey":"structure.aggregations.additionalProperties.oneOf.7.inputs.items.[INDEX].gte","typeLabel":"number"},"gt":{"type":"number","uniqueKey":"structure.aggregations.additionalProperties.oneOf.7.inputs.items.[INDEX].gt","typeLabel":"number"},"lte":{"type":"number","uniqueKey":"structure.aggregations.additionalProperties.oneOf.7.inputs.items.[INDEX].lte","typeLabel":"number"},"lt":{"type":"number","uniqueKey":"structure.aggregations.additionalProperties.oneOf.7.inputs.items.[INDEX].lt","typeLabel":"number"}},"required":["name"],"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.7.inputs.items.[INDEX]","typeLabel":"object"},"minItems":1,"maxItems":64,"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.7.inputs","typeLabel":"object[]"}},"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.7","typeLabel":"object"},{"type":"object","description":"Names an outcome for each range of one feature's value. `at` means ≥; `below` is under the first cut.","required":["type","input","below","cuts"],"properties":{"type":{"type":"string","enum":["bands"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.8.type","typeLabel":"enum\u003cstring>"},"input":{"type":"string","minLength":1,"maxLength":80,"description":"A feature's path: `questions.\u003cname>`, `attributes.\u003cname>` or `aggregations.\u003cname>`.","examples":["questions.asks_for_refund","aggregations.band"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.8.input","typeLabel":"string"},"below":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,63}$","isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.8.below","typeLabel":"string"},"cuts":{"type":"array","items":{"type":"object","properties":{"at":{"type":"number","isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.8.cuts.items.[INDEX].at","typeLabel":"number"},"outcome":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,63}$","isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.8.cuts.items.[INDEX].outcome","typeLabel":"string"}},"required":["at","outcome"],"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.8.cuts.items.[INDEX]","typeLabel":"object"},"minItems":1,"maxItems":9,"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.8.cuts","typeLabel":"object[]"},"abstain":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,63}$","uniqueKey":"structure.aggregations.additionalProperties.oneOf.8.abstain","typeLabel":"string"}},"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.8","typeLabel":"object"},{"type":"object","description":"The first rule whose condition is true names its `outcome`; otherwise `else`.","required":["type","rules","else"],"properties":{"type":{"type":"string","enum":["rules"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.type","typeLabel":"enum\u003cstring>"},"rules":{"type":"array","items":{"type":"object","properties":{"when":{"oneOf":[{"type":"object","required":["name"],"properties":{"name":{"type":"string","minLength":1,"maxLength":80,"description":"A feature's path: `questions.\u003cname>`, `attributes.\u003cname>` or `aggregations.\u003cname>`.","examples":["questions.asks_for_refund","aggregations.band"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].when.oneOf.0.name","typeLabel":"string"},"gte":{"type":"number","uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].when.oneOf.0.gte","typeLabel":"number"},"gt":{"type":"number","uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].when.oneOf.0.gt","typeLabel":"number"},"lte":{"type":"number","uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].when.oneOf.0.lte","typeLabel":"number"},"lt":{"type":"number","uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].when.oneOf.0.lt","typeLabel":"number"},"is":{"type":"string","minLength":1,"maxLength":200,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].when.oneOf.0.is","typeLabel":"string"},"in":{"type":"array","items":{"type":"string","minLength":1,"maxLength":200,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].when.oneOf.0.in.items.[INDEX]","typeLabel":"string"},"minItems":1,"maxItems":255,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].when.oneOf.0.in","typeLabel":"string[]"},"abstained":{"type":"boolean","enum":[true],"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].when.oneOf.0.abstained","typeLabel":"enum\u003cboolean>"}},"additionalProperties":false,"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].when.oneOf.0","typeLabel":"object"},{"type":"object","required":["all"],"properties":{"all":{"type":"array","items":{"$circularRef":"889c258c-6d37-45b1-895e-cb853a5b9e3e"},"minItems":1,"maxItems":64,"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].when.oneOf.1.all","typeLabel":"array"}},"additionalProperties":false,"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].when.oneOf.1","typeLabel":"object"},{"type":"object","required":["any"],"properties":{"any":{"type":"array","items":{"$circularRef":"889c258c-6d37-45b1-895e-cb853a5b9e3e"},"minItems":1,"maxItems":64,"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].when.oneOf.2.any","typeLabel":"array"}},"additionalProperties":false,"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].when.oneOf.2","typeLabel":"object"},{"type":"object","required":["not"],"properties":{"not":{"oneOf":[{"$circularRef":"889c258c-6d37-45b1-895e-cb853a5b9e3e","isRequired":true}],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].when.oneOf.3.not","typeLabel":"any"}},"additionalProperties":false,"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].when.oneOf.3","typeLabel":"object"}],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].when","typeLabel":"object"},"outcome":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,63}$","isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX].outcome","typeLabel":"string"}},"required":["when","outcome"],"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules.items.[INDEX]","typeLabel":"object"},"minItems":1,"maxItems":64,"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.rules","typeLabel":"object[]"},"else":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,63}$","isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9.else","typeLabel":"string"}},"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.9","typeLabel":"object"},{"type":"object","description":"Maps a label straight to an outcome.","required":["type","input","cases"],"properties":{"type":{"type":"string","enum":["match"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.10.type","typeLabel":"enum\u003cstring>"},"input":{"type":"string","minLength":1,"maxLength":80,"description":"A feature's path: `questions.\u003cname>`, `attributes.\u003cname>` or `aggregations.\u003cname>`.","examples":["questions.asks_for_refund","aggregations.band"],"isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.10.input","typeLabel":"string"},"cases":{"type":"object","additionalProperties":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,63}$","uniqueKey":"structure.aggregations.additionalProperties.oneOf.10.cases.additionalProperties","typeLabel":"string"},"description":"Maps labels to outcomes. \"__proto__\" is not supported as a case key; use a rules is-condition for that label.","isRequired":true,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.10.cases","typeLabel":"object"},"else":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,63}$","uniqueKey":"structure.aggregations.additionalProperties.oneOf.10.else","typeLabel":"string"},"abstain":{"type":"string","pattern":"^[a-z][a-z0-9_]{0,63}$","uniqueKey":"structure.aggregations.additionalProperties.oneOf.10.abstain","typeLabel":"string"}},"additionalProperties":false,"uniqueKey":"structure.aggregations.additionalProperties.oneOf.10","typeLabel":"object"}],"uniqueKey":"structure.aggregations.additionalProperties","typeLabel":"object"},"uniqueKey":"structure.aggregations","typeLabel":"object"}},"required":["questions","result"],"additionalProperties":false,"description":"Everything that defines a decision: questions, attribute declarations, aggregations and the result. Names are unique across all three maps.","isRequired":true,"uniqueKey":"structure","typeLabel":"object"},"suggestion_id":{"type":"string","pattern":"^sug_[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","uniqueKey":"suggestion_id","typeLabel":"string"},"base":{"type":"object","properties":{"structure_hash":{"type":"string","pattern":"^sha256:[0-9a-f]{64}$","isRequired":true,"uniqueKey":"base.structure_hash","typeLabel":"string"}},"required":["structure_hash"],"additionalProperties":false,"uniqueKey":"base","typeLabel":"object"}},"required":["structure"],"additionalProperties":false,"uniqueKey":"","typeLabel":"object"},"exampleType":"examples"}},"exampleType":"examples"},"responses":{"6e969e10-0c4c-4d36-b1e1-6bdf5128a1a8":{"description":"The preview","headers":{"Idempotent-Replayed":{"schema":{"type":"string","enum":["true"],"description":"Present and `true` when this is the stored result of an earlier request with the same Idempotency-Key."},"required":false,"description":"Present and `true` when this is the stored result of an earlier request with the same Idempotency-Key."}},"content":{"application/json":{"schema":{"type":"object","properties":{"preview_id":{"type":"string","pattern":"^run_[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","isRequired":true,"uniqueKey":"preview_id","typeLabel":"string"},"decision_id":{"type":"string","pattern":"^[a-z][a-z0-9_.-]{0,127}$","description":"A name you choose for this decision, such as `refund-exception`: a lowercase letter, then up to 127 of `a-z 0-9 _ . -`. Unique in your organization and shared by its test and production keys. The first registration or evaluation under a new one creates the decision.","isRequired":true,"uniqueKey":"decision_id","typeLabel":"string"},"environment":{"type":"string","enum":["production","test"],"description":"The environment of the key that made the call: `production` or `test`. Evaluations, feedback, reviews and suggestions are separate per environment; decisions and their versions belong to the whole organization.","isRequired":true,"uniqueKey":"environment","typeLabel":"enum\u003cstring>"},"base":{"oneOf":[{"type":"object","required":["version","model"],"x-moonbounce-sdk-known-fields":true,"properties":{"version":{"type":"object","properties":{"structure_hash":{"type":"string","pattern":"^sha256:[0-9a-f]{64}$","isRequired":true,"uniqueKey":"base.oneOf.0.version.structure_hash","typeLabel":"string"},"number":{"oneOf":[{"type":"integer","minimum":1,"description":"The version's number under the decision (1, 2, 3...). Null when the structure isn't registered: a dry run or a preview of a structure nobody registered. The hash is the identity.","isRequired":true,"uniqueKey":"base.oneOf.0.version.number.oneOf.0","typeLabel":"integer"},{"type":"null","minimum":1,"description":"The version's number under the decision (1, 2, 3...). Null when the structure isn't registered: a dry run or a preview of a structure nobody registered. The hash is the identity.","isRequired":true,"uniqueKey":"base.oneOf.0.version.number.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"base.oneOf.0.version.number","typeLabel":"integer | null"}},"required":["structure_hash","number"],"additionalProperties":true,"description":"The version that ran.","x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"base.oneOf.0.version","typeLabel":"object"},"model":{"type":"object","properties":{"profile":{"type":"string","description":"Every question's model and connector, as one opaque string. Compare it for equality (two runs are comparable when their profiles match); don't parse it.","isRequired":true,"uniqueKey":"base.oneOf.0.model.profile","typeLabel":"string"}},"required":["profile"],"additionalProperties":true,"description":"The model that answered, as its opaque profile.","x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"base.oneOf.0.model","typeLabel":"object"}},"additionalProperties":true,"isRequired":true,"uniqueKey":"base.oneOf.0","typeLabel":"object"},{"type":"null","isRequired":true,"uniqueKey":"base.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"base","typeLabel":"object | null"},"candidate":{"type":"object","properties":{"version":{"type":"object","properties":{"structure_hash":{"type":"string","pattern":"^sha256:[0-9a-f]{64}$","isRequired":true,"uniqueKey":"candidate.version.structure_hash","typeLabel":"string"},"number":{"oneOf":[{"type":"integer","minimum":1,"description":"The version's number under the decision (1, 2, 3...). Null when the structure isn't registered: a dry run or a preview of a structure nobody registered. The hash is the identity.","isRequired":true,"uniqueKey":"candidate.version.number.oneOf.0","typeLabel":"integer"},{"type":"null","minimum":1,"description":"The version's number under the decision (1, 2, 3...). Null when the structure isn't registered: a dry run or a preview of a structure nobody registered. The hash is the identity.","isRequired":true,"uniqueKey":"candidate.version.number.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"candidate.version.number","typeLabel":"integer | null"}},"required":["structure_hash","number"],"additionalProperties":true,"description":"The version that ran.","x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"candidate.version","typeLabel":"object"},"model":{"type":"object","properties":{"profile":{"type":"string","description":"Every question's model and connector, as one opaque string. Compare it for equality (two runs are comparable when their profiles match); don't parse it.","isRequired":true,"uniqueKey":"candidate.model.profile","typeLabel":"string"}},"required":["profile"],"additionalProperties":true,"description":"The model that answered, as its opaque profile.","x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"candidate.model","typeLabel":"object"}},"required":["version","model"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"candidate","typeLabel":"object"},"hashes_match":{"oneOf":[{"type":"object","required":["structure"],"x-moonbounce-sdk-known-fields":true,"properties":{"structure":{"type":"boolean","isRequired":true,"uniqueKey":"hashes_match.oneOf.0.structure","typeLabel":"boolean"}},"additionalProperties":true,"isRequired":true,"uniqueKey":"hashes_match.oneOf.0","typeLabel":"object"},{"type":"null","isRequired":true,"uniqueKey":"hashes_match.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"hashes_match","typeLabel":"object | null"},"evidence":{"type":"object","properties":{"label":{"type":"string","enum":["imported_examples","training_cases"],"isRequired":true,"uniqueKey":"evidence.label","typeLabel":"enum\u003cstring>"},"frozen_at":{"type":"string","format":"date-time","isRequired":true,"uniqueKey":"evidence.frozen_at","typeLabel":"string\u003cdate-time>"},"by_split":{"type":"object","properties":{"example":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_split.example.n","typeLabel":"integer"},"fixes":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_split.example.fixes","typeLabel":"integer"},"breaks":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_split.example.breaks","typeLabel":"integer"},"base_correct":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_split.example.base_correct","typeLabel":"integer"},"candidate_correct":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_split.example.candidate_correct","typeLabel":"integer"}},"required":["n","fixes","breaks","base_correct","candidate_correct"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"evidence.by_split.example","typeLabel":"object"},"train":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_split.train.n","typeLabel":"integer"},"fixes":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_split.train.fixes","typeLabel":"integer"},"breaks":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_split.train.breaks","typeLabel":"integer"},"base_correct":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_split.train.base_correct","typeLabel":"integer"},"candidate_correct":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_split.train.candidate_correct","typeLabel":"integer"}},"required":["n","fixes","breaks","base_correct","candidate_correct"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"evidence.by_split.train","typeLabel":"object"}},"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"evidence.by_split","typeLabel":"object"},"noise_rate":{"oneOf":[{"type":"number","minimum":0,"maximum":1,"isRequired":true,"uniqueKey":"evidence.noise_rate.oneOf.0","typeLabel":"number"},{"type":"null","minimum":0,"maximum":1,"isRequired":true,"uniqueKey":"evidence.noise_rate.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"evidence.noise_rate","typeLabel":"number | null"},"action_mix":{"oneOf":[{"type":"object","required":["before","after","on","n"],"x-moonbounce-sdk-known-fields":true,"properties":{"before":{"type":"object","additionalProperties":{"type":"number","minimum":0,"maximum":1,"uniqueKey":"evidence.action_mix.oneOf.0.before.additionalProperties","typeLabel":"number"},"isRequired":true,"uniqueKey":"evidence.action_mix.oneOf.0.before","typeLabel":"object"},"after":{"type":"object","additionalProperties":{"type":"number","minimum":0,"maximum":1,"uniqueKey":"evidence.action_mix.oneOf.0.after.additionalProperties","typeLabel":"number"},"isRequired":true,"uniqueKey":"evidence.action_mix.oneOf.0.after","typeLabel":"object"},"on":{"type":"string","isRequired":true,"uniqueKey":"evidence.action_mix.oneOf.0.on","typeLabel":"string"},"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.action_mix.oneOf.0.n","typeLabel":"integer"}},"additionalProperties":true,"isRequired":true,"uniqueKey":"evidence.action_mix.oneOf.0","typeLabel":"object"},{"type":"null","isRequired":true,"uniqueKey":"evidence.action_mix.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"evidence.action_mix","typeLabel":"object | null"},"changed_items":{"type":"array","items":{"type":"object","properties":{"evaluation_id":{"type":"string","pattern":"^evl_[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","isRequired":true,"uniqueKey":"evidence.changed_items.items.[INDEX].evaluation_id","typeLabel":"string"},"feedback_id":{"type":"string","pattern":"^fbk_[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","isRequired":true,"uniqueKey":"evidence.changed_items.items.[INDEX].feedback_id","typeLabel":"string"},"split":{"type":"string","enum":["example","train"],"isRequired":true,"uniqueKey":"evidence.changed_items.items.[INDEX].split","typeLabel":"enum\u003cstring>"},"change":{"type":"string","enum":["fix","break"],"isRequired":true,"uniqueKey":"evidence.changed_items.items.[INDEX].change","typeLabel":"enum\u003cstring>"},"before":{"type":"object","properties":{"answers":{"type":"object","additionalProperties":{"oneOf":[{"type":"number","minimum":0,"maximum":1,"uniqueKey":"evidence.changed_items.items.[INDEX].before.answers.additionalProperties.oneOf.0","typeLabel":"number"},{"type":"null","minimum":0,"maximum":1,"uniqueKey":"evidence.changed_items.items.[INDEX].before.answers.additionalProperties.oneOf.1","typeLabel":"null"}],"uniqueKey":"evidence.changed_items.items.[INDEX].before.answers.additionalProperties","typeLabel":"number | null"},"isRequired":true,"uniqueKey":"evidence.changed_items.items.[INDEX].before.answers","typeLabel":"object"},"outcome":{"type":"string","minLength":1,"maxLength":200,"description":"An outcome name, or the exact enum label when a string attribute is the result.","uniqueKey":"evidence.changed_items.items.[INDEX].before.outcome","typeLabel":"string"},"value":{"type":"number","uniqueKey":"evidence.changed_items.items.[INDEX].before.value","typeLabel":"number"}},"required":["answers"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"evidence.changed_items.items.[INDEX].before","typeLabel":"object"},"after":{"type":"object","properties":{"answers":{"type":"object","additionalProperties":{"oneOf":[{"type":"number","minimum":0,"maximum":1,"uniqueKey":"evidence.changed_items.items.[INDEX].after.answers.additionalProperties.oneOf.0","typeLabel":"number"},{"type":"null","minimum":0,"maximum":1,"uniqueKey":"evidence.changed_items.items.[INDEX].after.answers.additionalProperties.oneOf.1","typeLabel":"null"}],"uniqueKey":"evidence.changed_items.items.[INDEX].after.answers.additionalProperties","typeLabel":"number | null"},"isRequired":true,"uniqueKey":"evidence.changed_items.items.[INDEX].after.answers","typeLabel":"object"},"outcome":{"type":"string","minLength":1,"maxLength":200,"description":"An outcome name, or the exact enum label when a string attribute is the result.","uniqueKey":"evidence.changed_items.items.[INDEX].after.outcome","typeLabel":"string"},"value":{"type":"number","uniqueKey":"evidence.changed_items.items.[INDEX].after.value","typeLabel":"number"}},"required":["answers"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"evidence.changed_items.items.[INDEX].after","typeLabel":"object"}},"required":["evaluation_id","feedback_id","split","change","before","after"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"evidence.changed_items.items.[INDEX]","typeLabel":"object"},"isRequired":true,"uniqueKey":"evidence.changed_items","typeLabel":"object[]"},"credited_feedback":{"type":"array","items":{"type":"string","pattern":"^fbk_[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","uniqueKey":"evidence.credited_feedback.items.[INDEX]","typeLabel":"string"},"isRequired":true,"uniqueKey":"evidence.credited_feedback","typeLabel":"string[]"},"by_source":{"type":"object","properties":{"app_native":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.app_native.n","typeLabel":"integer"},"fixes":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.app_native.fixes","typeLabel":"integer"},"breaks":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.app_native.breaks","typeLabel":"integer"},"base_correct":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.app_native.base_correct","typeLabel":"integer"},"candidate_correct":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.app_native.candidate_correct","typeLabel":"integer"}},"required":["n","fixes","breaks","base_correct","candidate_correct"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"evidence.by_source.app_native","typeLabel":"object"},"app_resolution":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.app_resolution.n","typeLabel":"integer"},"fixes":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.app_resolution.fixes","typeLabel":"integer"},"breaks":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.app_resolution.breaks","typeLabel":"integer"},"base_correct":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.app_resolution.base_correct","typeLabel":"integer"},"candidate_correct":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.app_resolution.candidate_correct","typeLabel":"integer"}},"required":["n","fixes","breaks","base_correct","candidate_correct"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"evidence.by_source.app_resolution","typeLabel":"object"},"reviewer":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.reviewer.n","typeLabel":"integer"},"fixes":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.reviewer.fixes","typeLabel":"integer"},"breaks":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.reviewer.breaks","typeLabel":"integer"},"base_correct":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.reviewer.base_correct","typeLabel":"integer"},"candidate_correct":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.reviewer.candidate_correct","typeLabel":"integer"}},"required":["n","fixes","breaks","base_correct","candidate_correct"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"evidence.by_source.reviewer","typeLabel":"object"},"owner":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.owner.n","typeLabel":"integer"},"fixes":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.owner.fixes","typeLabel":"integer"},"breaks":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.owner.breaks","typeLabel":"integer"},"base_correct":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.owner.base_correct","typeLabel":"integer"},"candidate_correct":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.owner.candidate_correct","typeLabel":"integer"}},"required":["n","fixes","breaks","base_correct","candidate_correct"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"evidence.by_source.owner","typeLabel":"object"},"import":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.import.n","typeLabel":"integer"},"fixes":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.import.fixes","typeLabel":"integer"},"breaks":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.import.breaks","typeLabel":"integer"},"base_correct":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.import.base_correct","typeLabel":"integer"},"candidate_correct":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.import.candidate_correct","typeLabel":"integer"}},"required":["n","fixes","breaks","base_correct","candidate_correct"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"evidence.by_source.import","typeLabel":"object"},"synthetic_review":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.synthetic_review.n","typeLabel":"integer"},"fixes":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.synthetic_review.fixes","typeLabel":"integer"},"breaks":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.synthetic_review.breaks","typeLabel":"integer"},"base_correct":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.synthetic_review.base_correct","typeLabel":"integer"},"candidate_correct":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"evidence.by_source.synthetic_review.candidate_correct","typeLabel":"integer"}},"required":["n","fixes","breaks","base_correct","candidate_correct"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"evidence.by_source.synthetic_review","typeLabel":"object"}},"uniqueKey":"evidence.by_source","typeLabel":"object"}},"required":["label","frozen_at","by_split","noise_rate","action_mix","changed_items","credited_feedback"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"evidence","typeLabel":"object"},"cost":{"type":"object","properties":{"model_calls":{"type":"integer","minimum":0,"description":"Upstream model calls, retries included. One call can answer several questions (Jev compiles them into one).","isRequired":true,"uniqueKey":"cost.model_calls","typeLabel":"integer"},"questions":{"type":"integer","minimum":0,"description":"Questions sent to a model. An answer from the cache, or one a stored case kept, is not counted.","isRequired":true,"uniqueKey":"cost.questions","typeLabel":"integer"},"input_tokens":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"cost.input_tokens","typeLabel":"integer"},"output_tokens":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"cost.output_tokens","typeLabel":"integer"},"usd_micros":{"type":"integer","minimum":0,"description":"What the model calls cost, in millionths of a US dollar.","isRequired":true,"uniqueKey":"cost.usd_micros","typeLabel":"integer"},"cache_hits":{"type":"integer","minimum":0,"description":"Questions answered from the answer cache, when any were: previews and case runs always report it, evaluate and imports only when the cache served an answer.","uniqueKey":"cost.cache_hits","typeLabel":"integer"}},"required":["model_calls","questions","input_tokens","output_tokens","usd_micros"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"cost","typeLabel":"object"},"objective":{"oneOf":[{"type":"object","description":"The decision's stored objective revision this preview was scored with, or null when none is set (balanced losses). Absent only on a replay of a response stored before it was recorded.","required":["revision","objective_hash"],"x-moonbounce-sdk-known-fields":true,"properties":{"revision":{"type":"integer","minimum":1,"isRequired":true,"uniqueKey":"objective.oneOf.0.revision","typeLabel":"integer"},"objective_hash":{"type":"string","isRequired":true,"uniqueKey":"objective.oneOf.0.objective_hash","typeLabel":"string"}},"additionalProperties":true,"uniqueKey":"objective.oneOf.0","typeLabel":"object"},{"type":"null","description":"The decision's stored objective revision this preview was scored with, or null when none is set (balanced losses). Absent only on a replay of a response stored before it was recorded.","uniqueKey":"objective.oneOf.1","typeLabel":"null"}],"uniqueKey":"objective","typeLabel":"object | null"},"loss_report":{"type":"object","properties":{"objective":{"type":"object","properties":{"source":{"type":"string","enum":["stored","balanced"],"description":"`stored`: the decision's stored objective. `balanced`: none is set, so each outcome's mistakes weigh the same in total.","isRequired":true,"uniqueKey":"loss_report.objective.source","typeLabel":"enum\u003cstring>"},"revision":{"oneOf":[{"type":"integer","minimum":1,"isRequired":true,"uniqueKey":"loss_report.objective.revision.oneOf.0","typeLabel":"integer"},{"type":"null","minimum":1,"isRequired":true,"uniqueKey":"loss_report.objective.revision.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.objective.revision","typeLabel":"integer | null"},"objective_hash":{"oneOf":[{"type":"string","description":"The stored objective as written (null when balanced).","isRequired":true,"uniqueKey":"loss_report.objective.objective_hash.oneOf.0","typeLabel":"string"},{"type":"null","description":"The stored objective as written (null when balanced).","isRequired":true,"uniqueKey":"loss_report.objective.objective_hash.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.objective.objective_hash","typeLabel":"string | null"},"effective_hash":{"type":"string","description":"The hash of the severities scored by (each run weighs outcomes by its own cases; the target is left out, since it changes only `target_met`). Never compare reports under different ones.","isRequired":true,"uniqueKey":"loss_report.objective.effective_hash","typeLabel":"string"},"unknown_outcomes":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.objective.unknown_outcomes.items.[INDEX]","typeLabel":"string"},"isRequired":true,"uniqueKey":"loss_report.objective.unknown_outcomes","typeLabel":"string[]"}},"required":["source","revision","objective_hash","effective_hash","unknown_outcomes"],"additionalProperties":true,"description":"What this run scored with.","x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"loss_report.objective","typeLabel":"object"},"base":{"oneOf":[{"type":"object","description":"The base's report; null when there is no base.","required":["n","loss","loss_per_case","target_met","macro_f1","outcomes","by_mistake","by_mistake_total","by_question","missing_criterion","failed","near_cuts","severe","cases","cases_total","attributed"],"x-moonbounce-sdk-known-fields":true,"properties":{"n":{"type":"integer","minimum":0,"description":"Cases scored.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.n","typeLabel":"integer"},"loss":{"type":"number","minimum":0,"description":"Total loss of the mistakes.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.loss","typeLabel":"number"},"loss_per_case":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.loss_per_case","typeLabel":"number"},"target_met":{"oneOf":[{"type":"boolean","description":"Whether loss per case is at or below the objective's target; null without one.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.target_met.oneOf.0","typeLabel":"boolean"},{"type":"null","description":"Whether loss per case is at or below the objective's target; null without one.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.target_met.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.target_met","typeLabel":"boolean | null"},"macro_f1":{"oneOf":[{"type":"number","minimum":0,"maximum":1,"description":"Mean F1 over the outcomes expected or given: the headline. Not the loss.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.macro_f1.oneOf.0","typeLabel":"number"},{"type":"null","minimum":0,"maximum":1,"description":"Mean F1 over the outcomes expected or given: the headline. Not the loss.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.macro_f1.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.macro_f1","typeLabel":"number | null"},"outcomes":{"type":"array","items":{"type":"object","properties":{"outcome":{"type":"string","minLength":1,"maxLength":200,"description":"An outcome name, or the exact enum label when a string attribute is the result.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.outcomes.items.[INDEX].outcome","typeLabel":"string"},"expected":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.outcomes.items.[INDEX].expected","typeLabel":"integer"},"given":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.outcomes.items.[INDEX].given","typeLabel":"integer"},"precision":{"oneOf":[{"type":"number","minimum":0,"maximum":1,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.outcomes.items.[INDEX].precision.oneOf.0","typeLabel":"number"},{"type":"null","minimum":0,"maximum":1,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.outcomes.items.[INDEX].precision.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.outcomes.items.[INDEX].precision","typeLabel":"number | null"},"recall":{"oneOf":[{"type":"number","minimum":0,"maximum":1,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.outcomes.items.[INDEX].recall.oneOf.0","typeLabel":"number"},{"type":"null","minimum":0,"maximum":1,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.outcomes.items.[INDEX].recall.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.outcomes.items.[INDEX].recall","typeLabel":"number | null"},"f1":{"oneOf":[{"type":"number","minimum":0,"maximum":1,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.outcomes.items.[INDEX].f1.oneOf.0","typeLabel":"number"},{"type":"null","minimum":0,"maximum":1,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.outcomes.items.[INDEX].f1.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.outcomes.items.[INDEX].f1","typeLabel":"number | null"}},"required":["outcome","expected","given","precision","recall","f1"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"loss_report.base.oneOf.0.outcomes.items.[INDEX]","typeLabel":"object"},"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.outcomes","typeLabel":"object[]"},"by_mistake":{"type":"array","items":{"type":"object","properties":{"expected":{"type":"string","minLength":1,"maxLength":200,"description":"An outcome name, or the exact enum label when a string attribute is the result.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.by_mistake.items.[INDEX].expected","typeLabel":"string"},"got":{"oneOf":[{"type":"string","description":"An outcome name, or the exact enum label when a string attribute is the result.","maxLength":200,"minLength":1,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.by_mistake.items.[INDEX].got.oneOf.0","typeLabel":"string"},{"type":"null","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.by_mistake.items.[INDEX].got.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.by_mistake.items.[INDEX].got","typeLabel":"string | null"},"n":{"type":"integer","minimum":1,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.by_mistake.items.[INDEX].n","typeLabel":"integer"},"loss":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.by_mistake.items.[INDEX].loss","typeLabel":"number"},"examples":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.base.oneOf.0.by_mistake.items.[INDEX].examples.items.[INDEX]","typeLabel":"string"},"description":"A few of its cases, worst first, by the id each case carries.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.by_mistake.items.[INDEX].examples","typeLabel":"string[]"}},"required":["expected","got","n","loss","examples"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"loss_report.base.oneOf.0.by_mistake.items.[INDEX]","typeLabel":"object"},"description":"Loss by mistake cell, worst first: the loss-weighted confusion matrix. A response lists the worst 50.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.by_mistake","typeLabel":"object[]"},"by_mistake_total":{"type":"integer","minimum":0,"description":"How many mistake cells there are.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.by_mistake_total","typeLabel":"integer"},"by_question":{"type":"array","items":{"type":"object","properties":{"question":{"type":"string","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.by_question.items.[INDEX].question","typeLabel":"string"},"loss":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.by_question.items.[INDEX].loss","typeLabel":"number"},"share":{"type":"number","minimum":0,"maximum":1,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.by_question.items.[INDEX].share","typeLabel":"number"},"examples":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.base.oneOf.0.by_question.items.[INDEX].examples.items.[INDEX]","typeLabel":"string"},"description":"A few of its cases, worst first, by the id each case carries.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.by_question.items.[INDEX].examples","typeLabel":"string[]"}},"required":["question","loss","share","examples"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"loss_report.base.oneOf.0.by_question.items.[INDEX]","typeLabel":"object"},"description":"Loss by suspect question, worst first: each wrong case's loss spread over the questions a reviewer corrected, else those whose flip alone fixes it. A large share points at rewriting that question.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.by_question","typeLabel":"object[]"},"missing_criterion":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.missing_criterion.n","typeLabel":"integer"},"loss":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.missing_criterion.loss","typeLabel":"number"},"examples":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.base.oneOf.0.missing_criterion.examples.items.[INDEX]","typeLabel":"string"},"description":"A few of its cases, worst first, by the id each case carries.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.missing_criterion.examples","typeLabel":"string[]"}},"required":["n","loss","examples"],"additionalProperties":true,"description":"Wrong cases with no suspect: no reviewer correction, and no single unconfirmed answer whose flip fixes it. A criterion the structure lacks; clustered on one theme, it points at adding a question.","x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.missing_criterion","typeLabel":"object"},"failed":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.failed.n","typeLabel":"integer"},"loss":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.failed.loss","typeLabel":"number"}},"required":["n","loss"],"additionalProperties":true,"description":"Cases the version failed permanently, counted as its dearest mistake: an input or model problem.","x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.failed","typeLabel":"object"},"near_cuts":{"type":"array","items":{"type":"object","properties":{"aggregation":{"type":"string","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.near_cuts.items.[INDEX].aggregation","typeLabel":"string"},"at":{"type":"number","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.near_cuts.items.[INDEX].at","typeLabel":"number"},"lower_it":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.near_cuts.items.[INDEX].lower_it.n","typeLabel":"integer"},"loss":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.near_cuts.items.[INDEX].lower_it.loss","typeLabel":"number"}},"required":["n","loss"],"additionalProperties":true,"description":"Wrong cases just below the cut that belong in the band above it.","x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.near_cuts.items.[INDEX].lower_it","typeLabel":"object"},"raise_it":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.near_cuts.items.[INDEX].raise_it.n","typeLabel":"integer"},"loss":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.near_cuts.items.[INDEX].raise_it.loss","typeLabel":"number"}},"required":["n","loss"],"additionalProperties":true,"description":"Wrong cases just above the cut that belong in the band below it.","x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.near_cuts.items.[INDEX].raise_it","typeLabel":"object"},"examples":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.base.oneOf.0.near_cuts.items.[INDEX].examples.items.[INDEX]","typeLabel":"string"},"description":"A few of its cases, worst first, by the id each case carries.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.near_cuts.items.[INDEX].examples","typeLabel":"string[]"}},"required":["aggregation","at","lower_it","raise_it","examples"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"loss_report.base.oneOf.0.near_cuts.items.[INDEX]","typeLabel":"object"},"description":"Wrong cases within 0.1 of a cut of the result's bands whose expected band is the adjacent one, by the direction that would fix them.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.near_cuts","typeLabel":"object[]"},"severe":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.severe.n","typeLabel":"integer"},"human":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.severe.human","typeLabel":"integer"}},"required":["n","human"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.severe","typeLabel":"object"},"cases":{"type":"array","items":{"type":"object","properties":{"expected":{"type":"string","minLength":1,"maxLength":200,"description":"An outcome name, or the exact enum label when a string attribute is the result.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].expected","typeLabel":"string"},"got":{"oneOf":[{"type":"string","minLength":1,"maxLength":200,"description":"The outcome given; null when there is none: a permanent failure (`failed`), or an abstention whose result names no abstain outcome.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].got.oneOf.0","typeLabel":"string"},{"type":"null","minLength":1,"maxLength":200,"description":"The outcome given; null when there is none: a permanent failure (`failed`), or an abstention whose result names no abstain outcome.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].got.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].got","typeLabel":"string | null"},"failed":{"type":"boolean","description":"The version failed this case permanently: an input or model problem, not a missing criterion.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].failed","typeLabel":"boolean"},"loss":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].loss","typeLabel":"number"},"severe":{"type":"boolean","description":"The objective's severity for this mistake, before weighting by outcome, is at least `severe` (10). Always false without a stored objective.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].severe","typeLabel":"boolean"},"human":{"type":"boolean","description":"Labeled by a person, so it counts toward the no-new-severe-mistake rule.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].human","typeLabel":"boolean"},"margin":{"oneOf":[{"type":"object","description":"For a bands result: the boundary of the nearest band holding the expected outcome, and how far the value is from it. Null otherwise.","required":["value","cut","distance","adjacent"],"x-moonbounce-sdk-known-fields":true,"properties":{"value":{"type":"number","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].margin.oneOf.0.value","typeLabel":"number"},"cut":{"type":"number","description":"The cut to cross to reach the expected outcome's band.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].margin.oneOf.0.cut","typeLabel":"number"},"distance":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].margin.oneOf.0.distance","typeLabel":"number"},"adjacent":{"type":"boolean","description":"The expected band is next to the value's, so moving this one cut alone fixes it.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].margin.oneOf.0.adjacent","typeLabel":"boolean"}},"additionalProperties":true,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].margin.oneOf.0","typeLabel":"object"},{"type":"null","description":"For a bands result: the boundary of the nearest band holding the expected outcome, and how far the value is from it. Null otherwise.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].margin.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].margin","typeLabel":"object | null"},"attributed":{"type":"boolean","description":"Its fixing flips and suspects were worked out: one of the worst mistakes (see `attributed` on the report).","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].attributed","typeLabel":"boolean"},"fixing_flips":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].fixing_flips.items.[INDEX]","typeLabel":"string"},"description":"Questions whose opposite answer (0 or 1) alone gives the expected outcome. Empty: a missing criterion.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].fixing_flips","typeLabel":"string[]"},"flips_total":{"type":"integer","minimum":0,"description":"How many fixing flips there are; a response lists at most 10.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].flips_total","typeLabel":"integer"},"answer_conflicts":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].answer_conflicts.items.[INDEX]","typeLabel":"string"},"description":"Questions whose answer disagrees with a reviewer's `expected_answer` for this case.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].answer_conflicts","typeLabel":"string[]"},"suspects":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].suspects.items.[INDEX]","typeLabel":"string"},"description":"The questions blamed, as `by_question` counts them: the conflicts, else the fixing flips no reviewer confirmed. Empty on an attributed mistake: a missing criterion.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].suspects","typeLabel":"string[]"},"answers":{"type":"object","additionalProperties":{"oneOf":[{"type":"number","uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].answers.additionalProperties.oneOf.0","typeLabel":"number"},{"type":"null","uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].answers.additionalProperties.oneOf.1","typeLabel":"null"}],"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].answers.additionalProperties","typeLabel":"number | null"},"description":"The answers of its fixing flips and conflicting questions (a stored run keeps every answer).","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].answers","typeLabel":"object"},"evaluation_id":{"type":"string","pattern":"^evl_[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","description":"A stored case's evaluation; absent for a dataset item.","uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].evaluation_id","typeLabel":"string"},"item_id":{"type":"string","description":"A dataset item's id, as an evaluate run was given it; absent for a stored case.","uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].item_id","typeLabel":"string"},"flip_effects":{"type":"array","items":{"type":"object","properties":{"question":{"type":"string","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].flip_effects.items.[INDEX].question","typeLabel":"string"},"from":{"oneOf":[{"type":"number","description":"The answer side flipped: 1 when this case's answer is at least 0.5 (turned to 0), else 0 (turned to 1).","enum":[0],"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].flip_effects.items.[INDEX].from.oneOf.0","typeLabel":"enum\u003cnumber>"},{"type":"number","description":"The answer side flipped: 1 when this case's answer is at least 0.5 (turned to 0), else 0 (turned to 1).","enum":[1],"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].flip_effects.items.[INDEX].from.oneOf.1","typeLabel":"enum\u003cnumber>"}],"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].flip_effects.items.[INDEX].from","typeLabel":"enum\u003cnumber>"},"fixes":{"type":"integer","minimum":0,"description":"Wrong cases with an answer on that side that the flip alone gets right, this one included.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].flip_effects.items.[INDEX].fixes","typeLabel":"integer"},"breaks":{"type":"integer","minimum":0,"description":"Right cases with an answer on that side that the flip gets wrong.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].flip_effects.items.[INDEX].breaks","typeLabel":"integer"},"loss_change":{"type":"number","description":"Total loss after the flip minus before, over every case with an answer on that side: negative is better.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].flip_effects.items.[INDEX].loss_change","typeLabel":"number"},"broken":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].flip_effects.items.[INDEX].broken.items.[INDEX]","typeLabel":"string"},"description":"A few of the cases it breaks, costliest first.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].flip_effects.items.[INDEX].broken","typeLabel":"string[]"}},"required":["question","from","fixes","breaks","loss_change","broken"],"additionalProperties":true,"description":"The same flip on every case: what rewording the question so it answers the other way, wherever it now answers as it does here, would fix and break.","x-moonbounce-sdk-known-fields":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].flip_effects.items.[INDEX]","typeLabel":"object"},"description":"One per listed fixing flip, in the same order, on the worst 25 mistakes. A flip whose effect wasn't worked out (past the report's budget) is left out.","uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX].flip_effects","typeLabel":"object[]"}},"required":["expected","got","failed","loss","severe","human","margin","attributed","fixing_flips","flips_total","answer_conflicts","suspects","answers"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"loss_report.base.oneOf.0.cases.items.[INDEX]","typeLabel":"object"},"description":"The worst mistakes, at most 25, with the answers behind them. The roll-ups cover every case.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases","typeLabel":"object[]"},"cases_total":{"type":"integer","minimum":0,"description":"How many mistakes there are; `cases` lists the worst.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.cases_total","typeLabel":"integer"},"attributed":{"type":"integer","minimum":0,"description":"How many mistakes, worst first and permanent failures aside, `by_question`, `missing_criterion` and `fixing_flips` cover (at most 100). The other roll-ups cover every case.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.attributed","typeLabel":"integer"},"accuracy":{"oneOf":[{"type":"number","minimum":0,"maximum":1,"description":"Share of cases given their expected outcome; null with none. Unweighted, so the commonest outcome dominates it: the loss and `macro_f1` weigh each outcome equally. Absent only on a replay of a response stored before it was added.","uniqueKey":"loss_report.base.oneOf.0.accuracy.oneOf.0","typeLabel":"number"},{"type":"null","minimum":0,"maximum":1,"description":"Share of cases given their expected outcome; null with none. Unweighted, so the commonest outcome dominates it: the loss and `macro_f1` weigh each outcome equally. Absent only on a replay of a response stored before it was added.","uniqueKey":"loss_report.base.oneOf.0.accuracy.oneOf.1","typeLabel":"null"}],"uniqueKey":"loss_report.base.oneOf.0.accuracy","typeLabel":"number | null"},"confusion":{"type":"array","items":{"type":"object","properties":{"expected":{"type":"string","minLength":1,"maxLength":200,"description":"An outcome name, or the exact enum label when a string attribute is the result.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.confusion.items.[INDEX].expected","typeLabel":"string"},"got":{"oneOf":[{"type":"string","minLength":1,"maxLength":200,"description":"The outcome given; null when there was none (a failure or an abstention with no abstain outcome).","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.confusion.items.[INDEX].got.oneOf.0","typeLabel":"string"},{"type":"null","minLength":1,"maxLength":200,"description":"The outcome given; null when there was none (a failure or an abstention with no abstain outcome).","isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.confusion.items.[INDEX].got.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.confusion.items.[INDEX].got","typeLabel":"string | null"},"n":{"type":"integer","minimum":1,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0.confusion.items.[INDEX].n","typeLabel":"integer"}},"required":["expected","got","n"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"loss_report.base.oneOf.0.confusion.items.[INDEX]","typeLabel":"object"},"description":"The confusion matrix: every cell with a case, right answers (`expected` equals `got`) included, by expected outcome then `got` (null last). Unweighted counts; `by_mistake` weighs the wrong cells by the objective. A response lists the 400 largest cells. Absent only on a replay of a response stored before it was added.","uniqueKey":"loss_report.base.oneOf.0.confusion","typeLabel":"object[]"},"confusion_total":{"type":"integer","minimum":0,"description":"How many confusion-matrix cells there are. Absent only on a replay of a response stored before it was added.","uniqueKey":"loss_report.base.oneOf.0.confusion_total","typeLabel":"integer"}},"additionalProperties":true,"isRequired":true,"uniqueKey":"loss_report.base.oneOf.0","typeLabel":"object"},{"type":"null","description":"The base's report; null when there is no base.","isRequired":true,"uniqueKey":"loss_report.base.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.base","typeLabel":"object | null"},"candidate":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"description":"Cases scored.","isRequired":true,"uniqueKey":"loss_report.candidate.n","typeLabel":"integer"},"loss":{"type":"number","minimum":0,"description":"Total loss of the mistakes.","isRequired":true,"uniqueKey":"loss_report.candidate.loss","typeLabel":"number"},"loss_per_case":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.candidate.loss_per_case","typeLabel":"number"},"target_met":{"oneOf":[{"type":"boolean","description":"Whether loss per case is at or below the objective's target; null without one.","isRequired":true,"uniqueKey":"loss_report.candidate.target_met.oneOf.0","typeLabel":"boolean"},{"type":"null","description":"Whether loss per case is at or below the objective's target; null without one.","isRequired":true,"uniqueKey":"loss_report.candidate.target_met.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.candidate.target_met","typeLabel":"boolean | null"},"macro_f1":{"oneOf":[{"type":"number","minimum":0,"maximum":1,"description":"Mean F1 over the outcomes expected or given: the headline. Not the loss.","isRequired":true,"uniqueKey":"loss_report.candidate.macro_f1.oneOf.0","typeLabel":"number"},{"type":"null","minimum":0,"maximum":1,"description":"Mean F1 over the outcomes expected or given: the headline. Not the loss.","isRequired":true,"uniqueKey":"loss_report.candidate.macro_f1.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.candidate.macro_f1","typeLabel":"number | null"},"outcomes":{"type":"array","items":{"type":"object","properties":{"outcome":{"type":"string","minLength":1,"maxLength":200,"description":"An outcome name, or the exact enum label when a string attribute is the result.","isRequired":true,"uniqueKey":"loss_report.candidate.outcomes.items.[INDEX].outcome","typeLabel":"string"},"expected":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"loss_report.candidate.outcomes.items.[INDEX].expected","typeLabel":"integer"},"given":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"loss_report.candidate.outcomes.items.[INDEX].given","typeLabel":"integer"},"precision":{"oneOf":[{"type":"number","minimum":0,"maximum":1,"isRequired":true,"uniqueKey":"loss_report.candidate.outcomes.items.[INDEX].precision.oneOf.0","typeLabel":"number"},{"type":"null","minimum":0,"maximum":1,"isRequired":true,"uniqueKey":"loss_report.candidate.outcomes.items.[INDEX].precision.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.candidate.outcomes.items.[INDEX].precision","typeLabel":"number | null"},"recall":{"oneOf":[{"type":"number","minimum":0,"maximum":1,"isRequired":true,"uniqueKey":"loss_report.candidate.outcomes.items.[INDEX].recall.oneOf.0","typeLabel":"number"},{"type":"null","minimum":0,"maximum":1,"isRequired":true,"uniqueKey":"loss_report.candidate.outcomes.items.[INDEX].recall.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.candidate.outcomes.items.[INDEX].recall","typeLabel":"number | null"},"f1":{"oneOf":[{"type":"number","minimum":0,"maximum":1,"isRequired":true,"uniqueKey":"loss_report.candidate.outcomes.items.[INDEX].f1.oneOf.0","typeLabel":"number"},{"type":"null","minimum":0,"maximum":1,"isRequired":true,"uniqueKey":"loss_report.candidate.outcomes.items.[INDEX].f1.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.candidate.outcomes.items.[INDEX].f1","typeLabel":"number | null"}},"required":["outcome","expected","given","precision","recall","f1"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"loss_report.candidate.outcomes.items.[INDEX]","typeLabel":"object"},"isRequired":true,"uniqueKey":"loss_report.candidate.outcomes","typeLabel":"object[]"},"by_mistake":{"type":"array","items":{"type":"object","properties":{"expected":{"type":"string","minLength":1,"maxLength":200,"description":"An outcome name, or the exact enum label when a string attribute is the result.","isRequired":true,"uniqueKey":"loss_report.candidate.by_mistake.items.[INDEX].expected","typeLabel":"string"},"got":{"oneOf":[{"type":"string","description":"An outcome name, or the exact enum label when a string attribute is the result.","maxLength":200,"minLength":1,"isRequired":true,"uniqueKey":"loss_report.candidate.by_mistake.items.[INDEX].got.oneOf.0","typeLabel":"string"},{"type":"null","isRequired":true,"uniqueKey":"loss_report.candidate.by_mistake.items.[INDEX].got.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.candidate.by_mistake.items.[INDEX].got","typeLabel":"string | null"},"n":{"type":"integer","minimum":1,"isRequired":true,"uniqueKey":"loss_report.candidate.by_mistake.items.[INDEX].n","typeLabel":"integer"},"loss":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.candidate.by_mistake.items.[INDEX].loss","typeLabel":"number"},"examples":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.candidate.by_mistake.items.[INDEX].examples.items.[INDEX]","typeLabel":"string"},"description":"A few of its cases, worst first, by the id each case carries.","isRequired":true,"uniqueKey":"loss_report.candidate.by_mistake.items.[INDEX].examples","typeLabel":"string[]"}},"required":["expected","got","n","loss","examples"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"loss_report.candidate.by_mistake.items.[INDEX]","typeLabel":"object"},"description":"Loss by mistake cell, worst first: the loss-weighted confusion matrix. A response lists the worst 50.","isRequired":true,"uniqueKey":"loss_report.candidate.by_mistake","typeLabel":"object[]"},"by_mistake_total":{"type":"integer","minimum":0,"description":"How many mistake cells there are.","isRequired":true,"uniqueKey":"loss_report.candidate.by_mistake_total","typeLabel":"integer"},"by_question":{"type":"array","items":{"type":"object","properties":{"question":{"type":"string","isRequired":true,"uniqueKey":"loss_report.candidate.by_question.items.[INDEX].question","typeLabel":"string"},"loss":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.candidate.by_question.items.[INDEX].loss","typeLabel":"number"},"share":{"type":"number","minimum":0,"maximum":1,"isRequired":true,"uniqueKey":"loss_report.candidate.by_question.items.[INDEX].share","typeLabel":"number"},"examples":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.candidate.by_question.items.[INDEX].examples.items.[INDEX]","typeLabel":"string"},"description":"A few of its cases, worst first, by the id each case carries.","isRequired":true,"uniqueKey":"loss_report.candidate.by_question.items.[INDEX].examples","typeLabel":"string[]"}},"required":["question","loss","share","examples"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"loss_report.candidate.by_question.items.[INDEX]","typeLabel":"object"},"description":"Loss by suspect question, worst first: each wrong case's loss spread over the questions a reviewer corrected, else those whose flip alone fixes it. A large share points at rewriting that question.","isRequired":true,"uniqueKey":"loss_report.candidate.by_question","typeLabel":"object[]"},"missing_criterion":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"loss_report.candidate.missing_criterion.n","typeLabel":"integer"},"loss":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.candidate.missing_criterion.loss","typeLabel":"number"},"examples":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.candidate.missing_criterion.examples.items.[INDEX]","typeLabel":"string"},"description":"A few of its cases, worst first, by the id each case carries.","isRequired":true,"uniqueKey":"loss_report.candidate.missing_criterion.examples","typeLabel":"string[]"}},"required":["n","loss","examples"],"additionalProperties":true,"description":"Wrong cases with no suspect: no reviewer correction, and no single unconfirmed answer whose flip fixes it. A criterion the structure lacks; clustered on one theme, it points at adding a question.","x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"loss_report.candidate.missing_criterion","typeLabel":"object"},"failed":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"loss_report.candidate.failed.n","typeLabel":"integer"},"loss":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.candidate.failed.loss","typeLabel":"number"}},"required":["n","loss"],"additionalProperties":true,"description":"Cases the version failed permanently, counted as its dearest mistake: an input or model problem.","x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"loss_report.candidate.failed","typeLabel":"object"},"near_cuts":{"type":"array","items":{"type":"object","properties":{"aggregation":{"type":"string","isRequired":true,"uniqueKey":"loss_report.candidate.near_cuts.items.[INDEX].aggregation","typeLabel":"string"},"at":{"type":"number","isRequired":true,"uniqueKey":"loss_report.candidate.near_cuts.items.[INDEX].at","typeLabel":"number"},"lower_it":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"loss_report.candidate.near_cuts.items.[INDEX].lower_it.n","typeLabel":"integer"},"loss":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.candidate.near_cuts.items.[INDEX].lower_it.loss","typeLabel":"number"}},"required":["n","loss"],"additionalProperties":true,"description":"Wrong cases just below the cut that belong in the band above it.","x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"loss_report.candidate.near_cuts.items.[INDEX].lower_it","typeLabel":"object"},"raise_it":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"loss_report.candidate.near_cuts.items.[INDEX].raise_it.n","typeLabel":"integer"},"loss":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.candidate.near_cuts.items.[INDEX].raise_it.loss","typeLabel":"number"}},"required":["n","loss"],"additionalProperties":true,"description":"Wrong cases just above the cut that belong in the band below it.","x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"loss_report.candidate.near_cuts.items.[INDEX].raise_it","typeLabel":"object"},"examples":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.candidate.near_cuts.items.[INDEX].examples.items.[INDEX]","typeLabel":"string"},"description":"A few of its cases, worst first, by the id each case carries.","isRequired":true,"uniqueKey":"loss_report.candidate.near_cuts.items.[INDEX].examples","typeLabel":"string[]"}},"required":["aggregation","at","lower_it","raise_it","examples"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"loss_report.candidate.near_cuts.items.[INDEX]","typeLabel":"object"},"description":"Wrong cases within 0.1 of a cut of the result's bands whose expected band is the adjacent one, by the direction that would fix them.","isRequired":true,"uniqueKey":"loss_report.candidate.near_cuts","typeLabel":"object[]"},"severe":{"type":"object","properties":{"n":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"loss_report.candidate.severe.n","typeLabel":"integer"},"human":{"type":"integer","minimum":0,"isRequired":true,"uniqueKey":"loss_report.candidate.severe.human","typeLabel":"integer"}},"required":["n","human"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"loss_report.candidate.severe","typeLabel":"object"},"cases":{"type":"array","items":{"type":"object","properties":{"expected":{"type":"string","minLength":1,"maxLength":200,"description":"An outcome name, or the exact enum label when a string attribute is the result.","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].expected","typeLabel":"string"},"got":{"oneOf":[{"type":"string","minLength":1,"maxLength":200,"description":"The outcome given; null when there is none: a permanent failure (`failed`), or an abstention whose result names no abstain outcome.","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].got.oneOf.0","typeLabel":"string"},{"type":"null","minLength":1,"maxLength":200,"description":"The outcome given; null when there is none: a permanent failure (`failed`), or an abstention whose result names no abstain outcome.","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].got.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].got","typeLabel":"string | null"},"failed":{"type":"boolean","description":"The version failed this case permanently: an input or model problem, not a missing criterion.","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].failed","typeLabel":"boolean"},"loss":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].loss","typeLabel":"number"},"severe":{"type":"boolean","description":"The objective's severity for this mistake, before weighting by outcome, is at least `severe` (10). Always false without a stored objective.","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].severe","typeLabel":"boolean"},"human":{"type":"boolean","description":"Labeled by a person, so it counts toward the no-new-severe-mistake rule.","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].human","typeLabel":"boolean"},"margin":{"oneOf":[{"type":"object","description":"For a bands result: the boundary of the nearest band holding the expected outcome, and how far the value is from it. Null otherwise.","required":["value","cut","distance","adjacent"],"x-moonbounce-sdk-known-fields":true,"properties":{"value":{"type":"number","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].margin.oneOf.0.value","typeLabel":"number"},"cut":{"type":"number","description":"The cut to cross to reach the expected outcome's band.","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].margin.oneOf.0.cut","typeLabel":"number"},"distance":{"type":"number","minimum":0,"isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].margin.oneOf.0.distance","typeLabel":"number"},"adjacent":{"type":"boolean","description":"The expected band is next to the value's, so moving this one cut alone fixes it.","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].margin.oneOf.0.adjacent","typeLabel":"boolean"}},"additionalProperties":true,"isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].margin.oneOf.0","typeLabel":"object"},{"type":"null","description":"For a bands result: the boundary of the nearest band holding the expected outcome, and how far the value is from it. Null otherwise.","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].margin.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].margin","typeLabel":"object | null"},"attributed":{"type":"boolean","description":"Its fixing flips and suspects were worked out: one of the worst mistakes (see `attributed` on the report).","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].attributed","typeLabel":"boolean"},"fixing_flips":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.candidate.cases.items.[INDEX].fixing_flips.items.[INDEX]","typeLabel":"string"},"description":"Questions whose opposite answer (0 or 1) alone gives the expected outcome. Empty: a missing criterion.","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].fixing_flips","typeLabel":"string[]"},"flips_total":{"type":"integer","minimum":0,"description":"How many fixing flips there are; a response lists at most 10.","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].flips_total","typeLabel":"integer"},"answer_conflicts":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.candidate.cases.items.[INDEX].answer_conflicts.items.[INDEX]","typeLabel":"string"},"description":"Questions whose answer disagrees with a reviewer's `expected_answer` for this case.","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].answer_conflicts","typeLabel":"string[]"},"suspects":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.candidate.cases.items.[INDEX].suspects.items.[INDEX]","typeLabel":"string"},"description":"The questions blamed, as `by_question` counts them: the conflicts, else the fixing flips no reviewer confirmed. Empty on an attributed mistake: a missing criterion.","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].suspects","typeLabel":"string[]"},"answers":{"type":"object","additionalProperties":{"oneOf":[{"type":"number","uniqueKey":"loss_report.candidate.cases.items.[INDEX].answers.additionalProperties.oneOf.0","typeLabel":"number"},{"type":"null","uniqueKey":"loss_report.candidate.cases.items.[INDEX].answers.additionalProperties.oneOf.1","typeLabel":"null"}],"uniqueKey":"loss_report.candidate.cases.items.[INDEX].answers.additionalProperties","typeLabel":"number | null"},"description":"The answers of its fixing flips and conflicting questions (a stored run keeps every answer).","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].answers","typeLabel":"object"},"evaluation_id":{"type":"string","pattern":"^evl_[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$","description":"A stored case's evaluation; absent for a dataset item.","uniqueKey":"loss_report.candidate.cases.items.[INDEX].evaluation_id","typeLabel":"string"},"item_id":{"type":"string","description":"A dataset item's id, as an evaluate run was given it; absent for a stored case.","uniqueKey":"loss_report.candidate.cases.items.[INDEX].item_id","typeLabel":"string"},"flip_effects":{"type":"array","items":{"type":"object","properties":{"question":{"type":"string","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].flip_effects.items.[INDEX].question","typeLabel":"string"},"from":{"oneOf":[{"type":"number","description":"The answer side flipped: 1 when this case's answer is at least 0.5 (turned to 0), else 0 (turned to 1).","enum":[0],"isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].flip_effects.items.[INDEX].from.oneOf.0","typeLabel":"enum\u003cnumber>"},{"type":"number","description":"The answer side flipped: 1 when this case's answer is at least 0.5 (turned to 0), else 0 (turned to 1).","enum":[1],"isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].flip_effects.items.[INDEX].from.oneOf.1","typeLabel":"enum\u003cnumber>"}],"isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].flip_effects.items.[INDEX].from","typeLabel":"enum\u003cnumber>"},"fixes":{"type":"integer","minimum":0,"description":"Wrong cases with an answer on that side that the flip alone gets right, this one included.","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].flip_effects.items.[INDEX].fixes","typeLabel":"integer"},"breaks":{"type":"integer","minimum":0,"description":"Right cases with an answer on that side that the flip gets wrong.","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].flip_effects.items.[INDEX].breaks","typeLabel":"integer"},"loss_change":{"type":"number","description":"Total loss after the flip minus before, over every case with an answer on that side: negative is better.","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].flip_effects.items.[INDEX].loss_change","typeLabel":"number"},"broken":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.candidate.cases.items.[INDEX].flip_effects.items.[INDEX].broken.items.[INDEX]","typeLabel":"string"},"description":"A few of the cases it breaks, costliest first.","isRequired":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].flip_effects.items.[INDEX].broken","typeLabel":"string[]"}},"required":["question","from","fixes","breaks","loss_change","broken"],"additionalProperties":true,"description":"The same flip on every case: what rewording the question so it answers the other way, wherever it now answers as it does here, would fix and break.","x-moonbounce-sdk-known-fields":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX].flip_effects.items.[INDEX]","typeLabel":"object"},"description":"One per listed fixing flip, in the same order, on the worst 25 mistakes. A flip whose effect wasn't worked out (past the report's budget) is left out.","uniqueKey":"loss_report.candidate.cases.items.[INDEX].flip_effects","typeLabel":"object[]"}},"required":["expected","got","failed","loss","severe","human","margin","attributed","fixing_flips","flips_total","answer_conflicts","suspects","answers"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"loss_report.candidate.cases.items.[INDEX]","typeLabel":"object"},"description":"The worst mistakes, at most 25, with the answers behind them. The roll-ups cover every case.","isRequired":true,"uniqueKey":"loss_report.candidate.cases","typeLabel":"object[]"},"cases_total":{"type":"integer","minimum":0,"description":"How many mistakes there are; `cases` lists the worst.","isRequired":true,"uniqueKey":"loss_report.candidate.cases_total","typeLabel":"integer"},"attributed":{"type":"integer","minimum":0,"description":"How many mistakes, worst first and permanent failures aside, `by_question`, `missing_criterion` and `fixing_flips` cover (at most 100). The other roll-ups cover every case.","isRequired":true,"uniqueKey":"loss_report.candidate.attributed","typeLabel":"integer"},"accuracy":{"oneOf":[{"type":"number","minimum":0,"maximum":1,"description":"Share of cases given their expected outcome; null with none. Unweighted, so the commonest outcome dominates it: the loss and `macro_f1` weigh each outcome equally. Absent only on a replay of a response stored before it was added.","uniqueKey":"loss_report.candidate.accuracy.oneOf.0","typeLabel":"number"},{"type":"null","minimum":0,"maximum":1,"description":"Share of cases given their expected outcome; null with none. Unweighted, so the commonest outcome dominates it: the loss and `macro_f1` weigh each outcome equally. Absent only on a replay of a response stored before it was added.","uniqueKey":"loss_report.candidate.accuracy.oneOf.1","typeLabel":"null"}],"uniqueKey":"loss_report.candidate.accuracy","typeLabel":"number | null"},"confusion":{"type":"array","items":{"type":"object","properties":{"expected":{"type":"string","minLength":1,"maxLength":200,"description":"An outcome name, or the exact enum label when a string attribute is the result.","isRequired":true,"uniqueKey":"loss_report.candidate.confusion.items.[INDEX].expected","typeLabel":"string"},"got":{"oneOf":[{"type":"string","minLength":1,"maxLength":200,"description":"The outcome given; null when there was none (a failure or an abstention with no abstain outcome).","isRequired":true,"uniqueKey":"loss_report.candidate.confusion.items.[INDEX].got.oneOf.0","typeLabel":"string"},{"type":"null","minLength":1,"maxLength":200,"description":"The outcome given; null when there was none (a failure or an abstention with no abstain outcome).","isRequired":true,"uniqueKey":"loss_report.candidate.confusion.items.[INDEX].got.oneOf.1","typeLabel":"null"}],"isRequired":true,"uniqueKey":"loss_report.candidate.confusion.items.[INDEX].got","typeLabel":"string | null"},"n":{"type":"integer","minimum":1,"isRequired":true,"uniqueKey":"loss_report.candidate.confusion.items.[INDEX].n","typeLabel":"integer"}},"required":["expected","got","n"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"loss_report.candidate.confusion.items.[INDEX]","typeLabel":"object"},"description":"The confusion matrix: every cell with a case, right answers (`expected` equals `got`) included, by expected outcome then `got` (null last). Unweighted counts; `by_mistake` weighs the wrong cells by the objective. A response lists the 400 largest cells. Absent only on a replay of a response stored before it was added.","uniqueKey":"loss_report.candidate.confusion","typeLabel":"object[]"},"confusion_total":{"type":"integer","minimum":0,"description":"How many confusion-matrix cells there are. Absent only on a replay of a response stored before it was added.","uniqueKey":"loss_report.candidate.confusion_total","typeLabel":"integer"}},"required":["n","loss","loss_per_case","target_met","macro_f1","outcomes","by_mistake","by_mistake_total","by_question","missing_criterion","failed","near_cuts","severe","cases","cases_total","attributed"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"loss_report.candidate","typeLabel":"object"},"accepted":{"type":"boolean","description":"The candidate lowers loss per case and adds no new severe mistake on a human-labeled case. False without a base.","isRequired":true,"uniqueKey":"loss_report.accepted","typeLabel":"boolean"},"reasons":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.reasons.items.[INDEX]","typeLabel":"string"},"description":"Why it is not accepted; empty when it is.","isRequired":true,"uniqueKey":"loss_report.reasons","typeLabel":"string[]"},"new_severe_human":{"type":"array","items":{"type":"string","uniqueKey":"loss_report.new_severe_human.items.[INDEX]","typeLabel":"string"},"description":"Human-labeled cases the candidate gets severely wrong that the base did not.","isRequired":true,"uniqueKey":"loss_report.new_severe_human","typeLabel":"string[]"}},"required":["objective","base","candidate","accepted","reasons","new_severe_human"],"additionalProperties":true,"description":"Base and candidate judged by the decision's objective, on the same scored cases (each mistake's loss is its severity, every mistake `normal` without an objective, weighted so each expected outcome counts equally), and whether the candidate is accepted: lower loss per case and no new severe mistake on a human-labeled case. Absent only on a replay of a response stored before previews were judged.","x-moonbounce-sdk-known-fields":true,"uniqueKey":"loss_report","typeLabel":"object"}},"required":["preview_id","decision_id","environment","base","candidate","hashes_match","evidence","cost"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"","typeLabel":"object"},"examples":{"Example":{"title":"Example","value":{"preview_id":"\u003cstring>","decision_id":"\u003cstring>","environment":"production","base":{"version":{"structure_hash":"\u003cstring>","number":2},"model":{"profile":"\u003cstring>"}},"candidate":{"version":{"structure_hash":"\u003cstring>","number":2},"model":{"profile":"\u003cstring>"}},"hashes_match":{"structure":true},"evidence":{"label":"imported_examples","frozen_at":"2023-11-07T05:31:56Z","by_split":{"example":{"n":1,"fixes":1,"breaks":1,"base_correct":1,"candidate_correct":1},"train":{"n":1,"fixes":1,"breaks":1,"base_correct":1,"candidate_correct":1}},"noise_rate":0.5,"action_mix":{"before":{},"after":{},"on":"\u003cstring>","n":1},"changed_items":[{"evaluation_id":"\u003cstring>","feedback_id":"\u003cstring>","split":"example","change":"fix","before":{"answers":{},"outcome":"\u003cstring>","value":123},"after":{"answers":{},"outcome":"\u003cstring>","value":123}}],"credited_feedback":["\u003cstring>"],"by_source":{"app_native":{"n":1,"fixes":1,"breaks":1,"base_correct":1,"candidate_correct":1},"app_resolution":{"n":1,"fixes":1,"breaks":1,"base_correct":1,"candidate_correct":1},"reviewer":{"n":1,"fixes":1,"breaks":1,"base_correct":1,"candidate_correct":1},"owner":{"n":1,"fixes":1,"breaks":1,"base_correct":1,"candidate_correct":1},"import":{"n":1,"fixes":1,"breaks":1,"base_correct":1,"candidate_correct":1},"synthetic_review":{"n":1,"fixes":1,"breaks":1,"base_correct":1,"candidate_correct":1}}},"cost":{"model_calls":1,"questions":1,"input_tokens":1,"output_tokens":1,"usd_micros":1,"cache_hits":1},"objective":{"revision":2,"objective_hash":"\u003cstring>"},"loss_report":{"objective":{"source":"stored","revision":2,"objective_hash":"\u003cstring>","effective_hash":"\u003cstring>","unknown_outcomes":["\u003cstring>"]},"base":{"n":1,"loss":1,"loss_per_case":1,"target_met":true,"macro_f1":0.5,"outcomes":[{"outcome":"\u003cstring>","expected":1,"given":1,"precision":0.5,"recall":0.5,"f1":0.5}],"by_mistake":[{"expected":"\u003cstring>","got":"\u003cstring>","n":2,"loss":1,"examples":["\u003cstring>"]}],"by_mistake_total":1,"by_question":[{"question":"\u003cstring>","loss":1,"share":0.5,"examples":["\u003cstring>"]}],"missing_criterion":{"n":1,"loss":1,"examples":["\u003cstring>"]},"failed":{"n":1,"loss":1},"near_cuts":[{"aggregation":"\u003cstring>","at":123,"lower_it":{"n":1,"loss":1},"raise_it":{"n":1,"loss":1},"examples":["\u003cstring>"]}],"severe":{"n":1,"human":1},"cases":[{"expected":"\u003cstring>","got":"\u003cstring>","failed":true,"loss":1,"severe":true,"human":true,"margin":{"value":123,"cut":123,"distance":1,"adjacent":true},"attributed":true,"fixing_flips":["\u003cstring>"],"flips_total":1,"answer_conflicts":["\u003cstring>"],"suspects":["\u003cstring>"],"answers":{},"evaluation_id":"\u003cstring>","item_id":"\u003cstring>","flip_effects":[{"question":"\u003cstring>","from":0,"fixes":1,"breaks":1,"loss_change":123,"broken":["\u003cstring>"]}]}],"cases_total":1,"attributed":1,"accuracy":0.5,"confusion":[{"expected":"\u003cstring>","got":"\u003cstring>","n":2}],"confusion_total":1},"candidate":{"n":1,"loss":1,"loss_per_case":1,"target_met":true,"macro_f1":0.5,"outcomes":[{"outcome":"\u003cstring>","expected":1,"given":1,"precision":0.5,"recall":0.5,"f1":0.5}],"by_mistake":[{"expected":"\u003cstring>","got":"\u003cstring>","n":2,"loss":1,"examples":["\u003cstring>"]}],"by_mistake_total":1,"by_question":[{"question":"\u003cstring>","loss":1,"share":0.5,"examples":["\u003cstring>"]}],"missing_criterion":{"n":1,"loss":1,"examples":["\u003cstring>"]},"failed":{"n":1,"loss":1},"near_cuts":[{"aggregation":"\u003cstring>","at":123,"lower_it":{"n":1,"loss":1},"raise_it":{"n":1,"loss":1},"examples":["\u003cstring>"]}],"severe":{"n":1,"human":1},"cases":[{"expected":"\u003cstring>","got":"\u003cstring>","failed":true,"loss":1,"severe":true,"human":true,"margin":{"value":123,"cut":123,"distance":1,"adjacent":true},"attributed":true,"fixing_flips":["\u003cstring>"],"flips_total":1,"answer_conflicts":["\u003cstring>"],"suspects":["\u003cstring>"],"answers":{},"evaluation_id":"\u003cstring>","item_id":"\u003cstring>","flip_effects":[{"question":"\u003cstring>","from":0,"fixes":1,"breaks":1,"loss_change":123,"broken":["\u003cstring>"]}]}],"cases_total":1,"attributed":1,"accuracy":0.5,"confusion":[{"expected":"\u003cstring>","got":"\u003cstring>","n":2}],"confusion_total":1},"accepted":true,"reasons":["\u003cstring>"],"new_severe_human":["\u003cstring>"]}}}},"exampleType":"generated"}},"exampleType":"generated"},"d92c086e-b85e-4d96-9081-0d576e9fa7ce":{"description":"Invalid request: `param` names the field at fault when one applies","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"object","properties":{"type":{"type":"string","enum":["invalid_request","authentication","permission_denied","not_found","conflict","rate_limited","engine_failed","unavailable","internal"],"isRequired":true,"uniqueKey":"error.type","typeLabel":"enum\u003cstring>"},"message":{"type":"string","description":"For people; don't parse it.","isRequired":true,"uniqueKey":"error.message","typeLabel":"string"},"reason":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Set where a caller can act on it, one of the reasons in the ErrorReason schema. New reasons may be added: treat an unknown one by the status and `type`.","uniqueKey":"error.reason","typeLabel":"string"},"param":{"type":"string","description":"The request field at fault, as a dotted path (`attribute_values.region`, `claims.1.value`).","uniqueKey":"error.param","typeLabel":"string"},"details":{"type":"object","properties":{"issues":{"type":"array","items":{"type":"object","properties":{"param":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].param","typeLabel":"string"},"code":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Stable, extensible issue code; unknown codes still have a parameter and human-readable message.","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].code","typeLabel":"string"},"message":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].message","typeLabel":"string"}},"required":["param","code","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"error.details.issues.items.[INDEX]","typeLabel":"object"},"uniqueKey":"error.details.issues","typeLabel":"object[]"}},"additionalProperties":{"oneOf":[{}],"uniqueKey":"error.details.additionalProperties","typeLabel":"any"},"uniqueKey":"error.details","typeLabel":"object"}},"required":["type","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"error","typeLabel":"object"}},"required":["error"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"","typeLabel":"object"},"examples":{"Example":{"title":"Example","value":{"error":{"type":"invalid_request","message":"\u003cstring>","reason":"\u003cstring>","param":"\u003cstring>","details":{"issues":[{"param":"\u003cstring>","code":"\u003cstring>","message":"\u003cstring>"}]}}}}},"exampleType":"generated"}},"exampleType":"generated"},"966254c0-16c3-4273-9dad-e090d29da618":{"description":"Missing or invalid API key (reason KEY_INVALID)","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"object","properties":{"type":{"type":"string","enum":["invalid_request","authentication","permission_denied","not_found","conflict","rate_limited","engine_failed","unavailable","internal"],"isRequired":true,"uniqueKey":"error.type","typeLabel":"enum\u003cstring>"},"message":{"type":"string","description":"For people; don't parse it.","isRequired":true,"uniqueKey":"error.message","typeLabel":"string"},"reason":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Set where a caller can act on it, one of the reasons in the ErrorReason schema. New reasons may be added: treat an unknown one by the status and `type`.","uniqueKey":"error.reason","typeLabel":"string"},"param":{"type":"string","description":"The request field at fault, as a dotted path (`attribute_values.region`, `claims.1.value`).","uniqueKey":"error.param","typeLabel":"string"},"details":{"type":"object","properties":{"issues":{"type":"array","items":{"type":"object","properties":{"param":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].param","typeLabel":"string"},"code":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Stable, extensible issue code; unknown codes still have a parameter and human-readable message.","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].code","typeLabel":"string"},"message":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].message","typeLabel":"string"}},"required":["param","code","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"error.details.issues.items.[INDEX]","typeLabel":"object"},"uniqueKey":"error.details.issues","typeLabel":"object[]"}},"additionalProperties":{"oneOf":[{}],"uniqueKey":"error.details.additionalProperties","typeLabel":"any"},"uniqueKey":"error.details","typeLabel":"object"}},"required":["type","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"error","typeLabel":"object"}},"required":["error"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"","typeLabel":"object"},"examples":{"Example":{"title":"Example","value":{"error":{"type":"invalid_request","message":"\u003cstring>","reason":"\u003cstring>","param":"\u003cstring>","details":{"issues":[{"param":"\u003cstring>","code":"\u003cstring>","message":"\u003cstring>"}]}}}}},"exampleType":"generated"}},"exampleType":"generated"},"436544e6-8438-45d0-a099-8bd404440201":{"description":"The key may not do this","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"object","properties":{"type":{"type":"string","enum":["invalid_request","authentication","permission_denied","not_found","conflict","rate_limited","engine_failed","unavailable","internal"],"isRequired":true,"uniqueKey":"error.type","typeLabel":"enum\u003cstring>"},"message":{"type":"string","description":"For people; don't parse it.","isRequired":true,"uniqueKey":"error.message","typeLabel":"string"},"reason":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Set where a caller can act on it, one of the reasons in the ErrorReason schema. New reasons may be added: treat an unknown one by the status and `type`.","uniqueKey":"error.reason","typeLabel":"string"},"param":{"type":"string","description":"The request field at fault, as a dotted path (`attribute_values.region`, `claims.1.value`).","uniqueKey":"error.param","typeLabel":"string"},"details":{"type":"object","properties":{"issues":{"type":"array","items":{"type":"object","properties":{"param":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].param","typeLabel":"string"},"code":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Stable, extensible issue code; unknown codes still have a parameter and human-readable message.","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].code","typeLabel":"string"},"message":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].message","typeLabel":"string"}},"required":["param","code","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"error.details.issues.items.[INDEX]","typeLabel":"object"},"uniqueKey":"error.details.issues","typeLabel":"object[]"}},"additionalProperties":{"oneOf":[{}],"uniqueKey":"error.details.additionalProperties","typeLabel":"any"},"uniqueKey":"error.details","typeLabel":"object"}},"required":["type","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"error","typeLabel":"object"}},"required":["error"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"","typeLabel":"object"},"examples":{"Example":{"title":"Example","value":{"error":{"type":"invalid_request","message":"\u003cstring>","reason":"\u003cstring>","param":"\u003cstring>","details":{"issues":[{"param":"\u003cstring>","code":"\u003cstring>","message":"\u003cstring>"}]}}}}},"exampleType":"generated"}},"exampleType":"generated"},"e4d2bb6c-d819-40b3-9ab7-13c366029600":{"description":"Not found, or not in this key's organization and environment","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"object","properties":{"type":{"type":"string","enum":["invalid_request","authentication","permission_denied","not_found","conflict","rate_limited","engine_failed","unavailable","internal"],"isRequired":true,"uniqueKey":"error.type","typeLabel":"enum\u003cstring>"},"message":{"type":"string","description":"For people; don't parse it.","isRequired":true,"uniqueKey":"error.message","typeLabel":"string"},"reason":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Set where a caller can act on it, one of the reasons in the ErrorReason schema. New reasons may be added: treat an unknown one by the status and `type`.","uniqueKey":"error.reason","typeLabel":"string"},"param":{"type":"string","description":"The request field at fault, as a dotted path (`attribute_values.region`, `claims.1.value`).","uniqueKey":"error.param","typeLabel":"string"},"details":{"type":"object","properties":{"issues":{"type":"array","items":{"type":"object","properties":{"param":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].param","typeLabel":"string"},"code":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Stable, extensible issue code; unknown codes still have a parameter and human-readable message.","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].code","typeLabel":"string"},"message":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].message","typeLabel":"string"}},"required":["param","code","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"error.details.issues.items.[INDEX]","typeLabel":"object"},"uniqueKey":"error.details.issues","typeLabel":"object[]"}},"additionalProperties":{"oneOf":[{}],"uniqueKey":"error.details.additionalProperties","typeLabel":"any"},"uniqueKey":"error.details","typeLabel":"object"}},"required":["type","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"error","typeLabel":"object"}},"required":["error"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"","typeLabel":"object"},"examples":{"Example":{"title":"Example","value":{"error":{"type":"invalid_request","message":"\u003cstring>","reason":"\u003cstring>","param":"\u003cstring>","details":{"issues":[{"param":"\u003cstring>","code":"\u003cstring>","message":"\u003cstring>"}]}}}}},"exampleType":"generated"}},"exampleType":"generated"},"1930a3e4-d6ee-4aaf-b62f-57a172f11d39":{"description":"Conflict with the current state","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"object","properties":{"type":{"type":"string","enum":["invalid_request","authentication","permission_denied","not_found","conflict","rate_limited","engine_failed","unavailable","internal"],"isRequired":true,"uniqueKey":"error.type","typeLabel":"enum\u003cstring>"},"message":{"type":"string","description":"For people; don't parse it.","isRequired":true,"uniqueKey":"error.message","typeLabel":"string"},"reason":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Set where a caller can act on it, one of the reasons in the ErrorReason schema. New reasons may be added: treat an unknown one by the status and `type`.","uniqueKey":"error.reason","typeLabel":"string"},"param":{"type":"string","description":"The request field at fault, as a dotted path (`attribute_values.region`, `claims.1.value`).","uniqueKey":"error.param","typeLabel":"string"},"details":{"type":"object","properties":{"issues":{"type":"array","items":{"type":"object","properties":{"param":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].param","typeLabel":"string"},"code":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Stable, extensible issue code; unknown codes still have a parameter and human-readable message.","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].code","typeLabel":"string"},"message":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].message","typeLabel":"string"}},"required":["param","code","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"error.details.issues.items.[INDEX]","typeLabel":"object"},"uniqueKey":"error.details.issues","typeLabel":"object[]"}},"additionalProperties":{"oneOf":[{}],"uniqueKey":"error.details.additionalProperties","typeLabel":"any"},"uniqueKey":"error.details","typeLabel":"object"}},"required":["type","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"error","typeLabel":"object"}},"required":["error"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"","typeLabel":"object"},"examples":{"Example":{"title":"Example","value":{"error":{"type":"invalid_request","message":"\u003cstring>","reason":"\u003cstring>","param":"\u003cstring>","details":{"issues":[{"param":"\u003cstring>","code":"\u003cstring>","message":"\u003cstring>"}]}}}}},"exampleType":"generated"}},"exampleType":"generated"},"57c6da5a-3a4e-4691-832e-a8a3a4834739":{"description":"The Idempotency-Key's first result was deleted, so it cannot be replayed","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"object","properties":{"type":{"type":"string","enum":["invalid_request","authentication","permission_denied","not_found","conflict","rate_limited","engine_failed","unavailable","internal"],"isRequired":true,"uniqueKey":"error.type","typeLabel":"enum\u003cstring>"},"message":{"type":"string","description":"For people; don't parse it.","isRequired":true,"uniqueKey":"error.message","typeLabel":"string"},"reason":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Set where a caller can act on it, one of the reasons in the ErrorReason schema. New reasons may be added: treat an unknown one by the status and `type`.","uniqueKey":"error.reason","typeLabel":"string"},"param":{"type":"string","description":"The request field at fault, as a dotted path (`attribute_values.region`, `claims.1.value`).","uniqueKey":"error.param","typeLabel":"string"},"details":{"type":"object","properties":{"issues":{"type":"array","items":{"type":"object","properties":{"param":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].param","typeLabel":"string"},"code":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Stable, extensible issue code; unknown codes still have a parameter and human-readable message.","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].code","typeLabel":"string"},"message":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].message","typeLabel":"string"}},"required":["param","code","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"error.details.issues.items.[INDEX]","typeLabel":"object"},"uniqueKey":"error.details.issues","typeLabel":"object[]"}},"additionalProperties":{"oneOf":[{}],"uniqueKey":"error.details.additionalProperties","typeLabel":"any"},"uniqueKey":"error.details","typeLabel":"object"}},"required":["type","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"error","typeLabel":"object"}},"required":["error"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"","typeLabel":"object"},"examples":{"Example":{"title":"Example","value":{"error":{"type":"invalid_request","message":"\u003cstring>","reason":"\u003cstring>","param":"\u003cstring>","details":{"issues":[{"param":"\u003cstring>","code":"\u003cstring>","message":"\u003cstring>"}]}}}}},"exampleType":"generated"}},"exampleType":"generated"},"72572673-c7e8-4d26-80b7-8be239249c1c":{"description":"The HTTP body exceeds 1 MB (reason BODY_TOO_LARGE)","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"object","properties":{"type":{"type":"string","enum":["invalid_request","authentication","permission_denied","not_found","conflict","rate_limited","engine_failed","unavailable","internal"],"isRequired":true,"uniqueKey":"error.type","typeLabel":"enum\u003cstring>"},"message":{"type":"string","description":"For people; don't parse it.","isRequired":true,"uniqueKey":"error.message","typeLabel":"string"},"reason":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Set where a caller can act on it, one of the reasons in the ErrorReason schema. New reasons may be added: treat an unknown one by the status and `type`.","uniqueKey":"error.reason","typeLabel":"string"},"param":{"type":"string","description":"The request field at fault, as a dotted path (`attribute_values.region`, `claims.1.value`).","uniqueKey":"error.param","typeLabel":"string"},"details":{"type":"object","properties":{"issues":{"type":"array","items":{"type":"object","properties":{"param":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].param","typeLabel":"string"},"code":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Stable, extensible issue code; unknown codes still have a parameter and human-readable message.","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].code","typeLabel":"string"},"message":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].message","typeLabel":"string"}},"required":["param","code","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"error.details.issues.items.[INDEX]","typeLabel":"object"},"uniqueKey":"error.details.issues","typeLabel":"object[]"}},"additionalProperties":{"oneOf":[{}],"uniqueKey":"error.details.additionalProperties","typeLabel":"any"},"uniqueKey":"error.details","typeLabel":"object"}},"required":["type","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"error","typeLabel":"object"}},"required":["error"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"","typeLabel":"object"},"examples":{"Example":{"title":"Example","value":{"error":{"type":"invalid_request","message":"\u003cstring>","reason":"\u003cstring>","param":"\u003cstring>","details":{"issues":[{"param":"\u003cstring>","code":"\u003cstring>","message":"\u003cstring>"}]}}}}},"exampleType":"generated"}},"exampleType":"generated"},"4c13e08f-6ad0-49c0-add0-a2a6daeaeb33":{"description":"The request is well formed but cannot be processed (IDEMPOTENCY_KEY_REUSED requires a fresh key; REQUEST_TOO_LARGE requires a smaller request)","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"object","properties":{"type":{"type":"string","enum":["invalid_request","authentication","permission_denied","not_found","conflict","rate_limited","engine_failed","unavailable","internal"],"isRequired":true,"uniqueKey":"error.type","typeLabel":"enum\u003cstring>"},"message":{"type":"string","description":"For people; don't parse it.","isRequired":true,"uniqueKey":"error.message","typeLabel":"string"},"reason":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Set where a caller can act on it, one of the reasons in the ErrorReason schema. New reasons may be added: treat an unknown one by the status and `type`.","uniqueKey":"error.reason","typeLabel":"string"},"param":{"type":"string","description":"The request field at fault, as a dotted path (`attribute_values.region`, `claims.1.value`).","uniqueKey":"error.param","typeLabel":"string"},"details":{"type":"object","properties":{"issues":{"type":"array","items":{"type":"object","properties":{"param":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].param","typeLabel":"string"},"code":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Stable, extensible issue code; unknown codes still have a parameter and human-readable message.","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].code","typeLabel":"string"},"message":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].message","typeLabel":"string"}},"required":["param","code","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"error.details.issues.items.[INDEX]","typeLabel":"object"},"uniqueKey":"error.details.issues","typeLabel":"object[]"}},"additionalProperties":{"oneOf":[{}],"uniqueKey":"error.details.additionalProperties","typeLabel":"any"},"uniqueKey":"error.details","typeLabel":"object"}},"required":["type","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"error","typeLabel":"object"}},"required":["error"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"","typeLabel":"object"},"examples":{"Example":{"title":"Example","value":{"error":{"type":"invalid_request","message":"\u003cstring>","reason":"\u003cstring>","param":"\u003cstring>","details":{"issues":[{"param":"\u003cstring>","code":"\u003cstring>","message":"\u003cstring>"}]}}}}},"exampleType":"generated"}},"exampleType":"generated"},"f387853d-bba6-4e79-accb-261a8d5dd01c":{"description":"RATE_LIMITED: retry after Retry-After seconds; QUOTA_EXCEEDED: details.reset_at gives the daily reset time (no Retry-After)","headers":{"Retry-After":{"schema":{"type":"string","description":"Seconds to wait before retrying, when waiting helps."},"required":false,"description":"Seconds to wait before retrying, when waiting helps."}},"content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"object","properties":{"type":{"type":"string","enum":["invalid_request","authentication","permission_denied","not_found","conflict","rate_limited","engine_failed","unavailable","internal"],"isRequired":true,"uniqueKey":"error.type","typeLabel":"enum\u003cstring>"},"message":{"type":"string","description":"For people; don't parse it.","isRequired":true,"uniqueKey":"error.message","typeLabel":"string"},"reason":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Set where a caller can act on it, one of the reasons in the ErrorReason schema. New reasons may be added: treat an unknown one by the status and `type`.","uniqueKey":"error.reason","typeLabel":"string"},"param":{"type":"string","description":"The request field at fault, as a dotted path (`attribute_values.region`, `claims.1.value`).","uniqueKey":"error.param","typeLabel":"string"},"details":{"type":"object","properties":{"issues":{"type":"array","items":{"type":"object","properties":{"param":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].param","typeLabel":"string"},"code":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Stable, extensible issue code; unknown codes still have a parameter and human-readable message.","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].code","typeLabel":"string"},"message":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].message","typeLabel":"string"}},"required":["param","code","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"error.details.issues.items.[INDEX]","typeLabel":"object"},"uniqueKey":"error.details.issues","typeLabel":"object[]"}},"additionalProperties":{"oneOf":[{}],"uniqueKey":"error.details.additionalProperties","typeLabel":"any"},"uniqueKey":"error.details","typeLabel":"object"}},"required":["type","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"error","typeLabel":"object"}},"required":["error"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"","typeLabel":"object"},"examples":{"Example":{"title":"Example","value":{"error":{"type":"invalid_request","message":"\u003cstring>","reason":"\u003cstring>","param":"\u003cstring>","details":{"issues":[{"param":"\u003cstring>","code":"\u003cstring>","message":"\u003cstring>"}]}}}}},"exampleType":"generated"}},"exampleType":"generated"},"109e198e-46f0-48e9-9c58-7f6a7905ad05":{"description":"A model failed to answer a question the result needs (type engine_failed, reason ENGINE_FAILED)","content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"object","properties":{"type":{"type":"string","enum":["invalid_request","authentication","permission_denied","not_found","conflict","rate_limited","engine_failed","unavailable","internal"],"isRequired":true,"uniqueKey":"error.type","typeLabel":"enum\u003cstring>"},"message":{"type":"string","description":"For people; don't parse it.","isRequired":true,"uniqueKey":"error.message","typeLabel":"string"},"reason":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Set where a caller can act on it, one of the reasons in the ErrorReason schema. New reasons may be added: treat an unknown one by the status and `type`.","uniqueKey":"error.reason","typeLabel":"string"},"param":{"type":"string","description":"The request field at fault, as a dotted path (`attribute_values.region`, `claims.1.value`).","uniqueKey":"error.param","typeLabel":"string"},"details":{"type":"object","properties":{"issues":{"type":"array","items":{"type":"object","properties":{"param":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].param","typeLabel":"string"},"code":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Stable, extensible issue code; unknown codes still have a parameter and human-readable message.","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].code","typeLabel":"string"},"message":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].message","typeLabel":"string"}},"required":["param","code","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"error.details.issues.items.[INDEX]","typeLabel":"object"},"uniqueKey":"error.details.issues","typeLabel":"object[]"}},"additionalProperties":{"oneOf":[{}],"uniqueKey":"error.details.additionalProperties","typeLabel":"any"},"uniqueKey":"error.details","typeLabel":"object"}},"required":["type","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"error","typeLabel":"object"}},"required":["error"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"","typeLabel":"object"},"examples":{"Example":{"title":"Example","value":{"error":{"type":"invalid_request","message":"\u003cstring>","reason":"\u003cstring>","param":"\u003cstring>","details":{"issues":[{"param":"\u003cstring>","code":"\u003cstring>","message":"\u003cstring>"}]}}}}},"exampleType":"generated"}},"exampleType":"generated"},"8bd3be84-bfbb-4492-a3d1-8a6e0b9b6861":{"description":"Temporarily unavailable, with Retry-After (for example reason AUTH_UNAVAILABLE, ENGINE_UNAVAILABLE or STORE_UNAVAILABLE): retry with the same Idempotency-Key","headers":{"Retry-After":{"schema":{"type":"string","description":"Seconds to wait before retrying, when waiting helps."},"required":true,"description":"Seconds to wait before retrying, when waiting helps."}},"content":{"application/json":{"schema":{"type":"object","properties":{"error":{"type":"object","properties":{"type":{"type":"string","enum":["invalid_request","authentication","permission_denied","not_found","conflict","rate_limited","engine_failed","unavailable","internal"],"isRequired":true,"uniqueKey":"error.type","typeLabel":"enum\u003cstring>"},"message":{"type":"string","description":"For people; don't parse it.","isRequired":true,"uniqueKey":"error.message","typeLabel":"string"},"reason":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Set where a caller can act on it, one of the reasons in the ErrorReason schema. New reasons may be added: treat an unknown one by the status and `type`.","uniqueKey":"error.reason","typeLabel":"string"},"param":{"type":"string","description":"The request field at fault, as a dotted path (`attribute_values.region`, `claims.1.value`).","uniqueKey":"error.param","typeLabel":"string"},"details":{"type":"object","properties":{"issues":{"type":"array","items":{"type":"object","properties":{"param":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].param","typeLabel":"string"},"code":{"type":"string","pattern":"^[A-Z][A-Z0-9_]+$","description":"Stable, extensible issue code; unknown codes still have a parameter and human-readable message.","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].code","typeLabel":"string"},"message":{"type":"string","isRequired":true,"uniqueKey":"error.details.issues.items.[INDEX].message","typeLabel":"string"}},"required":["param","code","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"error.details.issues.items.[INDEX]","typeLabel":"object"},"uniqueKey":"error.details.issues","typeLabel":"object[]"}},"additionalProperties":{"oneOf":[{}],"uniqueKey":"error.details.additionalProperties","typeLabel":"any"},"uniqueKey":"error.details","typeLabel":"object"}},"required":["type","message"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"isRequired":true,"uniqueKey":"error","typeLabel":"object"}},"required":["error"],"additionalProperties":true,"x-moonbounce-sdk-known-fields":true,"uniqueKey":"","typeLabel":"object"},"examples":{"Example":{"title":"Example","value":{"error":{"type":"invalid_request","message":"\u003cstring>","reason":"\u003cstring>","param":"\u003cstring>","details":{"issues":[{"param":"\u003cstring>","code":"\u003cstring>","message":"\u003cstring>"}]}}}}},"exampleType":"generated"}},"exampleType":"generated"}},"security":{"cedf8060-3208-4cb4-ae4c-4d9c08a5df3c":{"bearerAuth":{"type":"http","scheme":"bearer","description":"An API key, sent as `Authorization: Bearer \u003ckey>`. Each operation lists the scope its key needs: `decisions:evaluate` (evaluate and read evaluations), `decisions:feedback` (your app's feedback) or `decisions:develop` (everything else, including owner feedback and the review queue). An operation listing no scope takes any scoped key. A key with no scope, or without the operation's, gets 403 INSUFFICIENT_SCOPE.","scopes":["decisions:develop"],"schemeName":"bearerAuth"}}},"servers":{"48eee345-ee42-4a1e-aa8f-c5d5e9868495":{"url":"https://api.moonbounce.ai","description":"Production"}},"schemas":{"889c258c-6d37-45b1-895e-cb853a5b9e3e":{"oneOf":[{"type":"object","required":["name"],"properties":{"name":{"type":"string","minLength":1,"maxLength":80,"description":"A feature's path: `questions.\u003cname>`, `attributes.\u003cname>` or `aggregations.\u003cname>`.","examples":["questions.asks_for_refund","aggregations.band"],"isRequired":true},"gte":{"type":"number"},"gt":{"type":"number"},"lte":{"type":"number"},"lt":{"type":"number"},"is":{"type":"string","minLength":1,"maxLength":200},"in":{"type":"array","items":{"type":"string","minLength":1,"maxLength":200},"minItems":1,"maxItems":255},"abstained":{"type":"boolean","enum":[true]}},"additionalProperties":false},{"type":"object","required":["all"],"properties":{"all":{"type":"array","items":{"$circularRef":"889c258c-6d37-45b1-895e-cb853a5b9e3e"},"minItems":1,"maxItems":64,"isRequired":true}},"additionalProperties":false},{"type":"object","required":["any"],"properties":{"any":{"type":"array","items":{"$circularRef":"889c258c-6d37-45b1-895e-cb853a5b9e3e"},"minItems":1,"maxItems":64,"isRequired":true}},"additionalProperties":false},{"type":"object","required":["not"],"properties":{"not":{"oneOf":[{"isRequired":true}],"isRequired":true}},"additionalProperties":false}]}},"processedSecurityOptions":[{"schemes":[{"scheme":{"type":"http","scheme":"bearer","description":"An API key, sent as `Authorization: Bearer \u003ckey>`. Each operation lists the scope its key needs: `decisions:evaluate` (evaluate and read evaluations), `decisions:feedback` (your app's feedback) or `decisions:develop` (everything else, including owner feedback and the review queue). An operation listing no scope takes any scoped key. A key with no scope, or without the operation's, gets 403 INSUFFICIENT_SCOPE.","scopes":["decisions:develop"],"schemeName":"bearerAuth"},"schema":{"title":"Bearer Auth","type":"string","isRequired":true,"description":"An API key, sent as `Authorization: Bearer \u003ckey>`. Each operation lists the scope its key needs: `decisions:evaluate` (evaluate and read evaluations), `decisions:feedback` (your app's feedback) or `decisions:develop` (everything else, including owner feedback and the review queue). An operation listing no scope takes any scoped key. A key with no scope, or without the operation's, gets 403 INSUFFICIENT_SCOPE.","format":"bearer","typeLabel":"string\u003cbearer>","deprecated":false},"schemeName":"bearerAuth","name":"Authorization"}],"title":"bearerAuth"}]},"originalFileLocation":"openapi.json"},"subdomain":"moonbounce","siteName":"Moonbounce Decisions API","title":"Preview a structure change against past feedback - Moonbounce Decisions API","navMaps":{"firstHrefInVersion":{},"firstHrefInLanguage":{}},"lastModified":"2026-11-07T04:38:11.956Z","originalFileLocation":"api-reference/previews-\u0026-tests/preview-a-structure-change-against-past-feedback.mdx"},"headHtml":"\u003cscript data-mint-head>(function(){var d=document.documentElement,p=null;if(!p){try{p=localStorage.getItem(\"isDarkMode\")}catch(e){}if(p==='true')p='dark';else if(p==='false')p='light';if(p!=='light'\u0026\u0026p!=='dark'\u0026\u0026p!=='system')p=\"system\"}var r=p==='system'?(matchMedia(\"(prefers-color-scheme: dark)\").matches?'dark':'light'):p;d.classList.remove(r==='dark'?'light':'dark');d.classList.add(r);d.setAttribute(\"data-theme-preference\",p)})();\u003c/script>\u003cmeta data-mint-head name=\"description\" content=\"Replays a structure change over past cases with feedback: it spends model calls and stores a run. Not a dry run of evaluate. Replays a local structure and its base over the decision\u0026#x27;s reviewed and imported cases in the key\u0026#x27;s environment (a test key previews against test cases only), and returns fixes, breaks, the action mix, the loss_report (base and candidate judged by the decision\u0026#x27;s objective, or balanced losses when none is set, with loss per case, macro-F1, where the cost sits, the costliest mistakes, and whether the candidate is accepted) and an evidence label: training_cases when a scored case is a training case (its deciding feedback is not an import), else imported_examples (every scored case is an imported example); either way the cases are ones a suggestion may have learned from. The result must decide outcomes (bands, rules or match): a value result is 400 OUTCOMES_INVALID. Each case reuses the answers its evaluation stored for questions whose hash is unchanged; changed and new questions are asked the way evaluate asks them (each question\u0026#x27;s reads, grouped into System One calls), so a preview costs what evaluating the changed questions would. Writes an eval run only: never an evaluation, never traffic, never acceptance. A preview does not register the candidate: before evaluating by its structure hash, register it with POST /v1/decisions (or send the structure inline on evaluate once). The base, in order: the base of suggestion_id when it is sent (then base is ignored), else base, else the version serving primary traffic in the key\u0026#x27;s environment, else the newest one examples were imported under there, else none. With suggestion_id, hashes_match says whether the structure is that suggestion\u0026#x27;s candidate; another environment\u0026#x27;s suggestion is 404. The evidence replays only cases with an outcome from the owner, reviewers or imported examples; the action mix replays a sample of the environment\u0026#x27;s recent primary traffic. For questions the candidate changes, before is a fresh re-run of the base question (the fair baseline), so model nondeterminism can move it between previews. noise_rate measures outcome flips from that re-run against the stored answers, not value drift. A test key\u0026#x27;s preview neither reads nor fills the shared cache of model answers. A decision nothing was registered, evaluated or imported under is 404.\">\u003cmeta data-mint-head name=\"application-name\" content=\"Moonbounce Decisions API\">\u003cmeta data-mint-head name=\"generator\" content=\"Mintlify\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"192x192\" href=\"/mintlify-assets/_mintlify/favicons/moonbounce/Pq6Jhs_BrzDTLcf2/_generated/favicon/android-chrome-192x192.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"apple-touch-icon\" type=\"image/png\" sizes=\"180x180\" href=\"/mintlify-assets/_mintlify/favicons/moonbounce/Pq6Jhs_BrzDTLcf2/_generated/favicon/apple-touch-icon.png\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"/mintlify-assets/_mintlify/favicons/moonbounce/Pq6Jhs_BrzDTLcf2/_generated/favicon/favicon-16x16.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"/mintlify-assets/_mintlify/favicons/moonbounce/Pq6Jhs_BrzDTLcf2/_generated/favicon/favicon-32x32.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"/mintlify-assets/_mintlify/favicons/moonbounce/Pq6Jhs_BrzDTLcf2/_generated/favicon/favicon.ico\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"192x192\" href=\"/mintlify-assets/_mintlify/favicons/moonbounce/Pq6Jhs_BrzDTLcf2/_generated/favicon-dark/android-chrome-192x192.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"/mintlify-assets/_mintlify/favicons/moonbounce/Pq6Jhs_BrzDTLcf2/_generated/favicon-dark/favicon-16x16.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"/mintlify-assets/_mintlify/favicons/moonbounce/Pq6Jhs_BrzDTLcf2/_generated/favicon-dark/favicon-32x32.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"/mintlify-assets/_mintlify/favicons/moonbounce/Pq6Jhs_BrzDTLcf2/_generated/favicon-dark/favicon.ico\" media=\"(prefers-color-scheme: dark)\">\u003cmeta data-mint-head property=\"og:title\" content=\"Preview a structure change against past feedback - Moonbounce Decisions API\">\u003cmeta data-mint-head property=\"og:description\" content=\"Replays a structure change over past cases with feedback: it spends model calls and stores a run. Not a dry run of evaluate. Replays a local structure and its base over the decision\u0026#x27;s reviewed and imported cases in the key\u0026#x27;s environment (a test key previews against test cases only), and returns fixes, breaks, the action mix, the loss_report (base and candidate judged by the decision\u0026#x27;s objective, or balanced losses when none is set, with loss per case, macro-F1, where the cost sits, the costliest mistakes, and whether the candidate is accepted) and an evidence label: training_cases when a scored case is a training case (its deciding feedback is not an import), else imported_examples (every scored case is an imported example); either way the cases are ones a suggestion may have learned from. The result must decide outcomes (bands, rules or match): a value result is 400 OUTCOMES_INVALID. Each case reuses the answers its evaluation stored for questions whose hash is unchanged; changed and new questions are asked the way evaluate asks them (each question\u0026#x27;s reads, grouped into System One calls), so a preview costs what evaluating the changed questions would. Writes an eval run only: never an evaluation, never traffic, never acceptance. A preview does not register the candidate: before evaluating by its structure hash, register it with POST /v1/decisions (or send the structure inline on evaluate once). The base, in order: the base of suggestion_id when it is sent (then base is ignored), else base, else the version serving primary traffic in the key\u0026#x27;s environment, else the newest one examples were imported under there, else none. With suggestion_id, hashes_match says whether the structure is that suggestion\u0026#x27;s candidate; another environment\u0026#x27;s suggestion is 404. The evidence replays only cases with an outcome from the owner, reviewers or imported examples; the action mix replays a sample of the environment\u0026#x27;s recent primary traffic. For questions the candidate changes, before is a fresh re-run of the base question (the fair baseline), so model nondeterminism can move it between previews. noise_rate measures outcome flips from that re-run against the stored answers, not value drift. A test key\u0026#x27;s preview neither reads nor fills the shared cache of model answers. A decision nothing was registered, evaluated or imported under is 404.\">\u003cmeta data-mint-head property=\"og:site_name\" content=\"Moonbounce Decisions API\">\u003cmeta data-mint-head property=\"og:url\" content=\"https://docs.moonbounce.ai/api-reference/previews-\u0026amp;-tests/preview-a-structure-change-against-past-feedback\">\u003cmeta data-mint-head property=\"og:type\" content=\"website\">\u003cmeta data-mint-head property=\"og:image\" content=\"https://moonbounce.mintlify.site/_mintlify/api/og/v2?division=Previews+%26+tests\u0026amp;title=Preview+a+structure+change+against+past+feedback\u0026amp;description=Replays+a+structure+change+over+past+cases+with+feedback%3A+it+spends+model+calls+and+stores+a+run.+Not+a+dry+run+of+evaluate.+Replays+a+local+structure+and+its+b\u0026amp;theme=e6d1608244ab860cac0ff47e\u0026amp;signature=98ae83cea424cd3855f1315c7ed18905eb155459bb7c77e24086fad46a14a3d7\">\u003cmeta data-mint-head property=\"og:image:width\" content=\"1200\">\u003cmeta data-mint-head property=\"og:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"twitter:card\" content=\"summary_large_image\">\u003cmeta data-mint-head name=\"twitter:title\" content=\"Preview a structure change against past feedback - Moonbounce Decisions API\">\u003cmeta data-mint-head name=\"twitter:description\" content=\"Replays a structure change over past cases with feedback: it spends model calls and stores a run. Not a dry run of evaluate. Replays a local structure and its base over the decision\u0026#x27;s reviewed and imported cases in the key\u0026#x27;s environment (a test key previews against test cases only), and returns fixes, breaks, the action mix, the loss_report (base and candidate judged by the decision\u0026#x27;s objective, or balanced losses when none is set, with loss per case, macro-F1, where the cost sits, the costliest mistakes, and whether the candidate is accepted) and an evidence label: training_cases when a scored case is a training case (its deciding feedback is not an import), else imported_examples (every scored case is an imported example); either way the cases are ones a suggestion may have learned from. The result must decide outcomes (bands, rules or match): a value result is 400 OUTCOMES_INVALID. Each case reuses the answers its evaluation stored for questions whose hash is unchanged; changed and new questions are asked the way evaluate asks them (each question\u0026#x27;s reads, grouped into System One calls), so a preview costs what evaluating the changed questions would. Writes an eval run only: never an evaluation, never traffic, never acceptance. A preview does not register the candidate: before evaluating by its structure hash, register it with POST /v1/decisions (or send the structure inline on evaluate once). The base, in order: the base of suggestion_id when it is sent (then base is ignored), else base, else the version serving primary traffic in the key\u0026#x27;s environment, else the newest one examples were imported under there, else none. With suggestion_id, hashes_match says whether the structure is that suggestion\u0026#x27;s candidate; another environment\u0026#x27;s suggestion is 404. The evidence replays only cases with an outcome from the owner, reviewers or imported examples; the action mix replays a sample of the environment\u0026#x27;s recent primary traffic. For questions the candidate changes, before is a fresh re-run of the base question (the fair baseline), so model nondeterminism can move it between previews. noise_rate measures outcome flips from that re-run against the stored answers, not value drift. A test key\u0026#x27;s preview neither reads nor fills the shared cache of model answers. A decision nothing was registered, evaluated or imported under is 404.\">\u003cmeta data-mint-head name=\"twitter:image\" content=\"https://moonbounce.mintlify.site/_mintlify/api/og/v2?division=Previews+%26+tests\u0026amp;title=Preview+a+structure+change+against+past+feedback\u0026amp;description=Replays+a+structure+change+over+past+cases+with+feedback%3A+it+spends+model+calls+and+stores+a+run.+Not+a+dry+run+of+evaluate.+Replays+a+local+structure+and+its+b\u0026amp;theme=e6d1608244ab860cac0ff47e\u0026amp;signature=98ae83cea424cd3855f1315c7ed18905eb155459bb7c77e24086fad46a14a3d7\">\u003cmeta data-mint-head name=\"twitter:image:width\" content=\"1200\">\u003cmeta data-mint-head name=\"twitter:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"msapplication-config\" content=\"/mintlify-assets/_mintlify/favicons/moonbounce/Pq6Jhs_BrzDTLcf2/_generated/favicon/browserconfig.xml\">\u003cmeta data-mint-head name=\"apple-mobile-web-app-title\" content=\"Moonbounce Decisions API\">\u003cmeta data-mint-head name=\"msapplication-TileColor\" content=\"#1D5AB5\">\u003clink data-mint-head rel=\"canonical\" href=\"https://docs.moonbounce.ai/api-reference/previews-\u0026amp;-tests/preview-a-structure-change-against-past-feedback\">\u003clink data-mint-head rel=\"alternate\" type=\"application/xml\" href=\"/sitemap.xml\">\u003clink data-mint-head rel=\"alternate\" type=\"text/markdown\" href=\"/api-reference/previews-%26-tests/preview-a-structure-change-against-past-feedback.md\">\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@graph\":[{\"@type\":\"Organization\",\"@id\":\"https://docs.moonbounce.ai/#organization\",\"name\":\"Moonbounce Decisions API\",\"url\":\"https://docs.moonbounce.ai\",\"logo\":{\"@type\":\"ImageObject\",\"url\":\"https://mintcdn.com/moonbounce/mWuXGW-nUJSEdABp/logo/lockup-light.svg?fit=max\u0026auto=format\u0026n=mWuXGW-nUJSEdABp\u0026q=85\u0026s=367a5dedb5799a16c5737d136af68dc5\"}},{\"@type\":\"WebSite\",\"@id\":\"https://docs.moonbounce.ai#website\",\"name\":\"Moonbounce Decisions API\",\"url\":\"https://docs.moonbounce.ai\",\"publisher\":{\"@id\":\"https://docs.moonbounce.ai/#organization\"}},{\"@type\":\"WebPage\",\"@id\":\"https://docs.moonbounce.ai/api-reference/previews-\u0026-tests/preview-a-structure-change-against-past-feedback#webpage\",\"url\":\"https://docs.moonbounce.ai/api-reference/previews-\u0026-tests/preview-a-structure-change-against-past-feedback\",\"name\":\"Preview a structure change against past feedback\",\"description\":\"Replays a structure change over past cases with feedback: it spends model calls and stores a run. Not a dry run of evaluate. Replays a local structure and its base over the decision's reviewed and imported cases in the key's environment (a test key previews against test cases only), and returns fixes, breaks, the action mix, the loss_report (base and candidate judged by the decision's objective, or balanced losses when none is set, with loss per case, macro-F1, where the cost sits, the costliest mistakes, and whether the candidate is accepted) and an evidence label: training_cases when a scored case is a training case (its deciding feedback is not an import), else imported_examples (every scored case is an imported example); either way the cases are ones a suggestion may have learned from. The result must decide outcomes (bands, rules or match): a value result is 400 OUTCOMES_INVALID. Each case reuses the answers its evaluation stored for questions whose hash is unchanged; changed and new questions are asked the way evaluate asks them (each question's reads, grouped into System One calls), so a preview costs what evaluating the changed questions would. Writes an eval run only: never an evaluation, never traffic, never acceptance. A preview does not register the candidate: before evaluating by its structure hash, register it with POST /v1/decisions (or send the structure inline on evaluate once). The base, in order: the base of suggestion_id when it is sent (then base is ignored), else base, else the version serving primary traffic in the key's environment, else the newest one examples were imported under there, else none. With suggestion_id, hashes_match says whether the structure is that suggestion's candidate; another environment's suggestion is 404. The evidence replays only cases with an outcome from the owner, reviewers or imported examples; the action mix replays a sample of the environment's recent primary traffic. For questions the candidate changes, before is a fresh re-run of the base question (the fair baseline), so model nondeterminism can move it between previews. noise_rate measures outcome flips from that re-run against the stored answers, not value drift. A test key's preview neither reads nor fills the shared cache of model answers. A decision nothing was registered, evaluated or imported under is 404.\",\"dateModified\":\"2026-11-07T04:38:11.956Z\",\"isPartOf\":{\"@id\":\"https://docs.moonbounce.ai#website\"},\"breadcrumb\":{\"@id\":\"https://docs.moonbounce.ai/api-reference/previews-\u0026-tests/preview-a-structure-change-against-past-feedback#breadcrumb\"}},{\"@type\":\"BreadcrumbList\",\"@id\":\"https://docs.moonbounce.ai/api-reference/previews-\u0026-tests/preview-a-structure-change-against-past-feedback#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Previews \u0026 tests\",\"item\":\"https://docs.moonbounce.ai/api-reference/previews-\u0026-tests/run-a-version-over-many-items\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Preview a structure change against past feedback\",\"item\":\"https://docs.moonbounce.ai/api-reference/previews-\u0026-tests/preview-a-structure-change-against-past-feedback\"}]},{\"@type\":[\"TechArticle\",\"APIReference\"],\"@id\":\"https://docs.moonbounce.ai/api-reference/previews-\u0026-tests/preview-a-structure-change-against-past-feedback#apireference\",\"headline\":\"Preview a structure change against past feedback\",\"name\":\"Preview a structure change against past feedback\",\"description\":\"Replays a structure change over past cases with feedback: it spends model calls and stores a run. Not a dry run of evaluate. Replays a local structure and its base over the decision's reviewed and imported cases in the key's environment (a test key previews against test cases only), and returns fixes, breaks, the action mix, the loss_report (base and candidate judged by the decision's objective, or balanced losses when none is set, with loss per case, macro-F1, where the cost sits, the costliest mistakes, and whether the candidate is accepted) and an evidence label: training_cases when a scored case is a training case (its deciding feedback is not an import), else imported_examples (every scored case is an imported example); either way the cases are ones a suggestion may have learned from. The result must decide outcomes (bands, rules or match): a value result is 400 OUTCOMES_INVALID. Each case reuses the answers its evaluation stored for questions whose hash is unchanged; changed and new questions are asked the way evaluate asks them (each question's reads, grouped into System One calls), so a preview costs what evaluating the changed questions would. Writes an eval run only: never an evaluation, never traffic, never acceptance. A preview does not register the candidate: before evaluating by its structure hash, register it with POST /v1/decisions (or send the structure inline on evaluate once). The base, in order: the base of suggestion_id when it is sent (then base is ignored), else base, else the version serving primary traffic in the key's environment, else the newest one examples were imported under there, else none. With suggestion_id, hashes_match says whether the structure is that suggestion's candidate; another environment's suggestion is 404. The evidence replays only cases with an outcome from the owner, reviewers or imported examples; the action mix replays a sample of the environment's recent primary traffic. For questions the candidate changes, before is a fresh re-run of the base question (the fair baseline), so model nondeterminism can move it between previews. noise_rate measures outcome flips from that re-run against the stored answers, not value drift. A test key's preview neither reads nor fills the shared cache of model answers. A decision nothing was registered, evaluated or imported under is 404.\",\"url\":\"https://docs.moonbounce.ai/api-reference/previews-\u0026-tests/preview-a-structure-change-against-past-feedback\",\"mainEntityOfPage\":{\"@id\":\"https://docs.moonbounce.ai/api-reference/previews-\u0026-tests/preview-a-structure-change-against-past-feedback#webpage\"},\"dateModified\":\"2026-11-07T04:38:11.956Z\",\"publisher\":{\"@id\":\"https://docs.moonbounce.ai/#organization\"},\"isPartOf\":{\"@id\":\"https://docs.moonbounce.ai#website\"}}]}\u003c/script>\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@type\":\"WebSite\",\"name\":\"Moonbounce Decisions API\",\"creator\":{\"@type\":\"Organization\",\"name\":\"Mintlify\",\"url\":\"https://mintlify.com\"}}\u003c/script>","siteName":"Moonbounce Decisions API","nav":{"global":null,"tabs":[{"tab":"Guides","pages":[{"title":"Guides","href":""}]},{"tab":"API reference","groups":[{"group":"Overview","pages":[{"title":"API reference overview","href":"/reference/overview"}]},{"group":"Evaluate","pages":[{"title":"Evaluate an item","deprecated":null,"version":null,"openapi":"/openapi.json post /v1/evaluate","href":"/api-reference/evaluate/evaluate-an-item"},{"title":"Get a past evaluation","deprecated":null,"version":null,"openapi":"/openapi.json get /v1/evaluations/{evaluation_id}","href":"/api-reference/evaluate/get-a-past-evaluation"},{"title":"List a decision's evaluations","deprecated":null,"version":null,"openapi":"/openapi.json get /v1/decisions/{decision_id}/evaluations","href":"/api-reference/evaluate/list-a-decisions-evaluations"}]},{"group":"Decisions","pages":[{"title":"Validate a decision file","deprecated":null,"version":null,"openapi":"/openapi.json post /v1/validate","href":"/api-reference/decisions/validate-a-decision-file"},{"title":"List decisions","deprecated":null,"version":null,"openapi":"/openapi.json get /v1/decisions","href":"/api-reference/decisions/list-decisions"},{"title":"Register a decision's version","deprecated":null,"version":null,"openapi":"/openapi.json post /v1/decisions","href":"/api-reference/decisions/register-a-decisions-version"},{"title":"Get a version","deprecated":null,"version":null,"openapi":"/openapi.json get /v1/decisions/{decision_id}/versions/{ref}","href":"/api-reference/decisions/get-a-version"},{"title":"Get a decision","deprecated":null,"version":null,"openapi":"/openapi.json get /v1/decisions/{decision_id}","href":"/api-reference/decisions/get-a-decision"},{"title":"List a decision's versions","deprecated":null,"version":null,"openapi":"/openapi.json get /v1/decisions/{decision_id}/versions","href":"/api-reference/decisions/list-a-decisions-versions"}]},{"group":"Feedback","pages":[{"title":"List a decision's labeled cases","deprecated":null,"version":null,"openapi":"/openapi.json get /v1/decisions/{decision_id}/cases","href":"/api-reference/feedback/list-a-decisions-labeled-cases"},{"title":"Submit feedback on evaluations or items","deprecated":null,"version":null,"openapi":"/openapi.json post /v1/feedback","href":"/api-reference/feedback/submit-feedback-on-evaluations-or-items"},{"title":"Get a feedback submission as stored","deprecated":null,"version":null,"openapi":"/openapi.json get /v1/feedback/{feedback_id}","href":"/api-reference/feedback/get-a-feedback-submission-as-stored"},{"title":"Export a decision's feedback as training data","deprecated":null,"version":null,"openapi":"/openapi.json get /v1/decisions/{decision_id}/feedback","href":"/api-reference/feedback/export-a-decisions-feedback-as-training-data"},{"title":"Import labeled examples","deprecated":null,"version":null,"openapi":"/openapi.json post /v1/decisions/{decision_id}/examples","href":"/api-reference/feedback/import-labeled-examples"}]},{"group":"Suggestions","pages":[{"title":"List suggestions for a decision","deprecated":null,"version":null,"openapi":"/openapi.json get /v1/decisions/{decision_id}/suggestions","href":"/api-reference/suggestions/list-suggestions-for-a-decision"},{"title":"Generate a suggestion on demand","deprecated":null,"version":null,"openapi":"/openapi.json post /v1/decisions/{decision_id}/suggestions","href":"/api-reference/suggestions/generate-a-suggestion-on-demand"},{"title":"Get a suggestion","deprecated":null,"version":null,"openapi":"/openapi.json get /v1/suggestions/{suggestion_id}","href":"/api-reference/suggestions/get-a-suggestion"},{"title":"Mark a suggestion pulled or rejected","deprecated":null,"version":null,"openapi":"/openapi.json post /v1/suggestions/{suggestion_id}/status","href":"/api-reference/suggestions/mark-a-suggestion-pulled-or-rejected"},{"title":"Get the improvement loop's status","deprecated":null,"version":null,"openapi":"/openapi.json get /v1/decisions/{decision_id}/loop","href":"/api-reference/suggestions/get-the-improvement-loops-status"}]},{"group":"Previews \u0026 tests","pages":[{"title":"Run a version over many items","description":"Evaluates up to 200 items under one version, each exactly as one evaluate would, and keeps the results under a `run_id` for loss_report and sweep_cut. The body is flat, as import_examples' is: the structure inline (`questions`, `attributes`, `aggregations`, `result`) or a registered one's `structure_hash`, never both; then `items` or `cases`, `dry_run` and `include`. The decision is the path's. **Items:** each `{item_id, state, attribute_values?, expected?, label_source?}`, `item_id` unique in the run. Each is stored as an evaluation, or with `dry_run: true` is a dry run that stores only usage; each result carries its `id` (`evl_` or `dry_`). Score versions with dry runs: a stored run's evaluations are real traffic. `expected` (an outcome) and `label_source` (`human`, `synthetic` or `unknown`) are the item's label, kept with the run for loss_report, never used to evaluate and never recorded as feedback. Every item's content is checked before any model call: content the structure refuses is 400 CONTENT_INVALID naming `items.\u003ci>`. An item whose model calls fail is that item's result (status `failed`, with an `error` whose `code` says why and `retryable` whether sending it again may help); the call fails only on an invalid request, auth, admission or a store error. An outage of any kind (unavailable, rate limited, timeout, auth) is `retryable`; only the content's own failure is not. **Stored cases:** send `cases` in place of `items` to evaluate the decision's labeled cases (the ones list_cases lists; this needs the `decisions:develop` scope): `{evaluation_ids}`, or list_cases' filters (`outcomes`, `structure_hash`, `label_source`, `since`, `until`) with `limit` (default 200) and `cursor` (`cases.next_cursor` from the previous run). They are always dry runs: each case is replayed as a preview replays it, reusing the answer it stored for a question that hasn't changed (same model) and asking the rest; `refresh` names questions to ask afresh. Each result carries the case's `evaluation_id`, and the run keeps its label, label source and answer corrections, so a preview equals a case run under the base (refreshing the questions the candidate changed or dropped), one under the candidate, and loss_report over the two, when both runs set `cases.outcomes` to the outcomes either version can produce and cover the same cases (a preview judges the newest 500: page both runs through them while no label changes; a page is not a snapshot), and both versions can read every case's stored content. The run reports `version` and `model` as evaluate does. It is kept until `expires_at`: pass its `run_id` to loss_report to judge it, so the results never pass through your context; a stored run's evaluations stay after it expires, and it can't be read back by id. `include` adds each result's `answers`, `attributes`, or `evaluation` (everything one evaluate returns for that item). Each item that succeeded is billed as one evaluate or dry run would be; a failed item's spend is recorded and shown as its `cost`. A run has a 60 s budget and takes no `model`; items it could not start fail with `NOT_STARTED`, and a case it ran out of time for with `ENGINE_UNAVAILABLE` (both retryable): send them in a new run with a new Idempotency-Key. 201 with the run; an Idempotency-Key replays the first response without spending again.","deprecated":null,"version":null,"openapi":"/openapi.json post /v1/decisions/{decision_id}/runs","href":"/api-reference/previews-\u0026-tests/run-a-version-over-many-items"},{"title":"Preview a structure change against past feedback","deprecated":null,"version":null,"openapi":"/openapi.json post /v1/decisions/{decision_id}/previews","href":"/api-reference/previews-\u0026-tests/preview-a-structure-change-against-past-feedback"}]},{"group":"Models","pages":[{"title":"List the models a decision can run on","description":"The models this deployment offers and what each reports: the resolved model, whether it is pinned, the question types and content it reads, and its list price. `default` is the model a decision runs on until you choose one. Choose one for a decision with `PUT /v1/decisions/{decision_id}/model`.","deprecated":null,"version":null,"openapi":"/openapi.json get /v1/models","href":"/api-reference/models/list-the-models-a-decision-can-run-on"},{"title":"Get the model a decision runs on","deprecated":null,"version":null,"openapi":"/openapi.json get /v1/decisions/{decision_id}/model","href":"/api-reference/models/get-the-model-a-decision-runs-on"},{"title":"Choose the model a decision runs on","deprecated":null,"version":null,"openapi":"/openapi.json put /v1/decisions/{decision_id}/model","href":"/api-reference/models/choose-the-model-a-decision-runs-on"}]},{"group":"Objective","pages":[{"title":"Sweep a cut over a run's answers","deprecated":null,"version":null,"openapi":"/openapi.json post /v1/decisions/{decision_id}/cut-sweeps","href":"/api-reference/objective/sweep-a-cut-over-a-runs-answers"},{"title":"Judge results under a decision's objective","deprecated":null,"version":null,"openapi":"/openapi.json post /v1/decisions/{decision_id}/loss-reports","href":"/api-reference/objective/judge-results-under-a-decisions-objective"},{"title":"Get what a decision optimizes for","deprecated":null,"version":null,"openapi":"/openapi.json get /v1/decisions/{decision_id}/objective","href":"/api-reference/objective/get-what-a-decision-optimizes-for"},{"title":"Set what a decision optimizes for","deprecated":null,"version":null,"openapi":"/openapi.json put /v1/decisions/{decision_id}/objective","href":"/api-reference/objective/set-what-a-decision-optimizes-for"}]},{"group":"Usage","pages":[{"title":"Get a decision's model cost, per evaluation and in total","deprecated":null,"version":null,"openapi":"/openapi.json get /v1/decisions/{decision_id}/usage","href":"/api-reference/usage/get-a-decisions-model-cost-per-evaluation-and-in-total"}]},{"group":"Account","pages":[{"title":"Health check","deprecated":null,"version":null,"openapi":"/openapi.json get /health","href":"/api-reference/account/health-check"},{"title":"Identify the calling key or sign-in","deprecated":null,"version":null,"openapi":"/openapi.json get /v1/whoami","href":"/api-reference/account/identify-the-calling-key-or-sign-in"}]}]}]},"basePath":"","title":"Preview a structure change against past feedback - Moonbounce Decisions API","buildId":"dpl_8LJbrAiZvr1K98WQNZXCyY5AvT34","deploymentId":"dpl_8LJbrAiZvr1K98WQNZXCyY5AvT34","dynamics":["10e4515597","c2aa737661","2b790eda92","b0f84f53ff","a9bb46feb3"],"chunks":["BannerMarkdown-BcDCEkh6.js","dist-CwjSCXIK.js","lib-DuNMoZ05.js","lib-B8HbPgcC.js","clsx-eKJ4rcii.js","html-void-elements-mCSPHezU.js","katex-YMM-FMeD.js","marked.esm-CZWythHk.js","OpenApiReferenceProvider-i2a3QZ8E.js","utils-ChLwp8m0.js","cloneDeep-izs3Hzxb.js","_Uint8Array-BVIqJ90l.js","isObjectLike-C8vmzPEN.js","isArray-Dcu1dQ5t.js","utils-DRktVTDc.js","toString-BX-iyh8G.js","isSymbol-CSH9U2Gm.js","addUniqueKeysToSchema-CA2pEs4T.js","snippetPresets-ClQ6XrKD.js","style-to-object-BY3JEAdk.js","v4-DIgz8zy6.js","ApiExamples-Ct_Lv36y.js","isEqual-BfaiBbcl.js","usePlaygroundInputsStore-Cn2PYw9e.js","shallow-rkyymjXQ.js","getApiPlaygroundDisplay-BB9vkXtE.js","CodeSnippet-Bm5RU7m1.js","CodeSnippets-BgU9Deuz.js","Page-BUDYNlhG.js","getPastedJson-BD-6NNum.js","icon-8Z6tud2e.js","MethodPill-Tuzr4g3V.js","DuplicateIcon-0ceWFYhN.js","createIcon-FBXow_ST.js","DynamicMarkdown-CD8713Yy.js","SectionHeading-7zgzQbH-.js","axios-vUo_bRtp.js","Markdown-D7Ofw1oz.js","AssistantPageLink-9EK5IAXV.js"]}