Use disagreement, annotations, and held-out examples to refine behavior.
Improve a Function
Each iteration surfaces a small number of cases for expert review to gather
additional context on how the task should be carried out.
Review three kinds of examples
Low confidence: samples that are ambiguous and require expert annotation.
High confidence: samples where there is majority consensus, but which are
worth auditing to ensure the task is well understood.
Held-out set (optional): preserves a fixed set to ensure the task
generalizes. These samples are not seen by the optimizer while learning the task.
Correct the label or extracted fields. Add a rationale when the prompt needs a
rule that is not obvious from the label alone.
If your rationale differs from the model’s, explain your decision rule directly
to guide its task understanding. Rationales are highly valuable for transferring
the nuances of your task preferences to the model.
Run the optimizer
After review, run the optimizer. Sutro abstracts your annotations, corrections, and
rationales into generalized rules for the next iteration, helping the Function
apply your feedback to new cases rather than memorize individual examples.
Complete an iteration
Use metrics to track progress
Model consensus measures task-level consistency across various models.
User/model agreement measures how often reviewed predictions match your annotations.
Optimization metrics measure how well a model can learn the task from the annotated data.
Review the prompt diff
Review the prompt diff and validation results before starting the next
iteration.Sutro automatically indicates when a Function appears to have learned the task
and is ready for deployment. From there, we recommend running a model sweep.
Select a deployment model
Use Model Sweep to compare supported models on your annotated rows. Sutro
reports quality and estimated cost, then lets you deploy the selected
prompt/model combination. This allows you to better understand cost and performance
trade-offs for your task, or choose between open-source and proprietary models.
Continue improving
You can return later with new production data and continue from the same
annotation history, or choose a new model to optimize for your task.
{"docsConfig":{"theme":"mint","$schema":"https://mintlify.com/docs.json","name":"Sutro Docs","colors":{"primary":"#2457DC","light":"#2457DC","dark":"#AAB3F3"},"logo":{"light":"https://mintcdn.com/sutro-d5dc9f29/2FChvrRfipPLBgk-/Logo/sutro-light.svg?fit=max\u0026auto=format\u0026n=2FChvrRfipPLBgk-\u0026q=85\u0026s=d97c22cc3edd64b9889c1397a07aa548","dark":"https://mintcdn.com/sutro-d5dc9f29/2FChvrRfipPLBgk-/Logo/sutro-dark.svg?fit=max\u0026auto=format\u0026n=2FChvrRfipPLBgk-\u0026q=85\u0026s=b71ebe7b47e61f8955a12bdbc478f6b0","href":"https://sutro.sh/"},"navbar":{"links":[{"href":"/reference/authentication","label":"Connect your deployment"}],"primary":{"type":"button","label":"Try Sutro","href":"https://cal.com/team/sutro/demo"}},"footer":{},"styling":{"latex":false},"navigation":{"pages":[]}},"deployment":{"deploymentMetadataValue":{"subdomain":"sutro-d5dc9f29","hasAudioNarration":false,"actualSubdomain":"sutro-d5dc9f29","customDomains":["docs.sutro.sh"],"gitSource":{"type":"github","owner":"sutro-sh","repo":"foundation","deployBranch":"main","contentDirectory":"docs","isPrivate":true,"mountPath":"","label":""},"entitlements":{},"buildId":"6aff40b95735af7f41d11b5d","clientVersion":"0.0.4227","searchFilterCounts":[{"language":"default","tag":"Reference","count":22},{"language":"default","tag":"API","count":11},{"language":"default","tag":"Python SDK","count":7},{"language":"default","tag":"Functions","count":3},{"language":"default","tag":"Product","count":3},{"language":"default","tag":"Batch","count":2},{"language":"default","tag":"Start","count":2},{"language":"default","tag":"MCP","count":1}]},"assistantConfig":{},"basePath":"","hasChatPermissions":false,"hasWebMcpPermissions":false,"toggles":[],"googleFontsInlined":false},"customCss":[]}
{"page":{"pagePath":"functions/improve","slug":"/functions/improve","pageMetadata":{"title":"Improve a Function","description":"Use disagreement, annotations, and held-out examples to refine behavior.","href":"/functions/improve"},"description":{"compiledSource":"\"use strict\";\nconst {jsx: _jsx} = arguments[0];\nconst {useMDXComponents: _provideComponents} = arguments[0];\nfunction _createMdxContent(props) {\n const _components = {\n p: \"p\",\n ..._provideComponents(),\n ...props.components\n };\n return _jsx(_components.p, {\n children: \"Use disagreement, annotations, and held-out examples to refine behavior.\"\n });\n}\nfunction MDXContent(props = {}) {\n const {wrapper: MDXLayout} = {\n ..._provideComponents(),\n ...props.components\n };\n return MDXLayout ? _jsx(MDXLayout, {\n ...props,\n children: _jsx(_createMdxContent, {\n ...props\n })\n }) : _createMdxContent(props);\n}\nreturn {\n default: MDXContent\n};\n","frontmatter":{},"scope":{}},"pageType":"default","mdx":{"mdxSource":{"compiledSource":"\"use strict\";const{Fragment:_Fragment,jsx:_jsx,jsxs:_jsxs}=arguments[0];const{useMDXComponents:_provideComponents}=arguments[0];function _createMdxContent(props){const _components={li:`li`,p:`p`,strong:`strong`,ul:`ul`,..._provideComponents(),...props.components},{Heading,Tip}=_components;if(!Heading)_missingMdxReference(`Heading`,true);if(!Tip)_missingMdxReference(`Tip`,true);return _jsxs(_Fragment,{children:[_jsx(Heading,{level:`1`,id:`improve-a-function`,children:`Improve a Function`}),`\n`,_jsx(_components.p,{children:`Each iteration surfaces a small number of cases for expert review to gather\nadditional context on how the task should be carried out.`}),`\n`,_jsx(Heading,{level:`2`,id:`review-three-kinds-of-examples`,children:`Review three kinds of examples`}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsxs(_components.li,{children:[_jsx(_components.strong,{children:`Low confidence:`}),` samples that are ambiguous and require expert annotation.`]}),`\n`,_jsxs(_components.li,{children:[_jsx(_components.strong,{children:`High confidence:`}),` samples where there is majority consensus, but which are\nworth auditing to ensure the task is well understood.`]}),`\n`,_jsxs(_components.li,{children:[_jsx(_components.strong,{children:`Held-out set (optional):`}),` preserves a fixed set to ensure the task\ngeneralizes. These samples are not seen by the optimizer while learning the task.`]}),`\n`]}),`\n`,_jsx(_components.p,{children:`Correct the label or extracted fields. Add a rationale when the prompt needs a\nrule that is not obvious from the label alone.`}),`\n`,_jsx(Tip,{children:_jsx(_components.p,{children:`If your rationale differs from the model’s, explain your decision rule directly\nto guide its task understanding. Rationales are highly valuable for transferring\nthe nuances of your task preferences to the model.`})}),`\n`,_jsx(Heading,{level:`2`,id:`run-the-optimizer`,children:`Run the optimizer`}),`\n`,_jsx(_components.p,{children:`After review, run the optimizer. Sutro abstracts your annotations, corrections, and\nrationales into generalized rules for the next iteration, helping the Function\napply your feedback to new cases rather than memorize individual examples.`}),`\n`,_jsx(Heading,{level:`2`,id:`complete-an-iteration`,children:`Complete an iteration`}),`\n`,_jsx(Heading,{level:`3`,id:`use-metrics-to-track-progress`,children:`Use metrics to track progress`}),`\n`,_jsxs(_components.p,{children:[_jsx(_components.strong,{children:`Model consensus`}),` measures task-level consistency across various models.\n`,_jsx(_components.strong,{children:`User/model agreement`}),` measures how often reviewed predictions match your annotations.\n`,_jsx(_components.strong,{children:`Optimization metrics`}),` measure how well a model can learn the task from the annotated data.`]}),`\n`,_jsx(Heading,{level:`3`,id:`review-the-prompt-diff`,children:`Review the prompt diff`}),`\n`,_jsx(_components.p,{children:`Review the prompt diff and validation results before starting the next\niteration.`}),`\n`,_jsx(_components.p,{children:`Sutro automatically indicates when a Function appears to have learned the task\nand is ready for deployment. From there, we recommend running a model sweep.`}),`\n`,_jsx(Heading,{level:`2`,id:`select-a-deployment-model`,children:`Select a deployment model`}),`\n`,_jsxs(_components.p,{children:[`Use `,_jsx(_components.strong,{children:`Model Sweep`}),` to compare supported models on your annotated rows. Sutro\nreports quality and estimated cost, then lets you deploy the selected\nprompt/model combination. This allows you to better understand cost and performance\ntrade-offs for your task, or choose between open-source and proprietary models.`]}),`\n`,_jsx(Heading,{level:`2`,id:`continue-improving`,children:`Continue improving`}),`\n`,_jsx(_components.p,{children:`You can return later with new production data and continue from the same\nannotation history, or choose a new model to optimize for your task.`})]})}function MDXContent(props={}){const{wrapper:MDXLayout}={..._provideComponents(),...props.components};return MDXLayout?_jsx(MDXLayout,{...props,children:_jsx(_createMdxContent,{...props})}):_createMdxContent(props)}return{default:MDXContent};function _missingMdxReference(id,component){throw new Error(`Expected `+(component?`component`:`object`)+\" `\"+id+\"` to be defined: you likely forgot to import, pass, or provide it.\")}","frontmatter":{},"scope":{"config":{},"pageMetadata":{"title":"Improve a Function","description":"Use disagreement, annotations, and held-out examples to refine behavior.","href":"/functions/improve"}}},"mdxSourceWithNoJs":null},"mdxExtracts":{"tableOfContents":[{"title":"Improve a Function","slug":"improve-a-function","depth":1,"children":[]},{"title":"Review three kinds of examples","slug":"review-three-kinds-of-examples","depth":2,"children":[]},{"title":"Run the optimizer","slug":"run-the-optimizer","depth":2,"children":[]},{"title":"Complete an iteration","slug":"complete-an-iteration","depth":2,"children":[{"title":"Use metrics to track progress","slug":"use-metrics-to-track-progress","depth":3,"children":[]},{"title":"Review the prompt diff","slug":"review-the-prompt-diff","depth":3,"children":[]}]},{"title":"Select a deployment model","slug":"select-a-deployment-model","depth":2,"children":[]},{"title":"Continue improving","slug":"continue-improving","depth":2,"children":[]}],"codeExamples":{}},"jsLinks":["/_mintlify/custom/sutro-d5dc9f29/226d2858a53bad9f/site-observability.js"],"subdomain":"sutro-d5dc9f29","siteName":"Sutro Docs","title":"Improve a Function - Sutro Docs","navMaps":{"firstHrefInVersion":{},"firstHrefInLanguage":{}},"lastModified":"2026-08-21T00:30:26.772Z","originalFileLocation":"functions/improve.mdx"},"headHtml":"\u003cscript data-mint-head>(function(){var d=document.documentElement,p=null;if(!p){try{p=localStorage.getItem(\"isDarkMode\")}catch(e){}if(p==='true')p='dark';else if(p==='false')p='light';if(p!=='light'\u0026\u0026p!=='dark'\u0026\u0026p!=='system')p=\"system\"}var r=p==='system'?(matchMedia(\"(prefers-color-scheme: dark)\").matches?'dark':'light'):p;d.classList.remove(r==='dark'?'light':'dark');d.classList.add(r);d.setAttribute(\"data-theme-preference\",p)})();\u003c/script>\u003cmeta data-mint-head name=\"description\" content=\"Use disagreement, annotations, and held-out examples to refine behavior.\">\u003cmeta data-mint-head name=\"application-name\" content=\"Sutro Docs\">\u003cmeta data-mint-head name=\"generator\" content=\"Mintlify\">\u003clink data-mint-head rel=\"apple-touch-icon\" type=\"image/png\" sizes=\"180x180\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/sutro-d5dc9f29/_generated/favicon/apple-touch-icon.png?v=3\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/sutro-d5dc9f29/_generated/favicon/favicon-32x32.png?v=3\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/sutro-d5dc9f29/_generated/favicon/favicon-16x16.png?v=3\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/sutro-d5dc9f29/_generated/favicon/favicon.ico?v=3\">\u003cmeta data-mint-head property=\"og:title\" content=\"Improve a Function - Sutro Docs\">\u003cmeta data-mint-head property=\"og:description\" content=\"Use disagreement, annotations, and held-out examples to refine behavior.\">\u003cmeta data-mint-head property=\"og:site_name\" content=\"Sutro Docs\">\u003cmeta data-mint-head property=\"og:url\" content=\"https://docs.sutro.sh/functions/improve\">\u003cmeta data-mint-head property=\"og:type\" content=\"website\">\u003cmeta data-mint-head property=\"og:image\" content=\"https://sutro-d5dc9f29.mintlify.site/_mintlify/api/og/v2?division=Documentation\u0026amp;title=Improve+a+Function\u0026amp;description=Use+disagreement%2C+annotations%2C+and+held-out+examples+to+refine+behavior.\u0026amp;theme=f9722fee9e8f402d13bdb22b\u0026amp;signature=75d3f990ea6d08827a5d5c03a1b3fed2bb8570e78a56872ae8283f09bfe8691a\">\u003cmeta data-mint-head property=\"og:image:width\" content=\"1200\">\u003cmeta data-mint-head property=\"og:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"twitter:card\" content=\"summary_large_image\">\u003cmeta data-mint-head name=\"twitter:title\" content=\"Improve a Function - Sutro Docs\">\u003cmeta data-mint-head name=\"twitter:description\" content=\"Use disagreement, annotations, and held-out examples to refine behavior.\">\u003cmeta data-mint-head name=\"twitter:image\" content=\"https://sutro-d5dc9f29.mintlify.site/_mintlify/api/og/v2?division=Documentation\u0026amp;title=Improve+a+Function\u0026amp;description=Use+disagreement%2C+annotations%2C+and+held-out+examples+to+refine+behavior.\u0026amp;theme=f9722fee9e8f402d13bdb22b\u0026amp;signature=75d3f990ea6d08827a5d5c03a1b3fed2bb8570e78a56872ae8283f09bfe8691a\">\u003cmeta data-mint-head name=\"twitter:image:width\" content=\"1200\">\u003cmeta data-mint-head name=\"twitter:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"msapplication-config\" content=\"https://mintlify.s3.us-west-1.amazonaws.com/sutro-d5dc9f29/_generated/favicon/browserconfig.xml?v=3\">\u003cmeta data-mint-head name=\"apple-mobile-web-app-title\" content=\"Sutro Docs\">\u003cmeta data-mint-head name=\"msapplication-TileColor\" content=\"#2457DC\">\u003clink data-mint-head rel=\"canonical\" href=\"https://docs.sutro.sh/functions/improve\">\u003clink data-mint-head rel=\"alternate\" type=\"application/xml\" href=\"/sitemap.xml\">\u003clink data-mint-head rel=\"alternate\" type=\"text/markdown\" href=\"/functions/improve.md\">\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@graph\":[{\"@type\":\"Organization\",\"@id\":\"https://docs.sutro.sh/#organization\",\"name\":\"Sutro Docs\",\"url\":\"https://docs.sutro.sh\",\"logo\":{\"@type\":\"ImageObject\",\"url\":\"https://mintcdn.com/sutro-d5dc9f29/2FChvrRfipPLBgk-/Logo/sutro-light.svg?fit=max\u0026auto=format\u0026n=2FChvrRfipPLBgk-\u0026q=85\u0026s=d97c22cc3edd64b9889c1397a07aa548\"}},{\"@type\":\"WebSite\",\"@id\":\"https://docs.sutro.sh#website\",\"name\":\"Sutro Docs\",\"url\":\"https://docs.sutro.sh\",\"publisher\":{\"@id\":\"https://docs.sutro.sh/#organization\"}},{\"@type\":\"WebPage\",\"@id\":\"https://docs.sutro.sh/functions/improve#webpage\",\"url\":\"https://docs.sutro.sh/functions/improve\",\"name\":\"Improve a Function\",\"description\":\"Use disagreement, annotations, and held-out examples to refine behavior.\",\"dateModified\":\"2026-08-21T00:30:26.772Z\",\"isPartOf\":{\"@id\":\"https://docs.sutro.sh#website\"}},{\"@type\":[\"Article\",\"TechArticle\"],\"@id\":\"https://docs.sutro.sh/functions/improve#article\",\"headline\":\"Improve a Function\",\"name\":\"Improve a Function\",\"description\":\"Use disagreement, annotations, and held-out examples to refine behavior.\",\"url\":\"https://docs.sutro.sh/functions/improve\",\"mainEntityOfPage\":{\"@id\":\"https://docs.sutro.sh/functions/improve#webpage\"},\"dateModified\":\"2026-08-21T00:30:26.772Z\",\"publisher\":{\"@id\":\"https://docs.sutro.sh/#organization\"},\"isPartOf\":{\"@id\":\"https://docs.sutro.sh#website\"}}]}\u003c/script>\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@type\":\"WebSite\",\"name\":\"Sutro Docs\",\"publisher\":{\"@type\":\"Organization\",\"name\":\"Sutro Docs\"},\"provider\":{\"@type\":\"Organization\",\"name\":\"Mintlify\",\"url\":\"https://mintlify.com\"}}\u003c/script>","agentPointerHref":"/functions/improve.md","siteName":"Sutro Docs","nav":{"global":null,"anchors":[{"anchor":"Start","icon":"house","pages":[{"title":"Start","href":""}]},{"anchor":"Functions","icon":"function","pages":[{"title":"Design a Function","description":"Choose a narrow decision, output shape, and representative dataset.","href":"/functions/build"},{"title":"Improve a Function","href":"/functions/improve"},{"title":"Run a Function","description":"Call a published Function in real time, or submit rows to Batch.","href":"/functions/run"}]},{"anchor":"Batch","icon":"layer-group","pages":[{"title":"Batch","href":"/batch/run-model"}]},{"anchor":"Reference","icon":"code","pages":[{"title":"Reference","href":"/reference/current-capabilities"}]}]},"basePath":"","title":"Improve a Function - Sutro Docs","deploymentId":"dpl_vSEmv961Ncyvz1psVwgUnTXQMSYk","buildId":"dpl_vSEmv961Ncyvz1psVwgUnTXQMSYk","dynamics":[],"chunks":[]}