From 16071ea7f652510bdfc31ef620742f906847ac99 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sat, 17 Oct 2026 18:11:11 -0400 Subject: [PATCH] content: reader-first openings, plain words, useful endings across live posts Signed-off-by: abrichr --- .../openadapt-quickstart-break-it/index.md | 17 +++- content/posts/openadapt-vs-api/index.md | 49 +++++------- .../posts/openadapt-vs-autohotkey/index.md | 32 ++++---- .../posts/openadapt-vs-playwright/index.md | 33 ++++---- .../openadapt-vs-power-automate/index.md | 18 ++--- content/posts/openadapt-vs-selenium/index.md | 34 ++++---- content/posts/openadapt-vs-uipath/index.md | 36 ++++----- content/posts/openemr-benchmark/index.md | 8 +- .../record-each-surface-then-compose/index.md | 16 ++-- content/posts/silent-wrong-action/index.md | 6 +- content/posts/the-500th-run/index.md | 12 +-- content/posts/the-write-audit/index.md | 8 +- scripts/benchmark_claims/registry.json | 79 ++++--------------- 13 files changed, 152 insertions(+), 196 deletions(-) diff --git a/content/posts/openadapt-quickstart-break-it/index.md b/content/posts/openadapt-quickstart-break-it/index.md index ded8bc3..c6831ab 100644 --- a/content/posts/openadapt-quickstart-break-it/index.md +++ b/content/posts/openadapt-quickstart-break-it/index.md @@ -1,17 +1,19 @@ --- -title: "See a bad save get caught: openadapt quickstart --break-it" +title: "Watch an automation report a save that never happened" date: 2026-08-29 lastmod: 2026-10-09 draft: false author: "Richard Abrich" tags: ["openadapt-flow", "quickstart", "safety", "validation", "automation"] -description: "Run openadapt quickstart --break-it to watch a fake clinic app say a note saved after its server rejected it, and see OpenAdapt read the record and stop." +description: "A few-minute demo on your own computer: a fake clinic app shows \"Encounter saved\" after its server rejected the note, and a record check catches it." thesis: "openadapt quickstart --break-it shows, on your own computer, an app that reports a save its server rejected, and OpenAdapt reading the record and stopping." audience: "developer" post_type: "note" --- -Run `openadapt quickstart --break-it` and you'll watch a fake clinic app with synthetic patients report a save that its server rejected. OpenAdapt reads the record through a separate interface and finds no saved note, so it stops at the Save step and asks a person to check the record. The demo takes a few minutes and keeps its files on your computer. +Most automation decides a save worked by looking at the screen. If the app says "Encounter saved," the run is marked done. That holds until the app is wrong about its own save, for example when the server rejects the write after the page has already shown its success banner. Every check that reads the screen passes, and the record is empty. + +You can watch that happen on your own computer in a few minutes. OpenAdapt comes with MockMed, a fake clinic app with synthetic patients, and a demo that replays one recorded task twice. The first time, the app behaves. The second time, its server quietly rejects the note. Both runs end on the same "Encounter saved" screen, and only one of them saved anything. ## Run it @@ -25,6 +27,13 @@ openadapt quickstart --break-it # the clean run, then the broken one On Python 3.13 or newer, pip installs an old release that has no `quickstart` command. If you use [uv](https://docs.astral.sh/uv/), run `uv tool install --python 3.12 openadapt` instead, then run the same commands. uv downloads Python 3.12 if you don't have it. If you installed `openadapt-flow` on its own, `openadapt-flow tutorial --break-it` runs the same loop, since `openadapt quickstart` delegates to it. +If you'd rather see the result as one page, the Flow package has a demo that runs a similar pair of runs and then opens a page with the two results side by side. In its broken run, the app shows "Encounter saved" and drops the note. The first run can take a few minutes while Playwright downloads Chromium. + +```bash +pip install 'openadapt-flow[browser]' +openadapt-flow demo +``` + ## What the screen said and what the record held Both runs replay one recorded task on MockMed, a fake clinic app that ships with OpenAdapt. The task opens a patient, starts a triage encounter, types a note, and clicks Save Encounter. In the second run, the server rejects the save after the app has already shown its success banner. @@ -36,7 +45,7 @@ This is what each run produced on 2026-10-09 with openadapt 1.16.0 (synthetic te | What the screen showed | "Encounter saved", and the note in the encounter list | The same screen | | What the record held | 1 saved encounter | 0 matching records | | What OpenAdapt reported | Done and checked | Stopped at Save Encounter for a person to check the record | -| Shareable receipt | Yes | No, only a local report | +| Report you can share | Yes | No, only a local report | These lines come from the broken run's `run-broken/REPORT.md`: diff --git a/content/posts/openadapt-vs-api/index.md b/content/posts/openadapt-vs-api/index.md index 5c69571..6b45183 100644 --- a/content/posts/openadapt-vs-api/index.md +++ b/content/posts/openadapt-vs-api/index.md @@ -5,9 +5,10 @@ lastmod: 2026-10-09 draft: false author: "Richard Abrich" # openadapt-vs-autohotkey, openadapt-vs-playwright, and openadapt-vs-selenium -# were merged into this post on 2026-10-09. Their old URLs land here. +# were merged into this post on 2026-10-09 and published again as standalone +# posts on 2026-10-16. This post links to them and carries no alias for them. tags: ["comparison", "api", "playwright", "selenium", "autohotkey", "gui-automation", "integration"] -description: "Use a supported API whenever it can make the change. Record an OpenAdapt workflow only for the screen-only step, and use the API to check what it saved." +description: "The vendor's API covers most of the task, but the last step only exists on screen. Keep the API for what it reaches. Here's how to handle that step." faq: - q: "Should I use an API or screen automation?" a: "Use the API whenever a supported one can make the change. Automate the screen only for the step no supported API can reach, and keep every other step on the API." @@ -22,7 +23,9 @@ audience: "practitioner" post_type: "essay" --- -If a supported API can make the whole change, use the API, not OpenAdapt. Use OpenAdapt for the step that's left over, the one that only happens on screen, when other people depend on it and you need proof that each save landed in the record. Most real tasks end up using both: the API for lookups and supported writes, and a recorded workflow for the last screen-only step. +The vendor's API handles lookups and most of the updates. The last step, the one that actually changes the record, only exists as a button in its Windows client. If that's your task, you can automate everything except one click, and the question is what to do about it. + +Use the API for everything it can reach, and automate the screen only for that leftover step. That's a much smaller job than replacing the integration, and it's the job OpenAdapt is built for. The API can still help there. It can't make the change, but it can often read the result, which gives you a way to check that the save on screen really landed. ## How the options compare @@ -40,49 +43,33 @@ Tool behavior in this post comes from each project's documentation, checked on 2 When a supported API exists, it's the better choice almost every time. A direct call skips the fragile parts of screen automation, such as window focus and finding the right field on a page that moved. You can test a request more easily than a click, and the service owner can redesign the screen without breaking your integration. OpenAdapt's [fit checks for a new workflow](https://openadapt.ai/qualify) start with the same rule: if a supported API exposes the operation, use it. -An API call still needs a check when the write matters. [RFC 9110](https://www.rfc-editor.org/rfc/rfc9110.html#name-status-codes) defines a 2xx status code to mean the server received and accepted the request. The server can return 200 after writing to the wrong record, and a later step in the vendor's system can still reject work that the first service accepted. +An API call still needs a check when the write matters. [RFC 9110](https://www.rfc-editor.org/rfc/rfc9110.html#name-status-codes) defines a 2xx status code to mean the server received and accepted the request. The server can return 200 after writing to the wrong record. A later step in the vendor's system can also reject work that the first service accepted. ## Move only the step the API can't reach An API often covers most of a task and stops short of one step. A product can expose customer lookup through an API and keep the final update in its Windows client. A payer portal may have an internal API that customers aren't allowed to call. An old system may support only the desktop app its vendor ships, and Citrix can put that app behind one more boundary your team doesn't control. -Keep API calls for search, data preparation, routing, and every supported write. Automate the screen only for the operation that has no supported machine interface. Often that's the last step, where someone enters approved information into the system and checks that it saved. For example, a task with 9 API steps and 1 screen step is easier to run than a task with 10 screen steps, and a layout change can't break an API call that never opens the page. +Keep API calls for search, data preparation, routing, and every supported write. Automate the screen only for the operation that has no supported machine interface. Often that's the last step, where someone enters approved information and checks that it saved. A task with 9 API steps and 1 screen step is easier to run than one with 10 screen steps, because a layout change can't break an API call that never opens the page. ## Let a read-only API check what the screen wrote -An API that can't make the change can often still read the result. OpenAdapt keeps the two jobs apart. The browser or desktop session clicks Save, and a different interface reads the stored record. OpenAdapt's [limits page](https://github.com/OpenAdaptAI/openadapt-flow/blob/main/docs/LIMITS.md) describes record checks through a REST or FHIR API or a document store. - -OpenAdapt doesn't work out which record to read from the recording itself. A person names the record and the fields to check for each workflow. Without that separate read, OpenAdapt can compare screens but can't report a run as done and checked. A success banner can sit on top of a save that the server rejected, duplicated, or only partly wrote. - -A timeout after Save is the hardest case. If the app accepted the click and the reply never arrived, a second click could save the entry twice. Instead of clicking again, OpenAdapt stops and asks a person to check the record, and nothing is retried until they do. It stops even when the save went through, because nothing confirmed it. A read-only API gives that person a quick way to see what the record holds. In the engine's output, a confirmed save is `VERIFIED` and this stop is `RECONCILIATION_REQUIRED`. - -## Keep a script while its owner watches it - -### Playwright - -OpenAdapt's [browser recorder](https://github.com/OpenAdaptAI/openadapt-flow/blob/main/docs/BROWSER_RECORDING.md) runs on a Playwright page. For each action, it keeps the target's DOM identity, such as its ID or test ID, its role, and its accessible name, along with its position on screen. When the automation runs, it compares that identity with the live page before it acts. Playwright's [locator guide](https://playwright.dev/docs/locators) prefers the same kind of identity to CSS and XPath selectors, which break when the page structure changes. The quickstart at the end of this post records and replays its task on a Playwright page. - -Two more Playwright features are good reasons to keep it in the browser path. Before a click, its [actionability checks](https://playwright.dev/docs/actionability) wait until the locator matches exactly 1 element that's visible, stable, enabled, and not covered by another element. The [trace viewer](https://playwright.dev/docs/trace-viewer) lets you step through a failed run after it ends. It shows the page before and after each action, the line of code that ran, and the network requests. - -If a developer owns a browser-only task and the script's assertions prove the whole result, keep the Playwright script. You still write the business rules yourself. Your code decides which record must be open before a write, what to do after a timeout that follows Submit, and what proves the write happened once. Keep the script until those rules cost more to write and maintain than the script itself. - -If the task leaves the browser for a desktop app, record each part as its own OpenAdapt automation. The browser part still runs on Playwright, and the desktop part starts only with a value the browser part saved and checked. [Two apps, one checked fact](/posts/record-each-surface-then-compose/) shows how that handoff works. +OpenAdapt keeps the two jobs apart. The browser or desktop session clicks Save, and a different interface reads the stored record. OpenAdapt's [limits page](https://github.com/OpenAdaptAI/openadapt-flow/blob/main/docs/LIMITS.md) describes record checks through a REST or FHIR API or a document store. -### Selenium +OpenAdapt doesn't guess which record to read. A person names the record and the fields to check for each workflow. Without that separate read, OpenAdapt can compare screens but won't report a run as done and checked, because a success banner can sit on top of a save that the server rejected, duplicated, or only partly wrote. -Selenium WebDriver is a [W3C Recommendation](https://www.selenium.dev/documentation/webdriver/). [Selenium Grid](https://www.selenium.dev/documentation/grid/) routes its commands to browsers on other machines, so you can run the same script across browser versions and operating systems. +A timeout after Save is the hardest case. If the app accepted the click and the reply never arrived, a second click could save the entry twice. So OpenAdapt doesn't click again. It stops and asks a person to check the record, and nothing is retried until they do. It stops even when the save went through, because nothing confirmed it. A read-only API gives that person a quick way to see what the record holds. -Selenium leaves everything around the browser to your code. That's fine in a test suite, where a failed test stops with a stack trace and waits for a developer. A job that writes business records also needs a check that the right record is open before the click, a rule for timeouts, a second read that confirms the save, and a person who handles the runs that stop. If your team already built that code and keeps it running, keep the fleet. +## If you already have a script -A timeout after a click that saves something has 2 possible meanings. The server may never have received the click, or it may have saved the change before the reply was lost. A retry fixes the first case and creates a duplicate in the second. The browser error can't tell you which case you're in, so your code has to read the record before it tries again. +A script is a fine answer while the person who wrote it runs it and watches it. Each of these tools has its own post on where that stops being true. -### AutoHotkey +Playwright is the one OpenAdapt builds on. Its [browser recorder](https://github.com/OpenAdaptAI/openadapt-flow/blob/main/docs/BROWSER_RECORDING.md) runs on a Playwright page and keeps the same kind of target identity, role and accessible name, that Playwright's [locator guide](https://playwright.dev/docs/locators) recommends. Keep a Playwright script while a developer owns it and its assertions prove the whole result ([OpenAdapt vs. Playwright](/posts/openadapt-vs-playwright/)). -AutoHotkey's [Send](https://www.autohotkey.com/docs/v2/lib/Send.htm) function sends simulated keys and mouse clicks to whichever window is active, so focus decides where the input lands. A careful author has better tools. [WinWait](https://www.autohotkey.com/docs/v2/lib/WinWait.htm) pauses the script until the expected window exists. [ControlClick](https://www.autohotkey.com/docs/v2/lib/ControlClick.htm) and [ControlSend](https://www.autohotkey.com/docs/v2/lib/ControlSend.htm) target a specific control, and they throw an error when they can't find the window or control. With logging and error handling added, a script like that can run for years. Keeping it correct is work for whoever maintains it. +Selenium leaves everything around the browser to your code, which is fine in a test suite. A job that writes business records also needs a rule for the click on Save that times out, since the browser error can't tell you whether the save landed ([OpenAdapt vs. Selenium](/posts/openadapt-vs-selenium/)). -A personal macro that its author runs and watches is fine. When the wrong window has focus, the person who can fix it sees the failure. +AutoHotkey's [Send](https://www.autohotkey.com/docs/v2/lib/Send.htm) goes to whichever window is active, so focus decides where the input lands. A careful author can use [WinWait](https://www.autohotkey.com/docs/v2/lib/WinWait.htm) and [ControlSend](https://www.autohotkey.com/docs/v2/lib/ControlSend.htm) to make a macro sturdy. The question is who's watching once coworkers rely on it ([OpenAdapt vs. AutoHotkey](/posts/openadapt-vs-autohotkey/)). -The job changes when coworkers copy the macro, a scheduler runs it after hours, or it starts writing records that are costly to fix. The author may not be there when it fails, and Windows accepts the keystrokes whether or not the business result is right. A recorded workflow has a named owner who reviews each change, and every run uses that reviewed version. Where a step has a right-record check, OpenAdapt confirms that the right record is open before it types, and it stops on a mismatch. +If the task leaves the browser for a desktop app, record each part as its own automation. [Two apps, one checked fact](/posts/record-each-surface-then-compose/) shows how the second part starts only with a value the first part saved and checked. ## When to record a workflow instead @@ -106,7 +93,7 @@ openadapt quickstart On Python 3.13 or newer, pip installs an old release that has no `quickstart` command. If you use [uv](https://docs.astral.sh/uv/), run `uv tool install --python 3.12 openadapt` instead, then run the same command. -The quickstart replays a recorded task in MockMed, a fake clinic app with synthetic patients that ships with OpenAdapt. It saves a record through the app's screens, then confirms the saved value through a read-only API that the app itself never calls. MockMed shows what the check looks like. Your application needs its own second way in, and the quickstart can't tell you whether it has one. +The quickstart replays a recorded task in MockMed, a fake clinic app with synthetic patients that ships with OpenAdapt. It saves a record through the app's screens, then confirms the saved value through a read-only API that the app itself never calls. Your application needs its own second way in, and the quickstart can't tell you whether it has one. To see the check catch a bad save, run `openadapt quickstart --break-it` ([what it shows](/posts/openadapt-quickstart-break-it/)). To record a task in your own web app, follow the [first-workflow guide](https://docs.openadapt.ai/get-started/first-workflow/). If you're comparing RPA suites, read the posts on [UiPath](/posts/openadapt-vs-uipath/) and [Power Automate](/posts/openadapt-vs-power-automate/). diff --git a/content/posts/openadapt-vs-autohotkey/index.md b/content/posts/openadapt-vs-autohotkey/index.md index 84d33e3..fc09314 100644 --- a/content/posts/openadapt-vs-autohotkey/index.md +++ b/content/posts/openadapt-vs-autohotkey/index.md @@ -2,16 +2,14 @@ title: "OpenAdapt vs. AutoHotkey: when to switch from a macro" date: 2026-08-26 lastmod: 2026-10-09 -# Merged on 2026-10-09 into content/posts/openadapt-vs-api/index.md ("API, -# script, or recorded workflow: how to choose"). That post carries this post's -# AutoHotkey row in its comparison table and the AutoHotkey facts below. The -# old URL /posts/openadapt-vs-autohotkey/ redirects there through that post's -# aliases. Rewritten 2026-10-16 as a standalone comparison page but left as a -# draft: publishing it needs that alias removed first. This post has no entries in scripts/benchmark_claims/registry.json. +# Merged on 2026-10-09 into content/posts/openadapt-vs-api/index.md, then +# rewritten 2026-10-16 as a standalone comparison page and published again. +# openadapt-vs-api no longer carries an alias for this URL. This post has no +# entries in scripts/benchmark_claims/registry.json. draft: false author: "Richard Abrich" tags: ["comparison", "autohotkey", "windows", "gui-automation"] -description: "Keep an AutoHotkey macro while its author runs and watches it. Switch to OpenAdapt once others depend on it or a wrong save costs real money." +description: "You wrote an AutoHotkey macro for yourself, and now coworkers run it too. How to tell when it needs more than a script, and what to keep in AutoHotkey." faq: - q: "Is AutoHotkey reliable enough for business data entry?" a: "It can be. WinWait, ControlClick, and ControlSend let a script wait for the right window and target specific controls, and both Control functions throw an error when they can't find their target. The reliability comes from how carefully someone writes and maintains the script." @@ -26,7 +24,9 @@ audience: "practitioner" post_type: "comparison" --- -If you wrote the AutoHotkey macro and you watch it run, keep using AutoHotkey. Switch the task to OpenAdapt when other people depend on it, it runs without you, or a wrong save is expensive to fix, because then you need a reviewed version, a check that the right record is open, and a read of the saved record. AutoHotkey can do a lot of that too, but you'd be writing and maintaining it yourself. +Maybe you wrote an AutoHotkey macro to skip some typing in a Windows app, and it worked. Then a coworker asked for a copy, and another set it to run from a scheduler at night. Now when it types into the wrong window, the person who notices may not be you. Nobody may notice at all, because Windows accepts the keystrokes either way. + +While you're the one running and watching it, AutoHotkey is the right tool. It can also be made sturdy enough to share, if someone keeps it that way. OpenAdapt is for after the handoff, when the task writes records other people rely on. Every run uses a version that a named owner has reviewed, and a save isn't counted as done until OpenAdapt has read it back from the record. ## OpenAdapt vs. AutoHotkey at a glance @@ -40,20 +40,22 @@ If you wrote the AutoHotkey macro and you watch it run, keep using AutoHotkey. S ## When AutoHotkey is the better choice -For hotkeys, text expansion, and personal macros, AutoHotkey is the better tool, and OpenAdapt doesn't try to do those jobs. It's also a fine choice for a shared script when someone careful maintains it. With WinWait, the Control functions, logging, and error handling, a script can run for years. The cost is the time it takes to design its checks and keep them correct as the application changes. - -A personal macro that its author runs and watches is fine. When the wrong window has focus, the person who can fix it sees the failure. +For hotkeys and text expansion, AutoHotkey is the better tool, and OpenAdapt doesn't try to do those jobs. It's also a fine choice for a shared script when someone careful maintains it. With WinWait and the Control functions, plus some logging and error handling, a script can run for years. What it costs is someone's time to design those checks and keep them correct as the application changes. -## What changes when a macro is shared +A macro you run and watch yourself doesn't need any of that. When the wrong window has focus, the person who can fix it is right there. -The job changes when coworkers copy the macro, a scheduler runs it after hours, or it starts writing records that are costly to fix. The author may not be there when it fails. Windows accepts the keystrokes whether or not the business result is right. +## What OpenAdapt adds once the macro is shared -OpenAdapt is built for that handoff. A person or an agent shows the task while OpenAdapt records it, and OpenAdapt builds the automation from one or more recordings. The workflow has a named owner who reviews each change, and every run uses that reviewed version. When a screen changes, OpenAdapt finds the same field again from what it recorded and logs the change for review, or it stops. Where a step has a right-record check, OpenAdapt confirms that the right record is open before it types, and it stops on a mismatch. +A person or an agent shows the task while OpenAdapt records it, and OpenAdapt builds the automation from one or more recordings. The workflow has a named owner who reviews each change, and every run uses that reviewed version. When a screen changes, OpenAdapt finds the same field again from what it recorded and logs the change for review, or it stops. -To confirm a save, OpenAdapt needs a second way to read the saved record, such as an API or a database view. Without one, it can compare screens, but it can't report a run as done and checked. The [limits page](https://github.com/OpenAdaptAI/openadapt-flow/blob/main/docs/LIMITS.md) describes what each check covers. +Where a step has a right-record check, OpenAdapt confirms that the right record is open before it types and stops on a mismatch. To confirm a save, it needs a second way to read the saved record, such as an API or a database view. Without one, it can compare screens, but it won't report a run as done and checked. The [limits page](https://github.com/OpenAdaptAI/openadapt-flow/blob/main/docs/LIMITS.md) describes what each check covers. If a supported API can make the change, use the API instead of either tool. [When to automate the screen instead of an API](/posts/openadapt-vs-api/) compares the options in one table. +## If you're keeping the macro for now + +Put a WinWait for the exact window title before each Send, or switch the riskiest keystrokes to ControlSend so they go to a specific control. Then have the macro log which record it saved and what it typed. The next person who runs it can check its work, and you'll know sooner if it went somewhere it shouldn't. + ## FAQ ### Is AutoHotkey reliable enough for business data entry? diff --git a/content/posts/openadapt-vs-playwright/index.md b/content/posts/openadapt-vs-playwright/index.md index b82822f..63db81e 100644 --- a/content/posts/openadapt-vs-playwright/index.md +++ b/content/posts/openadapt-vs-playwright/index.md @@ -2,19 +2,14 @@ title: "OpenAdapt vs. Playwright: which for browser automation?" date: 2026-08-26 lastmod: 2026-10-09 -# Merged 2026-10-09 into content/posts/openadapt-vs-api/index.md ("API, script, -# or recorded workflow: how to choose"). That post carries this post's row in -# its comparison table and its Playwright facts: the recorder runs on -# Playwright, the locator guidance, actionability checks, the trace viewer, and -# the rule to keep a developer-owned script. Its alias keeps the URL -# /posts/openadapt-vs-playwright/ working. Keep this file a draft: publishing it -# would collide with the alias. Rewritten 2026-10-16 as a standalone comparison -# page; still a draft for that reason. No registry entries to move. The original text -# is in git history. See .overhaul/REWRITE_PLAN.json. +# Merged 2026-10-09 into content/posts/openadapt-vs-api/index.md, then +# rewritten 2026-10-16 as a standalone comparison page and published again. +# openadapt-vs-api no longer carries an alias for this URL. No registry +# entries. The original text is in git history. draft: false author: "Richard Abrich" tags: ["comparison", "playwright", "browser-automation", "gui-automation"] -description: "Keep Playwright for a browser script a developer owns. Use OpenAdapt when others depend on the task, it leaves the browser, or each save needs checking." +description: "Your Playwright script works. When other people start relying on it, or it has to prove each save landed, here's where OpenAdapt fits and where it doesn't." faq: - q: "Is OpenAdapt built on Playwright?" a: "OpenAdapt's browser recorder runs on a Playwright page. It records each target's DOM identity and screen position, and a run compares that identity with the live page before it acts." @@ -29,7 +24,9 @@ audience: "practitioner" post_type: "note" --- -If a developer owns a browser-only task and the script's assertions prove the whole result, use Playwright. Use OpenAdapt when people other than that developer depend on the task, when it leaves the browser for a desktop app, or when each save needs to be read back from the record. You don't fully give up Playwright either way: OpenAdapt's browser recorder runs on it. +Take a Playwright script that submits a weekly batch of claims through a payer portal. It's reliable, partly because Playwright does a lot for you. Locators find buttons by role and name, and every click waits until its element is ready. Then the operations team asks to run it on their own schedule, and they want to know whether every claim really saved. + +Until that request, keep the script. Playwright is the better tool for a browser task a developer owns, and OpenAdapt's own browser recorder runs on it. What changes after the handoff is who maintains the task and what proves each save landed. Those two questions decide whether OpenAdapt is worth adding. ## OpenAdapt vs. Playwright at a glance @@ -44,17 +41,21 @@ If a developer owns a browser-only task and the script's assertions prove the wh ## When Playwright is the better choice -For end-to-end tests and for browser automation a developer maintains, Playwright is the better tool. Its locator guide prefers user-facing attributes such as role and accessible name over CSS and XPath, which break when page structure changes. Its auto-waiting removes most timing flakiness, and the trace viewer makes failures fast to diagnose. If your team writes code and the task stays in the browser, keep the script. +For end-to-end tests and for browser automation a developer maintains, Playwright is the better tool. Its locator guide prefers user-facing attributes such as role and accessible name over CSS and XPath, which break when page structure changes. Auto-waiting removes most timing flakiness. When a run does fail, the trace viewer shows you why. If your team writes code and the task stays in the browser, keep the script. -You still write the business rules yourself. Your code decides which record must be open before a write, what to do after a timeout that follows Submit, and what proves the write happened once. Keep the script until those rules cost more to write and maintain than the script itself. +You'll still write the business rules yourself. Your code decides which record must be open before a write. It decides what to do when Submit times out, and what proves the write happened exactly once. As long as writing and maintaining those rules costs less than the script saves, there's no reason to switch. -## Where OpenAdapt fits +## What OpenAdapt adds when others take over OpenAdapt's [browser recorder](https://github.com/OpenAdaptAI/openadapt-flow/blob/main/docs/BROWSER_RECORDING.md) runs on a Playwright page. For each action, it keeps the target's DOM identity, such as its ID or test ID, its role, and its accessible name, along with its position on screen. When the automation runs, it compares that identity with the live page before it acts. -In OpenAdapt, a person sets up a separate read of the saved record for each workflow. A run counts as done and checked only after that read confirms the save, and it doesn't retry on its own after a timeout. If the task leaves the browser for a desktop app, record each part as its own automation. [Two apps, one checked fact](/posts/record-each-surface-then-compose/) shows how that handoff works. +The difference is in what happens around the click. A person sets up a separate read of the saved record for each workflow, and a run isn't done and checked until that read confirms the save. After a timeout, it doesn't retry. It stops and asks a person to check the record. The operations team doesn't have to read code to see which runs need a look. + +If the task leaves the browser for a desktop app, record each part as its own automation. [Two apps, one checked fact](/posts/record-each-surface-then-compose/) shows how that handoff works. And if a supported API can make the change, use the API instead of either tool ([how to choose](/posts/openadapt-vs-api/)). + +## Before you hand the script over -If a supported API can make the change, use it instead. [When to automate the screen instead of an API](/posts/openadapt-vs-api/) puts Playwright in one table with the other options. +Write down what the script reads to decide that a claim saved. If the answer is a message on the page, that's the gap the new owners inherit. If an API can read the record, you can close the gap in Playwright by adding that read to the script. Or record the task in OpenAdapt and set up the read there. ## FAQ diff --git a/content/posts/openadapt-vs-power-automate/index.md b/content/posts/openadapt-vs-power-automate/index.md index b5c84d8..d83f698 100644 --- a/content/posts/openadapt-vs-power-automate/index.md +++ b/content/posts/openadapt-vs-power-automate/index.md @@ -10,7 +10,7 @@ author: "Richard Abrich" # show the Premium price, so that figure is from the 2026-10-09 check. Check them # on the day this post is promoted. Exemptions are in the benchmark registry. tags: ["comparison", "power-automate", "rpa", "gui-automation"] -description: "Use Power Automate wherever its connectors reach. Use OpenAdapt for the last screen-only entry, where each save must be read back from the record." +description: "Power Automate moves the referral right up to the EMR form, then a person types it in. What OpenAdapt adds at that last step, and when to skip it." faq: - q: "Can Power Automate automate a desktop app with no connector?" a: "Yes. Power Automate for desktop records mouse and keyboard activity against the app's UI elements and turns it into desktop and browser actions, using UI Automation or MSAA selectors." @@ -25,14 +25,16 @@ audience: "business" post_type: "comparison" --- -Use Power Automate for every step its connectors reach, and for desktop flows where a person watching the screen is enough of a check. Use OpenAdapt for the step connectors can't reach, usually the last one, where someone types approved information into an EMR form or a payer portal and you need the saved record read back. If your team already runs Power Automate, keep it and hand OpenAdapt only that step. +Say your clinic runs on Microsoft 365. A cloud flow files each faxed referral in SharePoint and asks for approval in Teams. Then the work reaches the EMR, which has no connector for creating a referral, so a staff member types it in and looks at the screen to see that it saved. + +Keep Power Automate for everything up to that form. It can handle the form too, with a desktop flow, if a person glancing at each result is enough of a check. OpenAdapt is for the case where it isn't. After it types the referral, it reads the saved record back from the EMR by a separate route and stops for a person if the two don't match. ## OpenAdapt vs. Power Automate at a glance | | Power Automate | OpenAdapt | |---|---|---| | Best at | Moving data between Microsoft 365 and other services through connectors | One screen-only entry step whose save needs checking | -| Screen automation | Desktop flows built with a [recorder](https://learn.microsoft.com/en-us/power-automate/desktop-flows/recording-flow) that captures UI Automation and MSAA selectors | A recording compiled into steps that replay without model calls | +| Screen automation | Desktop flows built with a [recorder](https://learn.microsoft.com/en-us/power-automate/desktop-flows/recording-flow) that captures UI Automation and MSAA selectors | Replays a recording of the task without calling an AI model | | Checking the save | A step the flow's author adds, if the system offers an API or query | A separate read of the saved record, set up for each workflow | | After a possible double save | Up to the flow's author | Stops with "check the record" and doesn't retry | | Unattended price | Process, $150 per bot a month, paid yearly ([pricing](https://www.microsoft.com/en-us/power-platform/products/power-automate/pricing)) | See the [comparison page](https://openadapt.ai/compare/power-automate) | @@ -57,7 +59,7 @@ Our [Power Automate comparison](https://openadapt.ai/compare/power-automate) set Some changes exist only in a form on screen, because the connector can't write that record or there's no API for the change. A desktop flow can still fill in the form and click Save. -The flow's author can then add a step that confirms the save by reading the record through an API or a database query, if the system offers one. In [our fault test](/posts/silent-wrong-action/) on an app we built for testing, a check that trusted only the success message passed most of the saves that left the record wrong. +What it can't do on its own is know the save worked. The flow's author can add a step that reads the record through an API or a database query, if the system offers one. Without that step, the flow is trusting the screen. In [our fault test](/posts/silent-wrong-action/) on an app we built for testing, a check that trusted only the success message passed most of the saves that left the record wrong. ## What OpenAdapt adds at that step @@ -77,9 +79,7 @@ If the screen changes, OpenAdapt finds the same field again or stops. Each run e | Check the record | The save may have gone through, for example if the app froze after Save. OpenAdapt won't send it again. A person checks the record before anyone retries. | | Didn't finish | Its report says what's known about whether anything was written. | -## One referral, before and after - -In this illustration, a clinic on Microsoft 365 gets referrals by fax, and its EMR has no connector for creating them. +## The same referral, step by step | Step | Today | With OpenAdapt | |---|---|---| @@ -96,9 +96,9 @@ OpenAdapt runs on a computer you control, or one we host for browser workflows. ## When Power Automate is the better choice -Pick Power Automate alone when a connector or API can make the change, because nothing on a screen can break it. It's also the better choice when your work lives in Outlook, Teams, SharePoint, and Dataverse and your administrators already govern flows there. If a desktop flow already enters the data and a person reviews each result, adding OpenAdapt buys you little. +Pick Power Automate alone when a connector or API can make the change, because nothing on a screen can break it. It's also the better choice when your work lives in Microsoft 365 and your administrators already govern flows there. If a desktop flow already enters the data and a person reviews each result, adding OpenAdapt buys you little. -For each step, ask first whether a connector or an API can make the change, and use it if one can. If the change only happens on screen and a wrong save would land in a patient's chart or a claim, give that step to OpenAdapt and keep the rest in Power Automate. +To decide, go through the flow one step at a time and ask whether a connector or an API can make that change. Where one can, use it. Where the change only happens on screen and a wrong save would land in a patient's chart or a claim, that's the step to give OpenAdapt. The rest stays in Power Automate. ## FAQ diff --git a/content/posts/openadapt-vs-selenium/index.md b/content/posts/openadapt-vs-selenium/index.md index de89900..3f6c852 100644 --- a/content/posts/openadapt-vs-selenium/index.md +++ b/content/posts/openadapt-vs-selenium/index.md @@ -2,18 +2,14 @@ title: "OpenAdapt vs. Selenium: which for business data entry?" date: 2026-08-26 lastmod: 2026-10-09 -# Merged on 2026-10-09 into content/posts/openadapt-vs-api/index.md ("API, -# script, or recorded workflow: how to choose"). That post carries this one's -# Selenium row in its comparison table and a Selenium section with the facts -# below. Its aliases keep the old URL /posts/openadapt-vs-selenium/ working. -# Keep this file a draft: publishing it would collide with that alias. -# Rewritten 2026-10-16 as a standalone comparison page. The -# original text is in git history. No registry entries. See -# .overhaul/REWRITE_PLAN.json. +# Merged on 2026-10-09 into content/posts/openadapt-vs-api/index.md, then +# rewritten 2026-10-16 as a standalone comparison page and published again. +# openadapt-vs-api no longer carries an alias for this URL. No registry +# entries. The original text is in git history. draft: false author: "Richard Abrich" tags: ["comparison", "selenium", "browser-automation", "gui-automation"] -description: "Keep Selenium for browser tests and a working fleet. Use OpenAdapt for a record-writing task where each save must be checked and timeouts never retry blindly." +description: "When a Selenium click on Save times out, the record may or may not have been written. What to do about it, and when OpenAdapt is worth adding." faq: - q: "Is Selenium still a good choice?" a: "Yes, for browser testing and for teams with a working Selenium fleet. WebDriver is a W3C Recommendation, and Selenium Grid runs the same script across browsers and operating systems on other machines." @@ -28,7 +24,9 @@ audience: "practitioner" post_type: "note" --- -Use Selenium for browser testing, cross-browser runs, and any fleet your team already runs well. Use OpenAdapt for a business task that writes records, when you want a right-record check, a read of the saved record, and the same rule after every timeout, without writing that code yourself. If your team has already built those pieces around WebDriver, keep the fleet. +A Selenium Grid that tests a web app across browsers every night already knows how to fill in its forms. So it's a fair question when someone in operations asks whether those scripts could do the real data entry too. The browser part would work. The trouble is something a test never has to handle: a click on Save that times out, with no way to tell from the error whether the record was written. + +For testing, keep Selenium. For a job that writes business records, you'll need code that checks each save and decides what happens after a timeout. You can write that around WebDriver yourself. OpenAdapt makes those checks part of each recorded workflow, though someone still has to tell it how to read the record. ## OpenAdapt vs. Selenium at a glance @@ -42,17 +40,21 @@ Use Selenium for browser testing, cross-browser runs, and any fleet your team al ## When Selenium is the better choice -For test suites, Selenium is the better fit. A failed test stops with a stack trace and waits for a developer, which is what you want. Grid lets you run one script across browser versions and operating systems, and OpenAdapt has nothing like it. If your team already runs a working fleet, there's no reason to replace it because a newer tool has a cleaner demo. +For test suites, Selenium is the better fit. A failed test stops with a stack trace and waits for a developer, which is what you'd want. Grid lets you run one script across browser versions and operating systems, and OpenAdapt doesn't have anything like it. If your team already runs a working fleet, a newer tool with a cleaner demo is no reason to replace it. + +## The timeout after Save + +A timeout after a click that saves something has 2 possible meanings. The server may never have received the click. Or it saved the change and the reply got lost on the way back. A retry fixes the first case and creates a duplicate in the second, and the browser error looks the same either way. -## What your code owns around WebDriver +So before you retry, read the record. If your job runs unattended, nobody's there to look, so the read has to be in the code, along with a check that the right record was open before the click and a person who picks up the runs that stop. None of it is exotic, but WebDriver leaves all of it to you. -Selenium leaves everything around the browser to your code. A job that writes business records also needs a check that the right record is open before the click, a rule for timeouts, a second read that confirms the save, and a person who handles the runs that stop. +An OpenAdapt workflow handles the case the same way on every run. When a timeout follows a save, it stops and asks a person to check the record, and it doesn't retry on its own. The [Flow README](https://github.com/OpenAdaptAI/openadapt-flow#readme) shows the report a stopped run leaves behind. -A timeout after a click that saves something has 2 possible meanings. The server may never have received the click, or it may have saved the change before the reply was lost. A retry fixes the first case and creates a duplicate in the second. The browser error can't tell you which case you're in, so read the record before you try again. +If a supported API can make the change, use it instead of either tool. [When to automate the screen instead of an API](/posts/openadapt-vs-api/) compares Selenium with the other options. -An OpenAdapt workflow treats that case the same way on every run. When a timeout follows a save, it stops and asks a person to check the record, and it doesn't retry on its own. The [Flow README](https://github.com/OpenAdaptAI/openadapt-flow#readme) shows the report a stopped run leaves behind. +## Check one job this week -If a supported API can make the change, use it instead. [When to automate the screen instead of an API](/posts/openadapt-vs-api/) compares Selenium with the other options. +Pick a Selenium job that clicks Save on a real record and find what it does after a timeout. If it retries without reading the record first, there's a chance it saves the same change twice. Adding that one read is the cheapest fix, whichever tool you end up using. ## FAQ diff --git a/content/posts/openadapt-vs-uipath/index.md b/content/posts/openadapt-vs-uipath/index.md index 934a43d..697b392 100644 --- a/content/posts/openadapt-vs-uipath/index.md +++ b/content/posts/openadapt-vs-uipath/index.md @@ -9,7 +9,7 @@ author: "Richard Abrich" # date in the text. Re-checked 2026-10-16: same Basic price; the plan table # lists Healing Agent as an add-on for Standard and included in Enterprise. tags: ["comparison", "uipath", "rpa", "gui-automation"] -description: "Keep UiPath to run your automation program. Use OpenAdapt for one screen-only entry step that must be read back from the record and stop on a mismatch." +description: "Your UiPath robots extract the referral, then a person keys it into the EMR. Where OpenAdapt fits in that one step, and where UiPath stays the better tool." thesis: "Keep UiPath for the program it runs and let OpenAdapt take one task's last step: enter approved information, check that it saved, and stop for a person on a mismatch." audience: "business" post_type: "comparison" @@ -25,7 +25,9 @@ faq: --- -Use UiPath to run an automation program: many robots, scheduling, and governance across teams. Use OpenAdapt for one step that only happens on screen, such as keying an approved referral into an EMR, where you want each save read back from the record and the run stopped for a person on a mismatch. If you already run UiPath, keep it and add OpenAdapt only for that step. +If you run a UiPath program, a referral process might look like this. Robots read the faxed referrals and pull out the details, and Orchestrator schedules them and logs their work. One step still sits with a staff member. Someone keys each approved referral into the EMR and reopens the chart to make sure it saved, because the EMR has no API for that and a wrong entry lands in a patient's chart. + +That step is the only place OpenAdapt belongs in this picture. It learns the entry from a recording. On each run it types the referral and then reads the saved record back from the EMR. If the record doesn't match, it stops and hands the case to a person. Everything else stays in UiPath. ## OpenAdapt vs. UiPath at a glance @@ -33,7 +35,7 @@ Use UiPath to run an automation program: many robots, scheduling, and governance |---|---|---| | What it's for | Running attended and unattended robots across an organization ([Orchestrator](https://www.uipath.com/product/orchestrator)) | One recorded screen task whose save needs checking | | How you build an automation | Studio, low-code or coded, or SDKs ([pricing page](https://www.uipath.com/pricing)) | A person or an agent shows the task while OpenAdapt records it | -| Model calls on a normal run | Depends on what you build; agentic automations use agent calls | None; a clean run replays the compiled steps | +| Model calls on a normal run | Depends on what you build; agentic automations use agent calls | None; a normal run replays the recorded steps | | When the screen changes | Healing Agent, an add-on in Standard and included in Enterprise ([pricing page](https://www.uipath.com/pricing)) | Finds the same field again or stops; a person approves any fix | | Checking the saved record | Your developers build it into the workflow | Part of each workflow, through a separate read such as an API | | Price | Basic from $25 a month; Standard and Enterprise through sales (2026-10-16) | See the [comparison page](https://openadapt.ai/compare/uipath) | @@ -42,29 +44,29 @@ Use UiPath to run an automation program: many robots, scheduling, and governance According to UiPath's [Orchestrator page](https://www.uipath.com/product/orchestrator), you use Orchestrator to deploy and start attended and unattended robots and to monitor their work. It logs what each robot does and can run in UiPath's cloud or on your own servers. -A program with hundreds of automations also needs someone to own each one and to control how changes roll out. UiPath has spent years on that work. +A program with hundreds of automations also needs someone to own each one and to control how changes roll out. UiPath has spent years on that work, and OpenAdapt doesn't try to replace it. On 2026-10-16, UiPath's [pricing page](https://www.uipath.com/pricing) listed Basic as starting at $25 per month, capped at 2 robots and up to 5 Basic, 5 Plus, and 1 Pro user, hosted in UiPath's European region. Standard and Enterprise are priced through sales. The page also says an updated Basic offering is coming soon. -## Where OpenAdapt fits - -Here's how OpenAdapt handles that step: +## What OpenAdapt does at the entry step 1. A person or an agent shows the task while OpenAdapt records it, one or more times. -2. OpenAdapt builds an automation from the recording. A normal run follows it without calling an AI model. +2. OpenAdapt turns the recording into an automation. A normal run follows it without calling an AI model. 3. Before it types, it checks that it's in the right patient's record. 4. After it saves, it reads the record back from the system to confirm the change. -5. When something doesn't match, it stops and asks a person. If a save may have gone through, a person checks the record before anything is retried. +5. When something doesn't match, it stops and asks a person. -Before go-live, a readiness test runs the automation on your system with your own cases and shows which steps carry each check. If a screen changes later, OpenAdapt finds the same field again or stops. A person approves any fix it proposes. +If a save may have gone through before the stop, a person checks the record before anything is retried. + +Before go-live, a readiness test runs the automation on your system with your own cases and shows which steps carry each check. If a screen changes later, OpenAdapt finds the same field again or stops, and a person approves any fix it proposes. The record check needs its own way to read the record, such as an API or a separate login. Without one, OpenAdapt can still stop when the screen looks wrong, but it won't report a run as done and checked. -UiPath can run similar checks when your developers build them in. In OpenAdapt, the checks are part of the automation you test. +UiPath can run similar checks when your developers build them in. In OpenAdapt, they're part of the automation you test. ## What changes in your team's day -This is an illustration, not a customer result. Say your UiPath process already reads faxed referrals and extracts the details, and a staff member keys each approved referral into the EMR. Two hospital studies put that manual work at 10 to 12 minutes per referral ([UCSF, JAMIA Open 2020](https://pmc.ncbi.nlm.nih.gov/articles/PMC7660949/); [an NHS trust, HFMA case study 2024](https://www.hfma.org.uk/system/files/2024-04/Using%20digital%20technologies%20to%20process%20admin%20tasks%20Case%20Study%20v7.pdf)). +Back to the referral example, which is an illustration rather than a customer result. Two hospital studies put the manual entry at 10 to 12 minutes per referral ([UCSF, JAMIA Open 2020](https://pmc.ncbi.nlm.nih.gov/articles/PMC7660949/); [an NHS trust, HFMA case study 2024](https://www.hfma.org.uk/system/files/2024-04/Using%20digital%20technologies%20to%20process%20admin%20tasks%20Case%20Study%20v7.pdf)). | Step | Today | With OpenAdapt on the last step | |---|---|---| @@ -72,15 +74,13 @@ This is an illustration, not a customer result. Say your UiPath process already | Type the referral into the EMR | A staff member | OpenAdapt, on a computer you control | | Confirm it's the right patient | A staff member, by eye | OpenAdapt, before it types | | Confirm it saved | A staff member, by reopening the chart | OpenAdapt reads the saved record back | -| What comes back to staff | Every referral | The referrals it stopped on, with the reason, plus a receipt for each finished entry | - -A receipt records what OpenAdapt entered and how it checked it. +| What comes back to staff | Every referral | The referrals it stopped on, with the reason, plus a record of what it entered and how it checked each finished one | We haven't measured time saved in a deployment like this, so there's no number here. The NHS trust's robots still left about 1 in 20 referrals to staff, such as complex cases and data quality problems. Expect OpenAdapt to stop on some referrals too, such as a patient it can't match, and count that review work on the "after" side. ## How the two would connect -There's no published OpenAdapt connector for UiPath today. You could connect them across a normal process boundary: your UiPath process would hand one case to OpenAdapt and act on the result it returns. Before that design handles real cases, it needs identity, authorization, and audit design in your environment, plus a way to keep the same case from being entered twice. +There's no published OpenAdapt connector for UiPath today. You could connect them across a normal process boundary: your UiPath process hands one case to OpenAdapt and acts on the result it returns. Before that handles real cases, you'd need to design how the two authenticate and what each one logs, plus a way to keep the same case from being entered twice. | Result OpenAdapt returns | What it means | What your process does next | |---|---|---| @@ -93,9 +93,9 @@ If your process retries failed jobs on its own, leave the last two results out o ## When UiPath is the better choice -If a UiPath robot already makes the entry and your team trusts the result, you don't need OpenAdapt for that task. UiPath is also the better fit when you need one platform for many processes: document extraction, BPMN process orchestration, test automation, and audit across tenants are all on its [plan table](https://www.uipath.com/pricing), and OpenAdapt does none of those. If the system has an API that can make the change, use the API instead of either tool. +If a UiPath robot already makes the entry and your team trusts the result, you don't need OpenAdapt for that task. UiPath is also the better fit when you need one platform for many processes. Document extraction, BPMN process orchestration, test automation, and audit across tenants are all on its [plan table](https://www.uipath.com/pricing), and OpenAdapt does none of those. If the system has an API that can make the change, use the API instead of either tool. -OpenAdapt is for the step that only happens on screen, where you want each entry checked against the record. For a feature-by-feature view, see the [OpenAdapt and UiPath comparison](https://openadapt.ai/compare/uipath). +Before you add anything, ask whoever owns the entry step one question: after a robot or a person clicks Save, what reads the record back? If the answer is "someone looks at the screen," that's the step to look at. The [OpenAdapt and UiPath comparison](https://openadapt.ai/compare/uipath) has a feature-by-feature view. ## FAQ diff --git a/content/posts/openemr-benchmark/index.md b/content/posts/openemr-benchmark/index.md index 4d4dfa8..5ed9405 100644 --- a/content/posts/openemr-benchmark/index.md +++ b/content/posts/openemr-benchmark/index.md @@ -1,11 +1,11 @@ --- -title: "OpenEMR benchmark: compiled replay vs a computer-use agent" +title: "Replaying a recording vs. a computer-use agent on a real EMR" date: 2026-07-08 lastmod: 2026-10-09 draft: false author: "Richard Abrich" tags: ["openadapt-flow", "benchmark", "computer-use", "openemr", "safety", "automation"] -description: "On the OpenEMR demo, compiled replay saved 19 of 20 notes at $0 in model fees; a Claude computer-use agent saved 10 of 10 at $0.55 a run. Method and data." +description: "We ran both on the OpenEMR demo. A Claude agent saved 10 of 10 notes at $0.55 a run; replaying a recording saved 19 of 20 without model fees." thesis: "Compiled replay finished 19 of 20 OpenEMR runs at $0 in model fees and didn't claim success on its miss; an agent finished 10 of 10 at $0.55 a run." audience: "practitioner" post_type: "essay" @@ -14,9 +14,9 @@ aliases: - /posts/2026-08-28-the-one-number-that-drifted-was-the-one-that-changed-why-we/ --- -On the public OpenEMR demo, a compiled replay finished 19 of 20 runs of an 18-step charting task and made no model calls. A computer-use agent finished 10 of 10 and spent about $0.55 a run on model fees. The agent had the higher success rate. On the one compiled miss, the program clicked Save, saw that the screen didn't change as recorded, and stopped instead of reporting success. Compiled replay was also 1.8x faster at the median. +A demo app with five tidy screens flatters any automation. [The 500th run](/posts/the-500th-run/) compared a replayed recording with a computer-use agent on MockMed, our own demo app, so on 2026-07-08 we ran the same comparison on a real EMR. [OpenEMR](https://www.open-emr.org/) is an open-source EMR with crowded screens, and the task had to scroll to find what it needed. Its public demo holds only fake patients. -A compiled replay is a program that OpenAdapt builds from one or more recordings of a person or an agent doing the task. A computer-use agent is an AI model that reads screenshots and picks each click. [The 500th run](/posts/the-500th-run/) compared the two on MockMed, our own demo app, so on 2026-07-08 we ran the same test on a real EMR. [OpenEMR](https://www.open-emr.org/) is an open-source EMR with dense screens, and its public demo holds only fake patients. +The agent did better on success. A computer-use agent, an AI model that chooses its clicks from screenshots, finished 10 of 10 runs of the 18-step charting task and spent about $0.55 a run on model fees. OpenAdapt's compiled replay, a program built from a recording of the task, finished 19 of 20 runs and made no model calls. It was also 1.8x faster at the median. On its one miss, the program clicked Save, saw that the screen didn't change as recorded, and stopped instead of reporting success. ## What we tested diff --git a/content/posts/record-each-surface-then-compose/index.md b/content/posts/record-each-surface-then-compose/index.md index e2dacaf..45e3121 100644 --- a/content/posts/record-each-surface-then-compose/index.md +++ b/content/posts/record-each-surface-then-compose/index.md @@ -8,7 +8,7 @@ author: "Richard Abrich" # alias below. See .overhaul/REWRITE_PLAN.json. aliases: ["/posts/admitted-capabilities-then-a-process/"] tags: ["openadapt-flow", "computer-use", "automation", "gui-automation"] -description: "When work crosses two apps, OpenAdapt starts the second one only with a fact the first one saved and checked. Here's how that handoff works." +description: "If billing posts a charge to whichever patient is on screen, it can bill the wrong one. How to pass the patient ID from the EMR to billing safely." thesis: "When work crosses two applications, the second one should start only from a fact the first one saved and checked." audience: "practitioner" post_type: "essay" @@ -49,7 +49,7 @@ openadapt-flow visualize composed -o composed.html The `--handoff` line names the one fact that may cross from intake to posting. The last command draws the run as one box per recording, with the handoff labeled `patient_id` and a final box that says "End of declared steps". That final box shows where the declared steps end, and it makes no claim that any run succeeded. The Flow repository has a [rendered example](https://github.com/OpenAdaptAI/openadapt-flow/tree/main/docs/showcase-compose), generated from a synthetic test case. If you use the `openadapt` launcher, `openadapt flow compose` runs the same command. -## Switching apps is where agents slip +## What agent benchmarks show about switching apps [WindowsWorld](https://arxiv.org/abs/2604.27776) (arXiv:2604.27776) tested computer-use agents on 181 professional Windows tasks across 17 applications, and 78% of those tasks need more than one app. The best final success in their table is 20.44%, from Gemini-3-flash-preview working from a screenshot plus an accessibility tree. @@ -67,20 +67,20 @@ So a task that crosses a browser and a desktop app takes two recordings. That's By default, the steps run in the order you list them. The `--after` option lets you declare which steps wait on which, and OpenAdapt refuses a loop when you build the automation. -## Process contracts for automations that passed a readiness test +## Chaining automations that already passed a readiness test -Compose is for recordings that haven't had a readiness test on your system yet. When each automation has passed its own readiness test, a process contract sequences them instead. Flow added two versions within days of this post's first version: +Compose is for recordings that haven't had a readiness test on your system yet. Once each automation has passed its own readiness test, you chain them with a process contract instead. That's Flow's file for a multi-step process: it lists the approved automations in order and what each one may pass to the next. Flow added two versions within days of this post's first version: - Version 0 ([Flow #434](https://github.com/OpenAdaptAI/openadapt-flow/pull/434), merged 2026-08-29) runs automations that each carry a signed, current approval from their readiness test. It passes along only facts that the earlier step's record check confirmed, the same rule compose uses. -- Version 1 ([Flow #444](https://github.com/OpenAdaptAI/openadapt-flow/pull/444), merged 2026-08-31) adds sealed Python steps and signed human tasks to the same process. It passes files by a fingerprint of their contents, and it won't report success until the checker you declared has confirmed those files. Version 0 files keep working. +- Version 1 ([Flow #444](https://github.com/OpenAdaptAI/openadapt-flow/pull/444), merged 2026-08-31) lets the same process include Python steps and signed tasks for a person to do. It passes files by a fingerprint of their contents, and it won't report success until the checker you declared has confirmed those files. Version 0 files keep working. -Before each step runs, the process checks that step's approval. If the approval expired, was withdrawn, or no longer matches the automation, the process stops before that step starts. A composed recording can't be a step, because it has no approval of its own. Version 1 ends with a signed receipt that covers each step's receipt. +Before each step runs, the process checks that step's approval. If the approval expired, was withdrawn, or no longer matches the automation, the process stops before that step starts. A composed recording can't be a step, because it has no approval of its own. Version 1 ends with one signed record of the whole process that includes each step's own signed record. -Process contracts keep the rule for uncertain saves. If a step may have saved but OpenAdapt couldn't confirm it, the process stops with a "check the record" result. The process won't run that step again on its own, and a person checks the record before anything is retried. The [process contract guide](https://github.com/OpenAdaptAI/openadapt-flow/blob/main/docs/PROCESS_CONTRACT.md) covers both versions. +Both versions keep the rule for uncertain saves. If a step may have saved but OpenAdapt couldn't confirm it, the process stops with a "check the record" result. The process won't run that step again on its own, and a person checks the record before anything is retried. The [process contract guide](https://github.com/OpenAdaptAI/openadapt-flow/blob/main/docs/PROCESS_CONTRACT.md) covers both versions. ## When an agent is still the right tool -A new screen, or a task nobody has shown OpenAdapt, still belongs to a computer-use agent. We compared the two approaches on a repeated task in [compiled automation vs. computer-use agents](/posts/the-500th-run/). This post covers the other case, where the same two-app task runs every week and the second app must never act on a guessed patient ID. +A new screen, or a task nobody has shown OpenAdapt, still belongs to a computer-use agent. We compared the two approaches on a repeated task in [what a computer-use agent costs by its 500th run](/posts/the-500th-run/). This post covers the other case, where the same two-app task runs every week and the second app must never act on a guessed patient ID. ## Technical details diff --git a/content/posts/silent-wrong-action/index.md b/content/posts/silent-wrong-action/index.md index 1c08aea..f2a3c9b 100644 --- a/content/posts/silent-wrong-action/index.md +++ b/content/posts/silent-wrong-action/index.md @@ -5,7 +5,7 @@ lastmod: 2026-10-09 draft: false author: "Richard Abrich" tags: ["openadapt-flow", "safety", "automation", "benchmark", "rpa", "validation"] -description: "Trusting the Saved banner passed 54 of 72 bad saves in our fault test. A separate read of the record passed 9. What two checks catch, and what they miss." +description: "We broke saves on purpose. A check that trusted the Saved banner passed 54 of 72 bad saves, and reading the record back passed 9. What they still miss." thesis: "In our fault test, a check that trusted the success banner passed 54 of 72 bad saves, and one separate read of the record passed 9." audience: "practitioner" post_type: "essay" @@ -13,9 +13,9 @@ aliases: - /posts/2026-07-20-compile-once-govern-every-repair/ --- -Most automation decides that a save worked by looking for the green "Saved" banner. We tested how far that banner can be trusted. In our fault test, 72 of 90 runs left the record wrong. A check that trusted the banner passed 54 of 72 bad saves (75.0%). A check that read the saved record back by a separate route passed 9 of 72 bad saves (12.5%). +Say an automation has been posting notes to patient charts for a month, and the runs have all come back green. Then a nurse finds a triage note in the wrong patient's chart. Nothing crashed and nobody was paged. The automation saw a "Saved" banner appear and never asked whose record it had saved to. -A bad save that reports success does more harm than a crash, because nobody gets paged. The error turns up later with whoever owns the record, as a note in the wrong patient's chart or a payment on the wrong loan. The automation's check confirmed that something was saved, without asking whose record it was. +We wanted to see which bad saves a banner check lets through, so we broke saves on purpose in a test service. In that fault test, 72 of 90 runs left the record wrong. A check that trusted the banner passed 54 of 72 bad saves (75.0%). A check that read the saved record back by a separate route passed 9 of 72 bad saves (12.5%). OpenAdapt can put two checks around a save. Before it clicks, a right-record check confirms it's in the intended patient's or customer's record. After the save, a record check reads the record back from the system. If either check fails or can't tell, the run stops and a person decides what happens next. diff --git a/content/posts/the-500th-run/index.md b/content/posts/the-500th-run/index.md index 956713a..9f8f577 100644 --- a/content/posts/the-500th-run/index.md +++ b/content/posts/the-500th-run/index.md @@ -1,5 +1,5 @@ --- -title: "Compiled automation vs. computer-use agents: cost at 500 runs" +title: "What a computer-use agent costs by its 500th run" date: 2026-07-08 lastmod: 2026-10-09 draft: false @@ -10,15 +10,15 @@ aliases: - /posts/the-user-is-an-agent/ - /posts/openadapt-vs-computer-use-agents/ tags: ["openadapt-flow", "benchmark", "computer-use", "comparison", "automation"] -description: "On MockMed in July 2026, compiled replay passed 100/100 runs at a 4.9 s median and $0 in model fees. An agent passed 20/20 at 37.5 s and $0.27 a run." +description: "An agent pays model fees run after run. In our July 2026 test, a replayed recording took 4.9 s a run without model fees; an agent took 37.5 s at $0.27." thesis: "For a repeated task, a compiled replay reuses one recording without model calls, while a computer-use agent pays the model to work through the same screens run after run." audience: "practitioner" post_type: "essay" --- -A computer-use agent is an AI model that works from screenshots and chooses each click as it goes. It doesn't need selectors, an application API, or a recording, so it can take on a task that nobody has automated yet. The cost shows up when the task repeats. The agent calls the model at each step of each run, even when the screens and the goal haven't changed since yesterday. +The first time a computer-use agent does one of your team's chores on its own, working through the screens from one sentence of instructions, the obvious next step is to run it every day. Before you do, look at what each of those runs costs, because the agent pays a model to reason through the same screens every time. -A compiled replay works the other way. A person or an agent shows the task while openadapt-flow records it, and the recording becomes a program that replays on later runs without calling a model. +A computer-use agent is an AI model that works from screenshots and chooses each click as it goes. It needs no selectors and no recording, which is why it can take on a task nobody has automated yet. A compiled replay works the other way. A person or an agent shows the task once while openadapt-flow records it, and the recording becomes a program (that's the "compiling") that replays on later runs without calling a model. On 2026-07-08 we measured both on one task in MockMed, the demo clinic app that ships with openadapt-flow. The compiled replay passed 100/100 runs at a 4.9 s median with $0 in model fees. The agent passed 20/20 runs at a 37.5 s median and $0.2716 a run at list price. @@ -81,7 +81,7 @@ Choose by how new the work is each time. For either one, keep a person in control of high-impact actions until the exact workflow has the evidence and approvals it needs. -The two can share one process. An agent can explore an unfamiliar app and help a person find the path. Once the task is stable and frequent, the person records it so the compiled program can run it from then on. OpenAdapt has no connector built for a particular agent provider. An agent that speaks MCP can list and run approved workflows on the same computer through [OpenAdapt Agent](https://openadapt.ai/platform/agent). +The two can share one process. An agent can explore an unfamiliar app and help a person find the path. Once the task is stable and frequent, the person records it so the compiled program can run it from then on. OpenAdapt has no connector built for a particular agent provider. An agent that speaks MCP (Model Context Protocol, a common way for AI agents to call tools) can list and run approved workflows on the same computer through [OpenAdapt Agent](https://openadapt.ai/platform/agent). ## Limits @@ -102,7 +102,7 @@ openadapt quickstart On Python 3.13 or newer, pip installs an old release that has no `quickstart` command. If you use [uv](https://docs.astral.sh/uv/), run `uv tool install --python 3.12 openadapt` instead, then run `openadapt quickstart`. -The run ends done and checked (`VERIFIED` in the report). The report lists each action and the separate evidence for the saved record. To record your own web app next, follow the [first-workflow guide](https://docs.openadapt.ai/get-started/first-workflow/). +The run should end done and checked. Its report lists each action and the separate evidence for the saved record. To record your own web app next, follow the [first-workflow guide](https://docs.openadapt.ai/get-started/first-workflow/). To rerun the benchmark, clone openadapt-flow. Today's runner repeats the same task, but it can't rebuild the July engine because that commit wasn't kept, so expect different numbers. The agent arm needs an Anthropic API key, and its runs cost $5.43 at list price when we ran them. diff --git a/content/posts/the-write-audit/index.md b/content/posts/the-write-audit/index.md index 6d68b80..5b6bcef 100644 --- a/content/posts/the-write-audit/index.md +++ b/content/posts/the-write-audit/index.md @@ -5,15 +5,15 @@ lastmod: 2026-10-09 draft: false author: "Richard Abrich" tags: ["automation", "rpa", "agents", "safety", "validation", "testing", "reliability"] -description: "Success rates that count finished runs include bad saves. In our fault test, trusting the banner passed 54 of 72 bad saves. Here's how to audit yours." +description: "If your success rate counts every run that finished, it counts the bad saves too. How to find them in a week, with tools you already have." thesis: "A success rate that counts finished runs includes runs that saved the wrong thing, so check saves against the record and report silent bad saves and unnecessary stops together." audience: "practitioner" post_type: "essay" --- -If your automation counts a run as a success when it reaches its last step without an error, that count includes the runs that saved the wrong thing. In our own fault test, a check that trusted the success banner passed 54 of 72 bad saves. +Look at how your automation decides a run succeeded. If reaching the last step without an error is enough, your success rate includes the runs that saved the wrong thing. They finish cleanly, so they count. In our own fault test, a check that trusted the success banner passed 54 of 72 bad saves. -A write audit separates those runs from the good ones. You compare what each run saved with the record itself, and you report two numbers: the bad saves that got through, and the good saves your automation stopped anyway. The method works on a recorded RPA process, a Playwright script, a Power Automate desktop flow, a computer-use agent, or a person following a checklist. You can run it with tools you already have. +A write audit pulls those runs out. You compare what each run saved with the record itself, and you report two numbers: the bad saves that got through, and the good saves your automation stopped anyway. It works the same whether the work is done by an RPA bot or a person following a checklist, and it needs only tools you already have. ## Sort every run into four outcomes @@ -37,7 +37,7 @@ Every automation uses some check to decide that a save worked, even if nobody ch | 0 | The screen: a banner, a toast, a new row, or a spinner that stopped | Any fault where the screen shows success while the server did something else | | 1 | The same screen again, after you go back to the record | Faults hidden by the cache, session, and display path the save itself used | | 2 | The application's API, queried by record ID | Changes to tables the query doesn't read, and anything the API reports from its own cached state | -| 3 | The system of record and what's downstream: the database, ledger, claims feed, or interface engine, compared before and after | Effects outside what you read, such as a message sent to another system, and any fault nobody thought to check for | +| 3 | The system of record and what's downstream: the database, ledger, claims feed, or interface engine, compared before and after | Changes outside what you read, such as a message sent to another system, and any fault nobody thought to check for | A recorder sees the screen, so recorded automation starts at tier 0 unless someone adds more. A computer-use agent that judges success from a screenshot sits at tier 0 too, and most of the faults in the next section leave a success message on the screen. diff --git a/scripts/benchmark_claims/registry.json b/scripts/benchmark_claims/registry.json index 3881a96..56a6bdf 100644 --- a/scripts/benchmark_claims/registry.json +++ b/scripts/benchmark_claims/registry.json @@ -71,7 +71,7 @@ "source": "openemr", "numerator": "/arms/compiled/success_count", "denominator": "/arms/compiled/n", - "context": "compiled replay saved 19 of 20 notes" + "context": "We ran both on the OpenEMR demo" }, { "file": "content/posts/openemr-benchmark/index.md", @@ -80,16 +80,7 @@ "source": "openemr", "numerator": "/arms/agent/success_count", "denominator": "/arms/agent/n", - "context": "compiled replay saved 19 of 20 notes" - }, - { - "file": "content/posts/openemr-benchmark/index.md", - "token": "$0", - "kind": "number", - "source": "openemr", - "pointer": "/arms/compiled/cost_usd_per_run", - "round": 0, - "context": "compiled replay saved 19 of 20 notes" + "context": "We ran both on the OpenEMR demo" }, { "file": "content/posts/openemr-benchmark/index.md", @@ -98,7 +89,7 @@ "source": "openemr", "pointer": "/arms/agent/cost_usd_per_run", "round": 2, - "context": "compiled replay saved 19 of 20 notes" + "context": "We ran both on the OpenEMR demo" }, { "file": "content/posts/openemr-benchmark/index.md", @@ -143,7 +134,7 @@ "source": "openemr", "numerator": "/arms/compiled/success_count", "denominator": "/arms/compiled/n", - "context": "On the public OpenEMR demo, a compiled replay finished" + "context": "The agent did better on success" }, { "file": "content/posts/openemr-benchmark/index.md", @@ -152,7 +143,7 @@ "source": "openemr", "numerator": "/arms/agent/success_count", "denominator": "/arms/agent/n", - "context": "On the public OpenEMR demo, a compiled replay finished" + "context": "The agent did better on success" }, { "file": "content/posts/openemr-benchmark/index.md", @@ -161,7 +152,7 @@ "source": "openemr", "pointer": "/arms/agent/cost_usd_per_run", "round": 2, - "context": "On the public OpenEMR demo, a compiled replay finished" + "context": "The agent did better on success" }, { "file": "content/posts/openemr-benchmark/index.md", @@ -170,7 +161,7 @@ "source": "comparison", "pointer": "/derived/openemr_speedup_p50", "round": 1, - "context": "On the public OpenEMR demo, a compiled replay finished" + "context": "The agent did better on success" }, { "file": "content/posts/openemr-benchmark/index.md", @@ -633,7 +624,7 @@ "source": "effect_e2e", "numerator": "/metrics/per_arm/screen/silent_wrong_count", "denominator": "/metrics/per_arm/screen/n_wrong_effect", - "context": "Trusting the Saved banner passed 54 of 72 bad saves" + "context": "A check that trusted the Saved banner passed 54 of 72 bad saves" }, { "file": "content/posts/silent-wrong-action/index.md", @@ -651,7 +642,7 @@ "source": "effect_e2e", "numerator": "/metrics/per_arm/screen/n_wrong_effect", "denominator": "/metrics/per_arm/screen/n_runs", - "context": "We tested how far that banner can be trusted" + "context": "so we broke saves on purpose in a test service" }, { "file": "content/posts/silent-wrong-action/index.md", @@ -660,7 +651,7 @@ "source": "effect_e2e", "numerator": "/metrics/per_arm/screen/silent_wrong_count", "denominator": "/metrics/per_arm/screen/n_wrong_effect", - "context": "We tested how far that banner can be trusted" + "context": "so we broke saves on purpose in a test service" }, { "file": "content/posts/silent-wrong-action/index.md", @@ -670,7 +661,7 @@ "pointer": "/metrics/per_arm/screen/undetected_wrong_rate", "round": 1, "scale": 100, - "context": "We tested how far that banner can be trusted" + "context": "so we broke saves on purpose in a test service" }, { "file": "content/posts/silent-wrong-action/index.md", @@ -679,7 +670,7 @@ "source": "effect_e2e", "numerator": "/metrics/per_arm/effect_rest/silent_wrong_count", "denominator": "/metrics/per_arm/effect_rest/n_wrong_effect", - "context": "We tested how far that banner can be trusted" + "context": "so we broke saves on purpose in a test service" }, { "file": "content/posts/silent-wrong-action/index.md", @@ -689,7 +680,7 @@ "pointer": "/metrics/per_arm/effect_rest/undetected_wrong_rate", "round": 1, "scale": 100, - "context": "We tested how far that banner can be trusted" + "context": "so we broke saves on purpose in a test service" }, { "file": "content/posts/silent-wrong-action/index.md", @@ -945,15 +936,6 @@ "round": 2, "context": "we changed the color theme of our synthetic MockMed app" }, - { - "file": "content/posts/the-500th-run/index.md", - "token": "100/100", - "kind": "ratio", - "source": "comparison", - "numerator": "/benchmarks/mockmed/arms/compiled/success_count", - "denominator": "/benchmarks/mockmed/arms/compiled/n", - "context": "On MockMed in July 2026, compiled replay passed" - }, { "file": "content/posts/the-500th-run/index.md", "token": "4.9 s", @@ -961,25 +943,7 @@ "source": "comparison", "pointer": "/benchmarks/mockmed/arms/compiled/wall_s_p50", "round": 1, - "context": "On MockMed in July 2026, compiled replay passed" - }, - { - "file": "content/posts/the-500th-run/index.md", - "token": "$0", - "kind": "number", - "source": "comparison", - "pointer": "/benchmarks/mockmed/arms/compiled/cost_usd_per_run", - "round": 0, - "context": "On MockMed in July 2026, compiled replay passed" - }, - { - "file": "content/posts/the-500th-run/index.md", - "token": "20/20", - "kind": "ratio", - "source": "comparison", - "numerator": "/benchmarks/mockmed/arms/agent/success_count", - "denominator": "/benchmarks/mockmed/arms/agent/n", - "context": "On MockMed in July 2026, compiled replay passed" + "context": "In our July 2026 test, a replayed recording took" }, { "file": "content/posts/the-500th-run/index.md", @@ -988,7 +952,7 @@ "source": "comparison", "pointer": "/benchmarks/mockmed/arms/agent/wall_s_p50", "round": 1, - "context": "On MockMed in July 2026, compiled replay passed" + "context": "In our July 2026 test, a replayed recording took" }, { "file": "content/posts/the-500th-run/index.md", @@ -997,7 +961,7 @@ "source": "comparison", "pointer": "/benchmarks/mockmed/arms/agent/cost_usd_per_run", "round": 2, - "context": "On MockMed in July 2026, compiled replay passed" + "context": "In our July 2026 test, a replayed recording took" }, { "file": "content/posts/the-500th-run/index.md", @@ -1288,15 +1252,6 @@ "round": 2, "context": "To rerun the benchmark, clone openadapt-flow" }, - { - "file": "content/posts/the-write-audit/index.md", - "token": "54 of 72", - "kind": "ratio", - "source": "effect_e2e", - "numerator": "/metrics/per_arm/screen/silent_wrong_count", - "denominator": "/metrics/per_arm/screen/n_wrong_effect", - "context": "trusting the banner passed 54 of 72 bad saves" - }, { "file": "content/posts/the-write-audit/index.md", "token": "54 of 72", @@ -2090,7 +2045,7 @@ }, { "file": "content/posts/openemr-benchmark/index.md", - "sentence": "On the public OpenEMR demo, a compiled replay finished 19 of 20 runs of an 18-step charting task and made no model calls.", + "sentence": "OpenAdapt's compiled replay, a program built from a recording of the task, finished 19 of 20 runs and made no model calls.", "reviewed": "2026-10-09", "evidence": { "claim": "equals_zero",