With AI tools now acting as code reviewers, how should we ensure the quality and reliability of these AI reviewers themselves?

AI promoted every developer to reviewer. Nobody tested the reviewer. - DEV Community body:has(.pageslug-aie) #topbar { display: none !important; } @media screen and (max-width: 768px) and (pointer: coarse) { /* Target WebKit/Blink specific behaviors */ @supports (-webkit-app-region: none) { body:has(.pageslug-aie) .dc-page { padding-top: 56px !important; } } } body:not(.hidden-shell) .pageslug-aie { margin-top: -56px !important; } /* ========================================================================== THE 418 CHALLENGE: THEME-AWARE RETRO EDITION ========================================================================== */ .articletag-418challenge { font-family: "Comic Sans MS", "Chalkboard SE", "Marker Felt", sans-serif !important; cursor: crosshair !important; } /* Destroy modern rounded corners and add brutal borders */ .articletag-418challenge .crayons-card, .articletag-418challenge .crayons-article__main, .articletag-418challenge .crayons-article__header { border-radius: 0 !important; margin-bottom: 2rem !important; } /* Typography specifics */ .articletag-418challenge h1, .articletag-418challenge h2, .articletag-418challenge h3 { text-transform: uppercase !important; } /* Fake effect for the main article title */ .articletag-418challenge h1.fs-3xl { animation: retro-slide 5s linear infinite alternate; overflow: visible; } /* Spin those avatars */ .articletag-418challenge .crayons-article__header__meta img { border-radius: 0 !important; animation: retro-spin 5s linear infinite; } /* Obnoxious tags */ .articletag-418challenge .crayons-tag { border-radius: 0 !important; font-weight: 900 !important; transform: rotate(-3deg); display: inline-block; } .articletag-418challenge .crayons-tag:nth-child(even) { transform: rotate(3deg); } /* -------------------------------------------------------------------------- 2. LIGHT THEME (Windows 95 / Office 97 Vibes) -------------------------------------------------------------------------- */ body:not(.dark-theme) .articletag-418challenge { background-color: #008080 !important; /* Win95 Teal Desktop */ } body:not(.dark-theme) .articletag-418challenge .crayons-card, body:not(.dark-theme) .articletag-418challenge .crayons-article__main, body:not(.dark-theme) .articletag-418challenge .crayons-article__header { background-color: #c0c0c0 !important; /* Classic dialog gray */ color: #000000 !important; border: 4px outset #ffffff !important; box-shadow: 8px 8px 0px #000000 !important; } body:not(.dark-theme) .articletag-418challenge h1, body:not(.dark-theme) .articletag-418challenge h2, body:not(.dark-theme) .articletag-418challenge h3 { color: #ff0000 !important; text-shadow: 2px 2px 0px #ffff00 !important; border-bottom: 3px dashed #0000ff !important; } body:not(.dark-theme) .articletag-418challenge a, body:not(.dark-theme) .articletag-418challenge .crayons-link { color: #0000ff !important; /* Standard unvisited blue */ text-decoration: underline !important; font-weight: bold !important; } body:not(.dark-theme) .articletag-418challenge a:hover, body:not(.dark-theme) .articletag-418challenge .crayons-link:hover { background-color: #ffff00 !important; color: #ff0000 !important; } body:not(.dark-theme) .articletag-418challenge .crayons-tag { background: #ffff00 !important; color: #ff0000 !important; border: 2px dotted #0000ff !important; } /* -------------------------------------------------------------------------- 3. DARK THEME (1999 Hacker / Deep GeoCities Vibes) -------------------------------------------------------------------------- */ body.dark-theme .articletag-418challenge { background-color: #000000 !important; /* Dumb CSS: A scrolling, eye-bleeding neon green hacker grid with CRT scanlines */ background-image: linear-gradient(transparent 50%, rgba(0, 255, 0, 0.25) 50%), linear-gradient(90deg, rgba(0, 255, 0, 0.5) 1px, transparent 1px), linear-gradient(rgba(0, 255, 0, 0.5) 1px, transparent 1px) !important; background-size: 100% 4px, 20px 20px, 20px 20px !important; animation: retro-pan 60s linear infinite !important; } body.dark-theme .articletag-418challenge .crayons-card, body.dark-theme .articletag-418challenge .crayons-article__main, body.dark-theme .articletag-418challenge .crayons-article__header { background-color: #111111 !important; color: #00ff00 !important; /* Terminal Green */ border: 4px outset #555555 !important; box-shadow: 8px 8px 0px #ff00ff !important; /* Hot pink shadow */ } body.dark-theme .articletag-418challenge h1, body.dark-theme .articletag-418challenge h2, body.dark-theme .articletag-418challenge h3 { color: #ff00ff !important; /* Magenta headers */ text-shadow: 2px 2px 0px #00ffff !important; /* Cyan shadow */ border-bottom: 3px dashed #00ff00 !important; } body.dark-theme .articletag-418challenge a, body.dark-theme .articletag-418challenge .crayons-link { color: #00ffff !important; /* Cyan links */ text-decoration: underline !important; font-weight: bold !important; } body.dark-theme .articletag-418challenge a:hover, body.dark-theme .articletag-418challenge .crayons-link:hover { background-color: #ff00ff !important; color: #ffffff !important; } body.dark-theme .articletag-418challenge .crayons-tag { background: #000000 !important; color: #00ff00 !important; border: 2px dotted #ff00ff !important; } /* -------------------------------------------------------------------------- 4. ANIMATIONS -------------------------------------------------------------------------- */ @keyframes retro-spin { 100% { transform: rotate(360deg); } } @keyframes retro-slide { 0% { transform: translateX(-2%); } 100% { transform: translateX(2%); } } @keyframes retro-pan { 0% { background-position: 0 0, 0 0, 0 0; } /* Numbers must be multiples of the background-size (4px and 20px) to loop perfectly */ 100% { background-position: 0 100px, 40px 40px, 40px 40px; } } /* -------------------------------------------------------------------------- 5. BONUS: CAUTION TAPE REACTION BAR -------------------------------------------------------------------------- */ .articletag-418challenge .crayons-article-actions { background: repeating-linear-gradient( 45deg, #ffff00, #ffff00 10px, #000000 10px, #000000 20px ) !important; border: 4px solid #ff0000 !important; border-radius: 0 !important; } .articletag-418challenge .crayons-article-actions button { background:black; color: #ff00ff !important; } .articletag-418challenge .crayons-article-actions .crayons-reaction__count { color: #ff00ff !important; font-weight: bold !important; } .articletag-418challenge .crayons-icon:not(.crayons-icon--default) * { fill: #ffff00 !important; } .articletag-418challenge .c-embed .crayons-btn--primary { color: #00ff00 !important; } .articletag-418challenge .popover-billboard { margin-bottom: 0 !important; } .crayons-story__contentpreview .ltag__link--embedded { margin-top: 0 !important; } .subscription-icon { max-height: 16px !important; display: inline-block !important; } @media screen and (min-width: 950px) { .event-show-layout { margin-top: 24px !important; } } @media screen and (max-width: 949px) { .event-show-layout { padding: 11px !important; } } if (navigator.userAgent === 'ForemWebView/1' || window.frameElement) { document.body.classList.add("hidden-shell"); } if (new Date() > new Date("2026-02-04T09:00:00-05:00")) { document.body.dataset.sideNavVisible = "false"; } :root { --accent-brand-lighter-rgb: 86, 105, 285; --accent-brand-rgb: 64, 78, 211; --accent-brand-darker-rgb: 51, 62, 169; } Skip to content Navigation menu Search Powered by Algolia Search Log in Create account DEV Community Close .html-variant-wrapper { display: none} {"@context":"http://schema.org","@type":"Article","mainEntityOfPage":{"@type":"WebPage","@id":"https://dev.to/heinrichneb/ai-promoted-every-developer-to-reviewer-nobody-tested-the-reviewer-m4h"},"url":"https://dev.to/heinrichneb/ai-promoted-every-developer-to-reviewer-nobody-tested-the-reviewer-m4h","image":["https://media2.dev.to/dynamic/image/width=1080,height=1080,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Farticles%2Fmhp0ukaw2xook5tccxej.png","https://media2.dev.to/dynamic/image/width=1280,height=720,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Farticles%2Fmhp0ukaw2xook5tccxej.png","https://media2.dev.to/dynamic/image/width=1600,height=900,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Farticles%2Fmhp0ukaw2xook5tccxej.png"],"publisher":{"@context":"http://schema.org","@type":"Organization","name":"DEV Community","logo":{"@context":"http://schema.org","@type":"ImageObject","url":"https://media2.dev.to/dynamic/image/width=192,height=,fit=scale-down,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2F8j7kvp660rqzt99zui8e.png","width":"192","height":"192"}},"headline":"AI promoted every developer to reviewer. Nobody tested the reviewer.","author":{"@context":"http://schema.org","@type":"Person","url":"https://dev.to/heinrichneb","name":"Heinrich Neb"},"datePublished":"2026-08-24T23:23:06Z","dateModified":"2026-08-24T23:23:06Z","mainEntity":{"@type":"DiscussionForumPosting","@id":"#article-discussion-4479046","headline":"AI promoted every developer to reviewer. Nobody tested the reviewer.","text":"\u003cp\u003e\u003cem\u003eI wanted to disagree with 'AI made me a worse reviewer' from Michael Amachree (\u003ca class=\"mentioned-user\" href=\"https://dev.to/dev_michael\"\u003e@dev_michael\u003c/a\u003e) . Instead I counted 204 of my own guards — and 89 % of them have never been asked to prove they can fail.\u003c/em\u003e\u003c/p\u003e\n\n\u003cp\u003eMichael wrote something that I couldn't put down: AI didn't make me a worse coder, it made me a worse reviewer. Here is the number, and it's worse than his thesis: of the 204 automated checks in my repositories that draw a conclusion, only 22 can prove they are able to fail. That's 11 %. The other 89 % have never once been shown a known-bad input. They are green. Whether they are green because everything is fine, or green because they are incapable of finding anything - I could not have told you last week. And I'm the person who wrote them.\u003c/p\u003e\n\n\u003ch2\u003e\n \u003ca name=\"what-i-actually-counted\" href=\"#what-i-actually-counted\"\u003e\n \u003c/a\u003e\n What I actually counted\n\u003c/h2\u003e\n\n\u003cp\u003eFirst the definition, so you can reject it or reuse it.\u003c/p\u003e\n\n\u003cp\u003eA \u003cstrong\u003econclusion-bearing guard\u003c/strong\u003e is any test that reads source code, config, or system state and asserts a claim about it. Not \"does this function return 4\" - but \"no workflow downloads its cache over the network\", \"every page passes the same quarter filter\", \"this feature flag matches the deployed spec\". The tests that stand in for a human reviewer.\u003c/p\u003e\n\n\u003cp\u003eA \u003cstrong\u003enegative control\u003c/strong\u003e is a probe that feeds that guard a known-bad input and asserts it gets rejected \u003cem\u003efor the expected reason\u003c/em\u003e. Our convention marks them \u003ccode\u003eKONTROLLE:\u003c/code\u003e in the test name.\u003c/p\u003e\n\n\u003cp\u003eCounting is mechanical: 204 guard files across three repositories, 22 with at least one control probe, 54 probes total. The counter is a proxy - marker-based, so unmarked controls and false-positive guard files put the true number at plus or minus a few points. The shape survives any correction: \u003cstrong\u003emost of my reviewers have never been reviewed.\u003c/strong\u003e\u003c/p\u003e\n\n\u003ch2\u003e\n \u003ca name=\"three-greenandblind-checks-one-ordinary-week\" href=\"#three-greenandblind-checks-one-ordinary-week\"\u003e\n \u003c/a\u003e\n Three green-and-blind checks, one ordinary week\n\u003c/h2\u003e\n\n\u003cp\u003eThis isn't theoretical. All three of these happened to me in the last seven days, in production tooling.\u003c/p\u003e\n\n\u003cp\u003e\u003cstrong\u003eThe deploy gate that died of its own medicine.\u003c/strong\u003e A pipeline step existed specifically to catch a silent failure mode - a missing tool falling back to an empty result. It called \u003ccode\u003enode -e\u003c/code\u003e to parse a health response. The deploy runner has no Node. Six consecutive deployments failed with exit 127 - the check \u003cem\u003eagainst\u003c/em\u003e missing tools failed \u003cem\u003eon a missing tool\u003c/em\u003e, and nothing shipped for six hours. The step had been green in review because nobody had ever run it where it actually runs.\u003c/p\u003e\n\n\u003cp\u003e\u003cstrong\u003eThe harvester that threw away its own work.\u003c/strong\u003e An autonomous job collected data from public repositories and judged each run by exit code. One run wrote seven perfectly good records, then hit a non-fatal warning and exited non-zero. The machine booked its own completed work as \"failed, retry later\" - because \u003cem\u003einterrupted-with-partial-results\u003c/em\u003e had no representation, only success and failure. We caught it because the result file was sitting on disk right next to the exit code that denied its existence.\u003c/p\u003e\n\n\u003cp\u003e\u003cstrong\u003eThe pattern that matched the wrong 500.\u003c/strong\u003e An error classifier looked for server errors with the pattern \u003ccode\u003e50[024]\u003c/code\u003e - anywhere in the output. It matched the \"500\" inside \u003cem\u003e\"4258 of 5000 quota points remaining\"\u003c/em\u003e and classified a successful run as a server failure. Every field it read was real. It was answering a different question than the one asked.\u003c/p\u003e\n\n\u003cp\u003eThree different systems. One shape: \u003cstrong\u003ethe check watched a messenger - an exit code, a pattern, a status - while the artifact that mattered told a different story.\u003c/strong\u003e\u003c/p\u003e\n\n\u003ch2\u003e\n \u003ca name=\"what-this-has-to-do-with-ai-making-you-a-worse-reviewer\" href=\"#what-this-has-to-do-with-ai-making-you-a-worse-reviewer\"\u003e\n \u003c/a\u003e\n What this has to do with AI making you a worse reviewer\n\u003c/h2\u003e\n\n\u003cp\u003eHere's where I think Michael's post lands harder than he says.\u003c/p\u003e\n\n\u003cp\u003eAI moved my job. I used to spend most of my day producing artifacts and a little of it verifying them. Now an agent produces most of the artifacts, and my job \u003cem\u003eis\u003c/em\u003e verification. Which means my real codebase - the one my judgment actually ships through - is those 204 guards.\u003c/p\u003e\n\n\u003cp\u003eAnd that codebase is held to a standard I would reject in application code. No test coverage (11 %). No review of the reviewer. Green as the default state, silence booked as success.\u003c/p\u003e\n\n\u003cp\u003eWhen Michael says AI made him a worse reviewer, I'd sharpen it: \u003cstrong\u003eAI promoted us all to reviewers, and none of us tested the reviewer.\u003c/strong\u003e The model isn't the weak link. The unfalsifiable green checkmark is.\u003c/p\u003e\n\n\u003ch2\u003e\n \u003ca name=\"the-rule-that-survived-the-week\" href=\"#the-rule-that-survived-the-week\"\u003e\n \u003c/a\u003e\n The rule that survived the week\n\u003c/h2\u003e\n\n\u003cp\u003eEverything above collapses into one sentence we now apply mechanically:\u003c/p\u003e\n\n\u003cp\u003e\u003cstrong\u003eJudge the artifact, not the messenger.\u003c/strong\u003e\u003c/p\u003e\n\n\u003cp\u003eExit codes are messengers. Summaries are messengers. The agent's own \"done\" is a messenger. Green badges are messengers. The artifact is the diff, the file on disk, the served response body, the row in the database. When a messenger and an artifact disagree, the artifact is right - and a check that only ever reads messengers should be treated as unverified, however green it is.\u003c/p\u003e\n\n\u003cp\u003eThe corollary for guards: \u003cstrong\u003ea green zero is the most dangerous answer a check can give.\u003c/strong\u003e \"Found no violations\" and \"is incapable of finding violations\" produce identical output. Only a negative control separates them.\u003c/p\u003e\n\n\u003ch2\u003e\n \u003ca name=\"count-your-own-ratio-60-seconds\" href=\"#count-your-own-ratio-60-seconds\"\u003e\n \u003c/a\u003e\n Count your own ratio (60 seconds)\n\u003c/h2\u003e\n\n\u003cp\u003eThis is the part you can use without believing me. Drop this in your repo root - it counts test files that read source or state, and how many carry a marked negative control (adjust the marker to your convention):\u003cbr\u003e\n\u003c/p\u003e\n\n\u003cdiv class=\"highlight js-code-highlight\"\u003e\n\u003cpre class=\"highlight javascript\"\u003e\u003ccode\u003e\u003cspan class=\"c1\"\u003e// count-controls.mjs — node count-controls.mjs\u003c/span\u003e\n\u003cspan class=\"k\"\u003eimport\u003c/span\u003e \u003cspan class=\"p\"\u003e{\u003c/span\u003e \u003cspan class=\"nx\"\u003ereadFileSync\u003c/span\u003e\u003cspan class=\"p\"\u003e,\u003c/span\u003e \u003cspan class=\"nx\"\u003ereaddirSync\u003c/span\u003e\u003cspan class=\"p\"\u003e,\u003c/span\u003e \u003cspan class=\"nx\"\u003estatSync\u003c/span\u003e \u003cspan class=\"p\"\u003e}\u003c/span\u003e \u003cspan class=\"k\"\u003efrom\u003c/span\u003e \u003cspan class=\"dl\"\u003e\"\u003c/span\u003e\u003cspan class=\"s2\"\u003enode:fs\u003c/span\u003e\u003cspan class=\"dl\"\u003e\"\u003c/span\u003e\u003cspan class=\"p\"\u003e;\u003c/span\u003e\n\u003cspan class=\"k\"\u003eimport\u003c/span\u003e \u003cspan class=\"p\"\u003e{\u003c/span\u003e \u003cspan class=\"nx\"\u003ejoin\u003c/span\u003e \u003cspan class=\"p\"\u003e}\u003c/span\u003e \u003cspan class=\"k\"\u003efrom\u003c/span\u003e \u003cspan class=\"dl\"\u003e\"\u003c/span\u003e\u003cspan class=\"s2\"\u003enode:path\u003c/span\u003e\u003cspan class=\"dl\"\u003e\"\u003c/span\u003e\u003cspan class=\"p\"\u003e;\u003c/span\u003e\n\u003cspan class=\"kd\"\u003econst\u003c/span\u003e \u003cspan class=\"nx\"\u003efiles\u003c/span\u003e \u003cspan class=\"o\"\u003e=\u003c/span\u003e \u003cspan class=\"p\"\u003e[];\u003c/span\u003e\n\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"kd\"\u003efunction\u003c/span\u003e \u003cspan class=\"nf\"\u003ewalk\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"nx\"\u003ed\u003c/span\u003e\u003cspan class=\"p\"\u003e)\u003c/span\u003e \u003cspan class=\"p\"\u003e{\u003c/span\u003e\n \u003cspan class=\"k\"\u003efor \u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"kd\"\u003econst\u003c/span\u003e \u003cspan class=\"nx\"\u003en\u003c/span\u003e \u003cspan class=\"k\"\u003eof\u003c/span\u003e \u003cspan class=\"nf\"\u003ereaddirSync\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"nx\"\u003ed\u003c/span\u003e\u003cspan class=\"p\"\u003e))\u003c/span\u003e \u003cspan class=\"p\"\u003e{\u003c/span\u003e\n \u003cspan class=\"k\"\u003eif \u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"nx\"\u003en\u003c/span\u003e \u003cspan class=\"o\"\u003e===\u003c/span\u003e \u003cspan class=\"dl\"\u003e\"\u003c/span\u003e\u003cspan class=\"s2\"\u003enode_modules\u003c/span\u003e\u003cspan class=\"dl\"\u003e\"\u003c/span\u003e \u003cspan class=\"o\"\u003e||\u003c/span\u003e \u003cspan class=\"nx\"\u003en\u003c/span\u003e \u003cspan class=\"o\"\u003e===\u003c/span\u003e \u003cspan class=\"dl\"\u003e\"\u003c/span\u003e\u003cspan class=\"s2\"\u003e.git\u003c/span\u003e\u003cspan class=\"dl\"\u003e\"\u003c/span\u003e \u003cspan class=\"o\"\u003e||\u003c/span\u003e \u003cspan class=\"nx\"\u003en\u003c/span\u003e \u003cspan class=\"o\"\u003e===\u003c/span\u003e \u003cspan class=\"dl\"\u003e\"\u003c/span\u003e\u003cspan class=\"s2\"\u003edist\u003c/span\u003e\u003cspan class=\"dl\"\u003e\"\u003c/span\u003e\u003cspan class=\"p\"\u003e)\u003c/span\u003e \u003cspan class=\"k\"\u003econtinue\u003c/span\u003e\u003cspan class=\"p\"\u003e;\u003c/span\u003e\n \u003cspan class=\"kd\"\u003econst\u003c/span\u003e \u003cspan class=\"nx\"\u003ep\u003c/span\u003e \u003cspan class=\"o\"\u003e=\u003c/span\u003e \u003cspan class=\"nf\"\u003ejoin\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"nx\"\u003ed\u003c/span\u003e\u003cspan class=\"p\"\u003e,\u003c/span\u003e \u003cspan class=\"nx\"\u003en\u003c/span\u003e\u003cspan class=\"p\"\u003e);\u003c/span\u003e\n \u003cspan class=\"nf\"\u003estatSync\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"nx\"\u003ep\u003c/span\u003e\u003cspan class=\"p\"\u003e).\u003c/span\u003e\u003cspan class=\"nf\"\u003eisDirectory\u003c/span\u003e\u003cspan class=\"p\"\u003e()\u003c/span\u003e \u003cspan class=\"p\"\u003e?\u003c/span\u003e \u003cspan class=\"nf\"\u003ewalk\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"nx\"\u003ep\u003c/span\u003e\u003cspan class=\"p\"\u003e)\u003c/span\u003e \u003cspan class=\"p\"\u003e:\u003c/span\u003e \u003cspan class=\"sr\"\u003e/\u003c/span\u003e\u003cspan class=\"se\"\u003e\\.\u003c/span\u003e\u003cspan class=\"sr\"\u003etest\u003c/span\u003e\u003cspan class=\"se\"\u003e\\.(\u003c/span\u003e\u003cspan class=\"sr\"\u003et|j\u003c/span\u003e\u003cspan class=\"se\"\u003e)\u003c/span\u003e\u003cspan class=\"sr\"\u003esx\u003c/span\u003e\u003cspan class=\"se\"\u003e?\u003c/span\u003e\u003cspan class=\"sr\"\u003e$/\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003etest\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"nx\"\u003en\u003c/span\u003e\u003cspan class=\"p\"\u003e)\u003c/span\u003e \u003cspan class=\"o\"\u003e\u0026amp;\u0026amp;\u003c/span\u003e \u003cspan class=\"nx\"\u003efiles\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003epush\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"nx\"\u003ep\u003c/span\u003e\u003cspan class=\"p\"\u003e);\u003c/span\u003e\n \u003cspan class=\"p\"\u003e}\u003c/span\u003e\n\u003cspan class=\"p\"\u003e})(\u003c/span\u003e\u003cspan class=\"dl\"\u003e\"\u003c/span\u003e\u003cspan class=\"s2\"\u003e.\u003c/span\u003e\u003cspan class=\"dl\"\u003e\"\u003c/span\u003e\u003cspan class=\"p\"\u003e);\u003c/span\u003e\n\u003cspan class=\"kd\"\u003elet\u003c/span\u003e \u003cspan class=\"nx\"\u003eguards\u003c/span\u003e \u003cspan class=\"o\"\u003e=\u003c/span\u003e \u003cspan class=\"mi\"\u003e0\u003c/span\u003e\u003cspan class=\"p\"\u003e,\u003c/span\u003e \u003cspan class=\"nx\"\u003ewithControl\u003c/span\u003e \u003cspan class=\"o\"\u003e=\u003c/span\u003e \u003cspan class=\"mi\"\u003e0\u003c/span\u003e\u003cspan class=\"p\"\u003e,\u003c/span\u003e \u003cspan class=\"nx\"\u003eprobes\u003c/span\u003e \u003cspan class=\"o\"\u003e=\u003c/span\u003e \u003cspan class=\"mi\"\u003e0\u003c/span\u003e\u003cspan class=\"p\"\u003e;\u003c/span\u003e\n\u003cspan class=\"k\"\u003efor \u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"kd\"\u003econst\u003c/span\u003e \u003cspan class=\"nx\"\u003ef\u003c/span\u003e \u003cspan class=\"k\"\u003eof\u003c/span\u003e \u003cspan class=\"nx\"\u003efiles\u003c/span\u003e\u003cspan class=\"p\"\u003e)\u003c/span\u003e \u003cspan class=\"p\"\u003e{\u003c/span\u003e\n \u003cspan class=\"kd\"\u003econst\u003c/span\u003e \u003cspan class=\"nx\"\u003et\u003c/span\u003e \u003cspan class=\"o\"\u003e=\u003c/span\u003e \u003cspan class=\"nf\"\u003ereadFileSync\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"nx\"\u003ef\u003c/span\u003e\u003cspan class=\"p\"\u003e,\u003c/span\u003e \u003cspan class=\"dl\"\u003e\"\u003c/span\u003e\u003cspan class=\"s2\"\u003eutf8\u003c/span\u003e\u003cspan class=\"dl\"\u003e\"\u003c/span\u003e\u003cspan class=\"p\"\u003e);\u003c/span\u003e\n \u003cspan class=\"k\"\u003eif \u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"o\"\u003e!\u003c/span\u003e\u003cspan class=\"sr\"\u003e/readFileSync|readdirSync|execSync/\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003etest\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"nx\"\u003et\u003c/span\u003e\u003cspan class=\"p\"\u003e))\u003c/span\u003e \u003cspan class=\"k\"\u003econtinue\u003c/span\u003e\u003cspan class=\"p\"\u003e;\u003c/span\u003e \u003cspan class=\"c1\"\u003e// \"reads state\" proxy\u003c/span\u003e\n \u003cspan class=\"nx\"\u003eguards\u003c/span\u003e\u003cspan class=\"o\"\u003e++\u003c/span\u003e\u003cspan class=\"p\"\u003e;\u003c/span\u003e\n \u003cspan class=\"kd\"\u003econst\u003c/span\u003e \u003cspan class=\"nx\"\u003en\u003c/span\u003e \u003cspan class=\"o\"\u003e=\u003c/span\u003e \u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"nx\"\u003et\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003ematch\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"sr\"\u003e/KONTROLLE|negative.control|can.\u003c/span\u003e\u003cspan class=\"se\"\u003e?\u003c/span\u003e\u003cspan class=\"sr\"\u003enot.\u003c/span\u003e\u003cspan class=\"se\"\u003e?\u003c/span\u003e\u003cspan class=\"sr\"\u003efind/gi\u003c/span\u003e\u003cspan class=\"p\"\u003e)\u003c/span\u003e \u003cspan class=\"o\"\u003e??\u003c/span\u003e \u003cspan class=\"p\"\u003e[]).\u003c/span\u003e\u003cspan class=\"nx\"\u003elength\u003c/span\u003e\u003cspan class=\"p\"\u003e;\u003c/span\u003e\n \u003cspan class=\"k\"\u003eif \u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"nx\"\u003en\u003c/span\u003e\u003cspan class=\"p\"\u003e)\u003c/span\u003e \u003cspan class=\"nx\"\u003ewithControl\u003c/span\u003e\u003cspan class=\"o\"\u003e++\u003c/span\u003e\u003cspan class=\"p\"\u003e;\u003c/span\u003e\n \u003cspan class=\"nx\"\u003eprobes\u003c/span\u003e \u003cspan class=\"o\"\u003e+=\u003c/span\u003e \u003cspan class=\"nx\"\u003en\u003c/span\u003e\u003cspan class=\"p\"\u003e;\u003c/span\u003e\n\u003cspan class=\"p\"\u003e}\u003c/span\u003e\n\u003cspan class=\"nx\"\u003econsole\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003elog\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"s2\"\u003e`\u003c/span\u003e\u003cspan class=\"p\"\u003e${\u003c/span\u003e\u003cspan class=\"nx\"\u003eguards\u003c/span\u003e\u003cspan class=\"p\"\u003e}\u003c/span\u003e\u003cspan class=\"s2\"\u003e conclusion-bearing guard files · \u003c/span\u003e\u003cspan class=\"p\"\u003e${\u003c/span\u003e\u003cspan class=\"nx\"\u003ewithControl\u003c/span\u003e\u003cspan class=\"p\"\u003e}\u003c/span\u003e\u003cspan class=\"s2\"\u003e with a negative control (\u003c/span\u003e\u003cspan class=\"p\"\u003e${\u003c/span\u003e\u003cspan class=\"nx\"\u003eguards\u003c/span\u003e \u003cspan class=\"p\"\u003e?\u003c/span\u003e \u003cspan class=\"nb\"\u003eMath\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003eround\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"mi\"\u003e100\u003c/span\u003e \u003cspan class=\"o\"\u003e*\u003c/span\u003e \u003cspan class=\"nx\"\u003ewithControl\u003c/span\u003e \u003cspan class=\"o\"\u003e/\u003c/span\u003e \u003cspan class=\"nx\"\u003eguards\u003c/span\u003e\u003cspan class=\"p\"\u003e)\u003c/span\u003e \u003cspan class=\"p\"\u003e:\u003c/span\u003e \u003cspan class=\"mi\"\u003e0\u003c/span\u003e\u003cspan class=\"p\"\u003e}\u003c/span\u003e\u003cspan class=\"s2\"\u003e %) · \u003c/span\u003e\u003cspan class=\"p\"\u003e${\u003c/span\u003e\u003cspan class=\"nx\"\u003eprobes\u003c/span\u003e\u003cspan class=\"p\"\u003e}\u003c/span\u003e\u003cspan class=\"s2\"\u003e probes`\u003c/span\u003e\u003cspan class=\"p\"\u003e);\u003c/span\u003e\n\u003c/code\u003e\u003c/pre\u003e\n\u003cdiv class=\"highlight__panel js-actions-panel\"\u003e\n\u003cdiv class=\"highlight__panel-action js-fullscreen-code-action\"\u003e\n \u003csvg xmlns=\"http://www.w3.org/2000/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"\u003e\u003ctitle\u003eEnter fullscreen mode\u003c/title\u003e\n \u003cpath d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\u003e\u003c/path\u003e\n\u003c/svg\u003e\n\n \u003csvg xmlns=\"http://www.w3.org/2000/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"\u003e\u003ctitle\u003eExit fullscreen mode\u003c/title\u003e\n \u003cpath d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\u003e\u003c/path\u003e\n\u003c/svg\u003e\n\n\u003c/div\u003e\n\u003c/div\u003e\n\u003c/div\u003e\n\n\n\n\u003cp\u003eIf your number is above 30 %, I'd genuinely like to know how you got there - that's the discussion I'm hoping for below.\u003c/p\u003e\n\n\u003ch2\u003e\n \u003ca name=\"where-i-was-the-punchline-twice-while-writing-this\" href=\"#where-i-was-the-punchline-twice-while-writing-this\"\u003e\n \u003c/a\u003e\n Where I was the punchline, twice, while writing this\n\u003c/h2\u003e\n\n\u003cp\u003eRule 2 of writing these posts is correcting yourself unprompted, so:\u003c/p\u003e\n\n\u003cp\u003eWhile building the feature this article's data comes from, my equivalence test failed by \u003cem\u003eexactly\u003c/em\u003e 0.25 - and the bug was in \u003cstrong\u003emy test\u003c/strong\u003e, not the code: min-max spreading turns a column of zeros into a column of 0.5s and adds a constant. I had built a probe that answered a different question than the one asked, in the middle of measuring exactly that failure class.\u003c/p\u003e\n\n\u003cp\u003eAnd one push in that same hour went out with a red test - because \u003ccode\u003enpm test | grep\u003c/code\u003e replaces the test's exit code with grep's. My pipeline read a messenger. The artifact - the failing test - sat right there.\u003c/p\u003e\n\n\u003cp\u003eThe person telling you to test your reviewers failed to test his reviewer, twice, in one evening. That's not irony. That's the base rate, and it's why conventions beat discipline.\u003c/p\u003e\n\n\u003ch2\u003e\n \u003ca name=\"what-this-does-not-prove\" href=\"#what-this-does-not-prove\"\u003e\n \u003c/a\u003e\n What this does not prove\n\u003c/h2\u003e\n\n\u003cp\u003eOne developer, three repositories, one week - this is a case series, not a sample. The 11 % is marker-based and approximate. And I have not shown that raising falsifiability coverage improves outcomes downstream; I've shown that at 11 % I couldn't distinguish my working guards from my decorative ones. Whether the number that matters is 30 % or 80 %, I don't know yet - we're raising ours and measuring as we go.\u003c/p\u003e\n\n\u003cp\u003eThere's also a fair objection: negative controls are themselves tests that can rot. True. But a control that rots fails \u003cem\u003eloudly\u003c/em\u003e the next time the guard changes - that's the asymmetry that makes them worth writing.\u003c/p\u003e\n\n\u003cp\u003e\u003cem\u003eSo: what's your ratio? And more interesting - what's the greenest check in your pipeline that you now suspect has never been able to fail?\u003c/em\u003e\u003c/p\u003e\n\n\n\u003chr\u003e\n\n\u003cp\u003eI build \u003cstrong\u003ecachly\u003c/strong\u003e — memory for AI coding assistants, over MCP. ChatGPT and Claude remember your conversations. cachly remembers \u003cstrong\u003eyour system\u003c/strong\u003e: the bug you fixed, why you chose Postgres, the deploy step that always breaks — and which earlier decision it contradicts. Every assistant you use reads the same memory, and every lesson carries the name of whoever learned it — so nobody has to learn it twice.\u003c/p\u003e\n\n\u003cp\u003eFree tier, hosted in the EU: \u003ca href=\"https://cachly.dev?utm_source=devto\u0026amp;utm_medium=article\u0026amp;utm_content=your-test-suite-is-green-can-it-even-fail\" target=\"_blank\" rel=\"noopener noreferrer\"\u003ecachly.dev\u003c/a\u003e\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Heinrich Neb","url":"https://dev.to/heinrichneb"},"datePublished":"2026-08-24T23:23:06Z","dateModified":"2026-08-24T23:23:06Z","url":"https://dev.to/heinrichneb/ai-promoted-every-developer-to-reviewer-nobody-tested-the-reviewer-m4h","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/CommentAction","userInteractionCount":40},{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":38}],"comment":[{"@type":"Comment","@id":"#comment-1610589","text":"\u003cp\u003eThe \"500 inside 4258 of 5000 quota points\" example is the same shape of bug I shipped without realizing it. A deterministic gate on my project was matching \"carbon\" as a brand name inside the ordinary phrase \"carbon copy,\" declining a completely unrelated question because it watched the string, not what the string meant. You caught it, not a test, because I didn't have one.\u003c/p\u003e\n\n\u003cp\u003eThe regression suite I added afterward has negative controls now, tests confirming genuine brand mentions still get caught, not just that the false ones stop firing. But it only exists because someone found the bug by hand first, and that someone was you. Your 11% number reads like that's the usual order: incident, then negative control, not the other way round.\u003c/p\u003e\n\n\u003cp\u003eGenuinely curious whether your data can actually tell the difference between teams that built the negative control proactively and teams that built it the way I just did.\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Daniel Nwaneri","url":"https://dev.to/dannwaneri"},"datePublished":"2026-08-25T12:04:52Z","dateModified":"2026-08-25T12:04:52Z","url":"https://dev.to/dannwaneri/comment/3dgdj","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":2}],"comment":[{"@type":"Comment","@id":"#comment-1610921","text":"\u003cp\u003eHonest answer to your genuinely curious question: no - our data cannot tell those two teams apart, and I want to be precise about why. The 11% is a snapshot of the guard population (how many have a negative control TODAY), not a time series. It doesn't see when a control was born or what prompted it. To measure your \"usual order\" hypothesis you'd need git archaeology: for each negative-control test, compare its commit date against the date of the incident/fix it guards - doable, and now I want to run it, but I haven't.\u003c/p\u003e\n\n\u003cp\u003eAnecdotally, on our own codebase the order is almost always yours: incident first, control second. Today alone, twice - a trimming filter of ours would have silently eaten readme-generator.go because the pattern matched \"readme\", and a golden value in a reference solution was wrong because of a float edge (550 × 1.19 = 654.4999...). Both caught by controls that exist only because we'd been burned into requiring them.\u003c/p\u003e\n\n\u003cp\u003eWhich is the one structural fix I know for the ordering problem: make the negative control an ADMISSION rule instead of a reaction. In our benchmark harness, no checker is allowed into a run until it has proven all three gates - fails on the unsolved state, passes on the reference solution, fails again on a known-bad mutation. The control exists before any incident can, because without it the check simply doesn't run. Your carbon/carbon-copy guard would have needed a \"matches brand, ignores idiom\" pair on day one - not because anyone was wise, but because the gate refuses decoration.\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Heinrich Neb","url":"https://dev.to/heinrichneb"},"datePublished":"2026-08-25T20:52:08Z","dateModified":"2026-08-25T20:52:08Z","url":"https://dev.to/heinrichneb/comment/3dh0d","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":2}],"parentItem":{"@type":"Comment","@id":"#comment-1610589"}}]},{"@type":"Comment","@id":"#comment-1610753","text":"\u003cp\u003eThe 89% number is brutal and I recognize it. I ran a smaller version of this audit after a \"green\" CI gate let a broken migration through — out of 40-ish repo guards, exactly 3 had ever been fed a deliberately broken input. The rest were tautologies wearing a badge.\u003c/p\u003e\n\n\u003cp\u003eThe \u003ccode\u003eKONTROLLE:\u003c/code\u003e naming convention is the part I'm stealing. Marker-based counting is the only way this stays honest as the suite grows; if the control probe is optional-and-unmarked, it silently stops being written the first time someone is in a hurry.\u003c/p\u003e\n\n\u003cp\u003eOne thing I'd add to the taxonomy: guards that \u003cem\u003ecan\u003c/em\u003e fail but only on inputs that no longer occur. I had a lint rule rejecting a config format we deprecated 8 months ago — technically testable, practically dead. Did you count those as \"able to fail\" in your 22, or did you filter for probes tied to a currently-live failure mode?\u003c/p\u003e\n\n\u003cp\u003eCurious how you handle the pushback when a negative control itself becomes the brittle part of the suite — probing a guard against a known-bad input that's \u003cem\u003etoo\u003c/em\u003e known-bad (nobody would ever actually write it) feels like testing the wrong thing. Where did you draw that line?\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Niuniu Ox","url":"https://dev.to/byteox2"},"datePublished":"2026-08-25T15:30:13Z","dateModified":"2026-08-25T15:30:13Z","url":"https://dev.to/byteox2/comment/3dgk1","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":2}],"comment":[{"@type":"Comment","@id":"#comment-1610930","text":"\u003cp\u003e3 of 40 - thank you for counting before commenting; may I add your 92.5% next to our 89% when I write the follow-up? Your two questions, honestly:\u003c/p\u003e\n\n\u003cp\u003eDead guards: you caught a real gap. Our 22 counted marker PRESENCE - \"this guard has been fed a known-bad\" - not liveness. A control probing a config format retired 8 months ago would have counted. Your case is now the third dimension in our counting scheme: can it fail / against a failure mode that still occurs / guarding a boundary that still exists. (Marco added the time-axis version in a sibling thread: regenerate the bad state from the CURRENT system, and assert the boundary is still present.)\u003c/p\u003e\n\n\u003cp\u003eWhere we draw the too-known-bad line: the known-bad must be the mistake a hurried human or agent would actually make, not a constructed absurdity. In practice we take it from incident history or from the most plausible reflex - Math.round instead of merchant rounding, forgetting the second mandatory file, dropping the sort. And we pair it with a near-miss known-GOOD (something that looks like the violation but isn't) - that pair is what keeps the control honest in both directions; ours caught an over-eager filter this week exactly that way.\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Heinrich Neb","url":"https://dev.to/heinrichneb"},"datePublished":"2026-08-25T21:02:20Z","dateModified":"2026-08-25T21:02:20Z","url":"https://dev.to/heinrichneb/comment/3dh0m","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":1}],"parentItem":{"@type":"Comment","@id":"#comment-1610753"}}]},{"@type":"Comment","@id":"#comment-1610674","text":"\u003cp\u003eThe 89% figure stuck with me because I keep seeing the same shape with coding agents. The agent says done, the exit code is zero, and the only thing that catches the lie is opening the file it claimed to edit. Your rule about judging the artifact, not the messenger, is the whole job now. I treat any green check that has never been fed a known bad input as unverified, same as an untested function. Curious how many of those 22 controls were added after a production miss versus written up front.\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Edu Peralta","url":"https://dev.to/eduzsh"},"datePublished":"2026-08-25T13:41:24Z","dateModified":"2026-08-25T13:41:24Z","url":"https://dev.to/eduzsh/comment/3dgh0","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":2}],"comment":[{"@type":"Comment","@id":"#comment-1610928","text":"\u003cp\u003eHonest answer: I can't tell you yet, and the reason is itself part of the finding. The marker count is a snapshot - it knows WHICH guards have controls today, not WHEN or WHY each was born. Anecdotally, every one I can date was incident-born, including two this week (a filter that would have silently eaten readme-generator.go, and a golden value that was wrong because of a float edge). Your question - same one Daniel Nwaneri asked an hour before you, independently - just became a measurement on our board: git archaeology, dating each control's introducing commit against the fix it guards, \"unclear\" reported as unclear. I'll ping this thread with the split when it's run. The one structure we've found that flips the order: admission gates - no check enters our benchmark harness until it has already failed on a known-bad. There, the control exists before any incident can.\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Heinrich Neb","url":"https://dev.to/heinrichneb"},"datePublished":"2026-08-25T21:01:05Z","dateModified":"2026-08-25T21:01:05Z","url":"https://dev.to/heinrichneb/comment/3dh0k","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":1}],"parentItem":{"@type":"Comment","@id":"#comment-1610674"}}]},{"@type":"Comment","@id":"#comment-1610910","text":"\u003cp\u003eThe 11 percent number would sting less if I did not immediately recognize all three of your failure shapes from my own repos, especially the classifier matching the \"500\" inside \"5000 quota points\". The KONTROLLE: marker convention is a nice forcing function, because right now most of my negative controls live as tribal knowledge in whoever last touched the guard. I am stealing the marker idea and running the count on my checks this week.\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Kartik N V J K","url":"https://dev.to/kartik-nvjk"},"datePublished":"2026-08-25T20:32:14Z","dateModified":"2026-08-25T20:32:14Z","url":"https://dev.to/kartik-nvjk/comment/3dh02","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":2}],"comment":[{"@type":"Comment","@id":"#comment-1610932","text":"\u003cp\u003eStealing the marker is its entire purpose - and \"negative controls living as tribal knowledge in whoever last touched the guard\" is a better one-line justification for it than anything in my article. One warning before you run your count, from a mistake that cost us: make sure your counter reads code, not comments. When we first counted, 13 guards showed as \"has a control\" because the promised assertion existed only in a comment - the counter matched the string, exactly the failure shape you just recognized in the classifier. Strip comments first, then count. And please post your number when you have it - we're at 89% (ours) and 92.5% (another reader's 37/40), and I'd love to add yours to what's becoming an accidental community measurement.\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Heinrich Neb","url":"https://dev.to/heinrichneb"},"datePublished":"2026-08-25T21:03:59Z","dateModified":"2026-08-25T21:03:59Z","url":"https://dev.to/heinrichneb/comment/3dh0o","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":1}],"parentItem":{"@type":"Comment","@id":"#comment-1610910"}}]},{"@type":"Comment","@id":"#comment-1610637","text":"\u003cp\u003eReviewing (checking) (testing) can grow very rapidly. Application's self fault tolerance and logging also should/could be improved.\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Karlis","url":"https://dev.to/acaciaman"},"datePublished":"2026-08-25T13:08:20Z","dateModified":"2026-08-25T13:08:20Z","url":"https://dev.to/acaciaman/comment/3dgff","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":2}],"comment":[{"@type":"Comment","@id":"#comment-1610926","text":"\u003cp\u003eAgreed on both - with one wrinkle we learned the hard way: logging is itself a guard that can go green-and-blind. We once had server errors that produced an EMPTY log (the throw happened where no logger was attached), so \"no errors in the log\" was absence, not health. Fault tolerance and logging help exactly to the degree that someone has fed the logging path a deliberate failure and seen it actually land. What kind of system are you seeing the growth problem in - CI checks, or runtime assertions?\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Heinrich Neb","url":"https://dev.to/heinrichneb"},"datePublished":"2026-08-25T20:59:28Z","dateModified":"2026-08-25T20:59:28Z","url":"https://dev.to/heinrichneb/comment/3dh0i","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":1}],"parentItem":{"@type":"Comment","@id":"#comment-1610637"}}]},{"@type":"Comment","@id":"#comment-1610298","text":"\u003cp\u003eThe 89% number is brutal and probably understated. I'd argue the same blind-trust bug is now repeating with LLM reviewers: most teams wire every call to one frontier model and call it a day, then never feed it a known-bad input either. The fix isn't a bigger model — it's routing by scenario. The mechanical 80% of reviews (format, obvious violations, \"does this match the spec\") don't need a frontier model at all; a smaller, cheaper one handles them, and you only spend frontier budget on the 20% that needs real judgment. That gets you the negative-control discipline you're describing \u003cem\u003eand\u003c/em\u003e a 70%+ cost drop, because the expensive model is finally used where it can actually fail differently. Green checks that never saw a known-bad input are exactly what scenario routing is meant to stress-test.\u003c/p\u003e\n\n","author":{"@type":"Person","name":"TokenLat","url":"https://dev.to/tokenlat"},"datePublished":"2026-08-25T07:16:04Z","dateModified":"2026-08-25T07:16:04Z","url":"https://dev.to/tokenlat/comment/3dg2e","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":2}],"comment":[{"@type":"Comment","@id":"#comment-1610311","text":"\u003cp\u003eThe extension to LLM reviewers is the right next domino: a model-based check that never saw a known-bad input is my 89 % with a bigger invoice. One friendly disagreement, though: routing and falsifiability are orthogonal. Routing changes who reviews; a negative control tests whether the reviewer can fail - and a cheap model that never sees a planted violation is exactly as blind as the frontier one, just cheaper per blind spot. So I'd flip the order: build the known-bad corpus first, run it through every tier, and let the measured catch rates set the routing thresholds - not the task taxonomy. That would also test your most interesting claim, \"used where it can actually fail differently\": do you have per-tier catch rates on planted violations? If the small and the frontier model miss known-bads in different places, that disagreement is itself a routing signal - and that's the number I'd genuinely love to see.\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Heinrich Neb","url":"https://dev.to/heinrichneb"},"datePublished":"2026-08-25T07:26:52Z","dateModified":"2026-08-25T07:26:52Z","url":"https://dev.to/heinrichneb/comment/3dg31","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":2}],"parentItem":{"@type":"Comment","@id":"#comment-1610298"}}]},{"@type":"Comment","@id":"#comment-1610033","text":"\u003cp\u003eThis framing is sharp. We invested heavily in 'AI review' but forgot that review is a feedback loop: the reviewer learns the codebase, the team, and the failure modes. If we don't validate that loop, we just speed up the approval of bad changes.\u003c/p\u003e\n\n","author":{"@type":"Person","name":"kevinbai","url":"https://dev.to/kevinbai"},"datePublished":"2026-08-25T00:08:22Z","dateModified":"2026-08-25T00:08:22Z","url":"https://dev.to/kevinbai/comment/3dfi9","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":2}],"comment":[{"@type":"Comment","@id":"#comment-1610039","text":"\u003cp\u003eThe loop framing sharpens something I only half-said: a negative control validates the reviewer at a point in time - it doesn't make the reviewer learn. I counted one week of my own agent-assisted failures and it was the same handful of classes recurring: four imports that silently started a main(), five checks that matched wording instead of meaning. A reviewer that had learned from Monday would have rejected Thursday's change. So the loop needs both halves: falsifiable guards (can it reject?) and accumulated failure modes (does it know what to reject here?). Most setups I've seen have neither wired in. What does validating the loop look like concretely on your team - do you measure whether review findings recur?\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Heinrich Neb","url":"https://dev.to/heinrichneb"},"datePublished":"2026-08-25T00:14:46Z","dateModified":"2026-08-25T00:14:46Z","url":"https://dev.to/heinrichneb/comment/3dfif","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":1}],"parentItem":{"@type":"Comment","@id":"#comment-1610033"}}]},{"@type":"Comment","@id":"#comment-1610347","text":"\u003cp\u003eAnd that's exactly why I say AI is a senior dev's tool, not a juniors. If you produced garbage code before, AI just makes the pile bigger. Yes, it corrects alot, but if you had a 5% error rate, 5% of 1000 LOC is manageable for a senior to audit, 5% of 10k LOC isnt. The rush to get everyone on AI, skipped the important training stage, where people get taught how to use AI responsibly. Because I bet none of you got training on how to use AI-assistants? And that's exactly the problem.\u003c/p\u003e\n\n","author":{"@type":"Person","name":"UnitBuilds","url":"https://dev.to/unitbuilds"},"datePublished":"2026-08-25T08:04:56Z","dateModified":"2026-08-25T08:04:56Z","url":"https://dev.to/unitbuilds/comment/3dg4b","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":2}],"comment":[{"@type":"Comment","@id":"#comment-1610398","text":"\u003cp\u003eHalf agree - and the half I'd push back on is where the fix lives. The 5 %-of-10k problem is real, but seniority doesn't solve it: nobody audits 10k LOC, senior or not. I'm the senior in my own article, and I was the punchline twice in one evening - experience didn't protect me, conventions did. What seniors actually have isn't better eyes; it's habits that shrink what needs eyes. They don't audit the pile, they audit the gates the pile must pass. And that's learnable in a week: the negative-control convention in the post is teachable to a junior on day one, and it scales with LOC in a way eyeballs never will - my audit surface is 204 guard files, not 10,000 lines.\u003c/p\u003e\n\n\u003cp\u003eOn training: agreed that nobody got it - but I'd sharpen what the missing course actually is. Not \"how to prompt.\" It's \"how to review\" - and that course never existed for humans either. AI didn't create the gap; it promoted everyone into the seat where the gap was always sitting. The syllabus is more or less this comment thread: judge the artifact, not the messenger; every conclusion-bearing check gets shown a known-bad input; a green zero is the most dangerous answer a check can give. Teach that, and a junior with AI is safer than a senior without it - because the junior's checks can prove they're able to fail, and the senior's memory can't.\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Heinrich Neb","url":"https://dev.to/heinrichneb"},"datePublished":"2026-08-25T09:14:18Z","dateModified":"2026-08-25T09:14:18Z","url":"https://dev.to/heinrichneb/comment/3dg6a","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":2}],"parentItem":{"@type":"Comment","@id":"#comment-1610347"}}]},{"@type":"Comment","@id":"#comment-1610214","text":"\u003cp\u003eMy answer to your last question is a publish check that read one field off an API response. The field is not in that endpoint's representation at all: three variants of the request, with the key, without it, and without the vendor accept header, all came back with no \u003ccode\u003epublished\u003c/code\u003e key, and \u003ccode\u003e.get()\u003c/code\u003e on a missing key hands you \u003ccode\u003eNone\u003c/code\u003e, which the guard scored as \"not published\". What made it hard to catch with a known-bad input is that there is no bad input to feed, since the guard was reading absence and reporting a value, so both worlds looked identical from inside the test. The control that separated them was asserting key presence separately from key value, and then moving the real check onto a signal that endpoint does carry, an unauthenticated \u003ccode\u003eGET\u003c/code\u003e returning 200, because a draft fetched by id returns 404 even with the owner's key.\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Vinh Nguyen","url":"https://dev.to/vinhnguyenthanhdn"},"datePublished":"2026-08-25T05:46:24Z","dateModified":"2026-08-25T05:46:24Z","url":"https://dev.to/vinhnguyenthanhdn/comment/3dfp8","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":2}],"comment":[{"@type":"Comment","@id":"#comment-1610303","text":"\u003cp\u003eThis is the sharpest reply the article has received, because it names a class my counter cannot see: guards that read absence. My negative-control definition quietly assumes a known-bad input exists - your check had no bad input to construct, because .get() on a missing key manufactures a value out of absence, and both worlds look identical from inside the test. \"Assert key presence separately from key value\" is the move that makes the class testable at all: it converts absence back from a value into a state. And relocating the check onto a signal the endpoint actually carries - unauthenticated GET, 200 vs. 404 - is judge-the-artifact in its purest form.\u003c/p\u003e\n\n\u003cp\u003eI'm adding \"absence-readers\" as a fourth green-and-blind shape next to the three messenger-readers in the article; with your permission I'll cite this comment when I write it up. One question back: after the fix, did you add a probe asserting the 404-for-drafts behavior stays - or is the vendor's draft semantics now the new untested assumption underneath the guard?\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Heinrich Neb","url":"https://dev.to/heinrichneb"},"datePublished":"2026-08-25T07:19:37Z","dateModified":"2026-08-25T07:19:37Z","url":"https://dev.to/heinrichneb/comment/3dg2j","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":1}],"parentItem":{"@type":"Comment","@id":"#comment-1610214"}}]},{"@type":"Comment","@id":"#comment-1610306","text":"\u003cp\u003eWhat I find most interesting here is that the hard part isn't making the security check fail when something is obviously broken. It's proving that the check still has a meaningful boundary to test against as the system evolves.\u003c/p\u003e\n\n\u003cp\u003eA green test can survive a broken invariant, a stale fixture, or even a defence that no longer has anything to protect. That's why I increasingly like the idea of treating negative controls as first-class engineering artifacts: if we can't deliberately cross the failure boundary and make the test go red, we should question what the test is actually proving.\u003c/p\u003e\n\n\u003cp\u003eThe “run, don't trust the comment” conclusion is probably the part I'll take away from this thread. 🔐\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Marco","url":"https://dev.to/mk023"},"datePublished":"2026-08-25T07:22:31Z","dateModified":"2026-08-25T07:22:31Z","url":"https://dev.to/mk023/comment/3dg2m","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":2}],"comment":[{"@type":"Comment","@id":"#comment-1610391","text":"\u003cp\u003eThe evolution point deserves its own article, because it's the time-axis my count ignores: a negative control proves the guard can fail today - nothing proves the boundary it crosses still exists next quarter. We hit the pure form of this: a guard with a hard-coded threshold that kept validating a world that had moved on. It was green every day, faithfully guarding the past. Two practices that have helped since: (1) generate the planted-bad from the current system at test time instead of storing it as a fixture - a frozen bad input is a frozen boundary, and it rots at exactly the rate of the system around it; (2) pair the crossing-control with a freshness assertion on the boundary itself - my favorite example this week is a test that fails the moment a corpus and its denylist stop matching exactly: it doesn't test the crossing, it tests that there's still a fence where the map says one is. And \"run, don't trust the comment\" cuts both ways: your prompt-injection piece is the sibling failure - the test passed while the attack worked, which is this thread's 89 % wearing a security label.\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Heinrich Neb","url":"https://dev.to/heinrichneb"},"datePublished":"2026-08-25T09:06:20Z","dateModified":"2026-08-25T09:06:20Z","url":"https://dev.to/heinrichneb/comment/3dg63","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":2}],"parentItem":{"@type":"Comment","@id":"#comment-1610306"}}]}]}} if (/Twitter for (iPhone|iPad|Android)/i.test(navigator.userAgent)) { document.documentElement.classList.add('is-twitter-in-app'); } Add reaction Like Unicorn Exploding Head Raised Hands Fire Jump to Comments Save Boost More... Copy link Copy link Copied to Clipboard Share to X Share to LinkedIn Share to Facebook Share to Mastodon Share Post via... Report Abuse try { if(localStorage) { let currentUser = localStorage.getItem('current_user'); if (currentUser) { currentUser = JSON.parse(currentUser); if (currentUser.id === 4074633) { document.getElementById('article-show-container').classList.add('current-user-is-article-author'); } } } } catch (e) { console.error(e); } Heinrich Neb Posted on Aug 24 • Originally published at cachly.dev AI promoted every developer to reviewer. Nobody tested the reviewer. # testing # ai # productivity # discuss 89 percent of guardrails never tested for failure I wanted to disagree with 'AI made me a worse reviewer' from Michael Amachree ( @dev_michael ) . Instead I counted 204 of my own guards — and 89 % of them have never been asked to prove they can fail. Michael wrote something that I couldn't put down: AI didn't make me a worse coder, it made me a worse reviewer. Here is the number, and it's worse than his thesis: of the 204 automated checks in my repositories that draw a conclusion, only 22 can prove they are able to fail. That's 11 %. The other 89 % have never once been shown a known-bad input. They are green. Whether they are green because everything is fine, or green because they are incapable of finding anything - I could not have told you last week. And I'm the person who wrote them. What I actually counted First the definition, so you can reject it or reuse it. A conclusion-bearing guard is any test that reads source code, config, or system state and asserts a claim about it. Not "does this function return 4" - but "no workflow downloads its cache over the network", "every page passes the same quarter filter", "this feature flag matches the deployed spec". The tests that stand in for a human reviewer. A negative control is a probe that feeds that guard a known-bad input and asserts it gets rejected for the expected reason . Our convention marks them KONTROLLE: in the test name. Counting is mechanical: 204 guard files across three repositories, 22 with at least one control probe, 54 probes total. The counter is a proxy - marker-based, so unmarked controls and false-positive guard files put the true number at plus or minus a few points. The shape survives any correction: most of my reviewers have never been reviewed. Three green-and-blind checks, one ordinary week This isn't theoretical. All three of these happened to me in the last seven days, in production tooling. The deploy gate that died of its own medicine. A pipeline step existed specifically to catch a silent failure mode - a missing tool falling back to an empty result. It called node -e to parse a health response. The deploy runner has no Node. Six consecutive deployments failed with exit 127 - the check against missing tools failed on a missing tool , and nothing shipped for six hours. The step had been green in review because nobody had ever run it where it actually runs. The harvester that threw away its own work. An autonomous job collected data from public repositories and judged each run by exit code. One run wrote seven perfectly good records, then hit a non-fatal warning and exited non-zero. The machine booked its own completed work as "failed, retry later" - because interrupted-with-partial-results had no representation, only success and failure. We caught it because the result file was sitting on disk right next to the exit code that denied its existence. The pattern that matched the wrong 500. An error classifier looked for server errors with the pattern 50[024] - anywhere in the output. It matched the "500" inside "4258 of 5000 quota points remaining" and classified a successful run as a server failure. Every field it read was real. It was answering a different question than the one asked. Three different systems. One shape: the check watched a messenger - an exit code, a pattern, a status - while the artifact that mattered told a different story. What this has to do with AI making you a worse reviewer Here's where I think Michael's post lands harder than he says. AI moved my job. I used to spend most of my day producing artifacts and a little of it verifying them. Now an agent produces most of the artifacts, and my job is verification. Which means my real codebase - the one my judgment actually ships through - is those 204 guards. And that codebase is held to a standard I would reject in application code. No test coverage (11 %). No review of the reviewer. Green as the default state, silence booked as success. When Michael says AI made him a worse reviewer, I'd sharpen it: AI promoted us all to reviewers, and none of us tested the reviewer. The model isn't the weak link. The unfalsifiable green checkmark is. The rule that survived the week Everything above collapses into one sentence we now apply mechanically: Judge the artifact, not the messenger. Exit codes are messengers. Summaries are messengers. The agent's own "done" is a messenger. Green badges are messengers. The artifact is the diff, the file on disk, the served response body, the row in the database. When a messenger and an artifact disagree, the artifact is right - and a check that only ever reads messengers should be treated as unverified, however green it is. The corollary for guards: a green zero is the most dangerous answer a check can give. "Found no violations" and "is incapable of finding violations" produce identical output. Only a negative control separates them. Count your own ratio (60 seconds) This is the part you can use without believing me. Drop this in your repo root - it counts test files that read source or state, and how many carry a marked negative control (adjust the marker to your convention): // count-controls.mjs — node count-controls.mjs import { readFileSync , readdirSync , statSync } from " node:fs " ; import { join } from " node:path " ; const files = []; ( function walk ( d ) { for ( const n of readdirSync ( d )) { if ( n === " node_modules " || n === " .git " || n === " dist " ) continue ; const p = join ( d , n ); statSync ( p ). isDirectory () ? walk ( p ) : / \. test \.( t|j ) sx ? $/ . test ( n ) && files . push ( p ); } })( " . " ); let guards = 0 , withControl = 0 , probes = 0 ; for ( const f of files ) { const t = readFileSync ( f , " utf8 " ); if ( ! /readFileSync|readdirSync|execSync/ . test ( t )) continue ; // "reads state" proxy guards ++ ; const n = ( t . match ( /KONTROLLE|negative.control|can. ? not. ? find/gi ) ?? []). length ; if ( n ) withControl ++ ; probes += n ; } console . log ( ` ${ guards } conclusion-bearing guard files · ${ withControl } with a negative control ( ${ guards ? Math . round ( 100 * withControl / guards ) : 0 } %) · ${ probes } probes` ); Enter fullscreen mode Exit fullscreen mode If your number is above 30 %, I'd genuinely like to know how you got there - that's the discussion I'm hoping for below. Where I was the punchline, twice, while writing this Rule 2 of writing these posts is correcting yourself unprompted, so: While building the feature this article's data comes from, my equivalence test failed by exactly 0.25 - and the bug was in my test , not the code: min-max spreading turns a column of zeros into a column of 0.5s and adds a constant. I had built a probe that answered a different question than the one asked, in the middle of measuring exactly that failure class. And one push in that same hour went out with a red test - because npm test | grep replaces the test's exit code with grep's. My pipeline read a messenger. The artifact - the failing test - sat right there. The person telling you to test your reviewers failed to test his reviewer, twice, in one evening. That's not irony. That's the base rate, and it's why conventions beat discipline. What this does not prove One developer, three repositories, one week - this is a case series, not a sample. The 11 % is marker-based and approximate. And I have not shown that raising falsifiability coverage improves outcomes downstream; I've shown that at 11 % I couldn't distinguish my working guards from my decorative ones. Whether the number that matters is 30 % or 80 %, I don't know yet - we're raising ours and measuring as we go. There's also a fair objection: negative controls are themselves tests that can rot. True. But a control that rots fails loudly the next time the guard changes - that's the asymmetry that makes them worth writing. So: what's your ratio? And more interesting - what's the greenest check in your pipeline that you now suspect has never been able to fail? I build cachly — memory for AI coding assistants, over MCP. ChatGPT and Claude remember your conversations. cachly remembers your system : the bug you fixed, why you chose Postgres, the deploy step that always breaks — and which earlier decision it contradicts. Every assistant you use reads the same memory, and every lesson carries the name of whoever learned it — so nobody has to learn it twice. Free tier, hosted in the EU: cachly.dev Top comments (40) Subscribe Personal Trusted User Create template Templates let you quickly answer FAQs or store snippets for re-use. Submit Preview Dismiss Collapse Expand Daniel Nwaneri Daniel Nwaneri Daniel Nwaneri Follow Full-Stack & AI Agent Developer | Python • Cloudflare Workers • MCP Specialist Email danielnwaneri41@gmail.com Location Port Harcourt, Nigeria Education B.TECH. Geophysics, Federal University of Technology Owerri Pronouns He/Him Work Full-Stack & AI Agent Developer | Python • Cloudflare Workers • MCP Joined Nov 11, 2025 • Aug 25 Dropdown menu Copy link Hide The "500 inside 4258 of 5000 quota points" example is the same shape of bug I shipped without realizing it. A deterministic gate on my project was matching "carbon" as a brand name inside the ordinary phrase "carbon copy," declining a completely unrelated question because it watched the string, not what the string meant. You caught it, not a test, because I didn't have one. The regression suite I added afterward has negative controls now, tests confirming genuine brand mentions still get caught, not just that the false ones stop firing. But it only exists because someone found the bug by hand first, and that someone was you. Your 11% number reads like that's the usual order: incident, then negative control, not the other way round. Genuinely curious whether your data can actually tell the difference between teams that built the negative control proactively and teams that built it the way I just did. Like comment: Like comment: 2 likes Like Comment button Reply Collapse Expand Heinrich Neb Heinrich Neb Heinrich Neb Follow Building cachly (MCP): your coding agent remembers your codebase and every decision made — not just the chat. Follows you between editors, stays when someone leaves. I post what I learn shipping it. Location Argenthal, Germany Work Founder, cachly.dev Joined Aug 12, 2026 • Aug 25 Dropdown menu Copy link Hide Honest answer to your genuinely curious question: no - our data cannot tell those two teams apart, and I want to be precise about why. The 11% is a snapshot of the guard population (how many have a negative control TODAY), not a time series. It doesn't see when a control was born or what prompted it. To measure your "usual order" hypothesis you'd need git archaeology: for each negative-control test, compare its commit date against the date of the incident/fix it guards - doable, and now I want to run it, but I haven't. Anecdotally, on our own codebase the order is almost always yours: incident first, control second. Today alone, twice - a trimming filter of ours would have silently eaten readme-generator.go because the pattern matched "readme", and a golden value in a reference solution was wrong because of a float edge (550 × 1.19 = 654.4999...). Both caught by controls that exist only because we'd been burned into requiring them. Which is the one structural fix I know for the ordering problem: make the negative control an ADMISSION rule instead of a reaction. In our benchmark harness, no checker is allowed into a run until it has proven all three gates - fails on the unsolved state, passes on the reference solution, fails again on a known-bad mutation. The control exists before any incident can, because without it the check simply doesn't run. Your carbon/carbon-copy guard would have needed a "matches brand, ignores idiom" pair on day one - not because anyone was wise, but because the gate refuses decoration. Like comment: Like comment: 2 likes Like Comment button Reply Collapse Expand Niuniu Ox Niuniu Ox Niuniu Ox Follow Joined Aug 23, 2026 • Aug 25 Dropdown menu Copy link Hide The 89% number is brutal and I recognize it. I ran a smaller version of this audit after a "green" CI gate let a broken migration through — out of 40-ish repo guards, exactly 3 had ever been fed a deliberately broken input. The rest were tautologies wearing a badge. The KONTROLLE: naming convention is the part I'm stealing. Marker-based counting is the only way this stays honest as the suite grows; if the control probe is optional-and-unmarked, it silently stops being written the first time someone is in a hurry. One thing I'd add to the taxonomy: guards that can fail but only on inputs that no longer occur. I had a lint rule rejecting a config format we deprecated 8 months ago — technically testable, practically dead. Did you count those as "able to fail" in your 22, or did you filter for probes tied to a currently-live failure mode? Curious how you handle the pushback when a negative control itself becomes the brittle part of the suite — probing a guard against a known-bad input that's too known-bad (nobody would ever actually write it) feels like testing the wrong thing. Where did you draw that line? Like comment: Like comment: 2 likes Like Comment button Reply Collapse Expand Heinrich Neb Heinrich Neb Heinrich Neb Follow Building cachly (MCP): your coding agent remembers your codebase and every decision made — not just the chat. Follows you between editors, stays when someone leaves. I post what I learn shipping it. Location Argenthal, Germany Work Founder, cachly.dev Joined Aug 12, 2026 • Aug 25 Dropdown menu Copy link Hide 3 of 40 - thank you for counting before commenting; may I add your 92.5% next to our 89% when I write the follow-up? Your two questions, honestly: Dead guards: you caught a real gap. Our 22 counted marker PRESENCE - "this guard has been fed a known-bad" - not liveness. A control probing a config format retired 8 months ago would have counted. Your case is now the third dimension in our counting scheme: can it fail / against a failure mode that still occurs / guarding a boundary that still exists. (Marco added the time-axis version in a sibling thread: regenerate the bad state from the CURRENT system, and assert the boundary is still present.) Where we draw the too-known-bad line: the known-bad must be the mistake a hurried human or agent would actually make, not a constructed absurdity. In practice we take it from incident history or from the most plausible reflex - Math.round instead of merchant rounding, forgetting the second mandatory file, dropping the sort. And we pair it with a near-miss known-GOOD (something that looks like the violation but isn't) - that pair is what keeps the control honest in both directions; ours caught an over-eager filter this week exactly that way. Like comment: Like comment: 1 like Like Comment button Reply Collapse Expand Edu Peralta Edu Peralta Edu Peralta Follow Making the terminal where your AI agents actually live. Multiple agents, one window, zero context lost. Sharing the journey http://runlinea.com Location Barcelona Joined Jul 4, 2026 • Aug 25 Dropdown menu Copy link Hide The 89% figure stuck with me because I keep seeing the same shape with coding agents. The agent says done, the exit code is zero, and the only thing that catches the lie is opening the file it claimed to edit. Your rule about judging the artifact, not the messenger, is the whole job now. I treat any green check that has never been fed a known bad input as unverified, same as an untested function. Curious how many of those 22 controls were added after a production miss versus written up front. Like comment: Like comment: 2 likes Like Comment button Reply Collapse Expand Heinrich Neb Heinrich Neb Heinrich Neb Follow Building cachly (MCP): your coding agent remembers your codebase and every decision made — not just the chat. Follows you between editors, stays when someone leaves. I post what I learn shipping it. Location Argenthal, Germany Work Founder, cachly.dev Joined Aug 12, 2026 • Aug 25 Dropdown menu Copy link Hide Honest answer: I can't tell you yet, and the reason is itself part of the finding. The marker count is a snapshot - it knows WHICH guards have controls today, not WHEN or WHY each was born. Anecdotally, every one I can date was incident-born, including two this week (a filter that would have silently eaten readme-generator.go, and a golden value that was wrong because of a float edge). Your question - same one Daniel Nwaneri asked an hour before you, independently - just became a measurement on our board: git archaeology, dating each control's introducing commit against the fix it guards, "unclear" reported as unclear. I'll ping this thread with the split when it's run. The one structure we've found that flips the order: admission gates - no check enters our benchmark harness until it has already failed on a known-bad. There, the control exists before any incident can. Like comment: Like comment: 1 like Like Comment button Reply Collapse Expand Kartik N V J K Kartik N V J K Kartik N V J K Follow I write when an idea won't leave me alone 🧠 Building AI agents and the tools to build AI agents. Love connecting AI with other fields and yapping about all of them. Education IIIT Dharwad Work AI Developer Joined Jun 10, 2026 • Aug 25 Dropdown menu Copy link Hide The 11 percent number would sting less if I did not immediately recognize all three of your failure shapes from my own repos, especially the classifier matching the "500" inside "5000 quota points". The KONTROLLE: marker convention is a nice forcing function, because right now most of my negative controls live as tribal knowledge in whoever last touched the guard. I am stealing the marker idea and running the count on my checks this week. Like comment: Like comment: 2 likes Like Comment button Reply Collapse Expand Heinrich Neb Heinrich Neb Heinrich Neb Follow Building cachly (MCP): your coding agent remembers your codebase and every decision made — not just the chat. Follows you between editors, stays when someone leaves. I post what I learn shipping it. Location Argenthal, Germany Work Founder, cachly.dev Joined Aug 12, 2026 • Aug 25 Dropdown menu Copy link Hide Stealing the marker is its entire purpose - and "negative controls living as tribal knowledge in whoever last touched the guard" is a better one-line justification for it than anything in my article. One warning before you run your count, from a mistake that cost us: make sure your counter reads code, not comments. When we first counted, 13 guards showed as "has a control" because the promised assertion existed only in a comment - the counter matched the string, exactly the failure shape you just recognized in the classifier. Strip comments first, then count. And please post your number when you have it - we're at 89% (ours) and 92.5% (another reader's 37/40), and I'd love to add yours to what's becoming an accidental community measurement. Like comment: Like comment: 1 like Like Comment button Reply Collapse Expand Karlis Karlis Karlis Follow Solo, AI‑assisted projects and games by a software developer. Location Riga, Latvia Education Riga Work Open to work Joined Feb 25, 2026 • Aug 25 Dropdown menu Copy link Hide Reviewing (checking) (testing) can grow very rapidly. Application's self fault tolerance and logging also should/could be improved. Like comment: Like comment: 2 likes Like Comment button Reply Collapse Expand Heinrich Neb Heinrich Neb Heinrich Neb Follow Building cachly (MCP): your coding agent remembers your codebase and every decision made — not just the chat. Follows you between editors, stays when someone leaves. I post what I learn shipping it. Location Argenthal, Germany Work Founder, cachly.dev Joined Aug 12, 2026 • Aug 25 Dropdown menu Copy link Hide Agreed on both - with one wrinkle we learned the hard way: logging is itself a guard that can go green-and-blind. We once had server errors that produced an EMPTY log (the throw happened where no logger was attached), so "no errors in the log" was absence, not health. Fault tolerance and logging help exactly to the degree that someone has fed the logging path a deliberate failure and seen it actually land. What kind of system are you seeing the growth problem in - CI checks, or runtime assertions? Like comment: Like comment: 1 like Like Comment button Reply Collapse Expand Karlis Karlis Karlis Follow Solo, AI‑assisted projects and games by a software developer. Location Riga, Latvia Education Riga Work Open to work Joined Feb 25, 2026 • Aug 25 Dropdown menu Copy link Hide Software development in general. More usual problem is that logs become too big and unreadable. I know that operations teams and admins look at dashboards and fix only red flags. Sometimes it is very frustrating. Or error message - if You see it, ask for the guy, who already left the company. Everything cannot be foreseen. Once server room was painted, and painters removed our server because at the moment no one knew, what it is doing. We searched for it a day 🤣 I am a developer, and I have seen time to time, that my work suddenly changes or expires. With that expires also effort put into its quality. Like comment: Like comment: 2 likes Like Thread Thread Heinrich Neb Heinrich Neb Heinrich Neb Follow Building cachly (MCP): your coding agent remembers your codebase and every decision made — not just the chat. Follows you between editors, stays when someone leaves. I post what I learn shipping it. Location Argenthal, Germany Work Founder, cachly.dev Joined Aug 12, 2026 • Aug 25 Dropdown menu Copy link Hide The painters story is the best parable in this whole thread, and I don't think it's off-topic at all: the server was green, healthy, doing its job - and got unplugged anyway, because nowhere on or near it was written WHAT it did and WHO would scream. That's the same failure as the log nobody can read and the error message whose author left: the system carried the fact but not the why. We can't foresee everything - you're right - but the why is cheap to write down at the moment someone still knows it, and it's the only thing that survives the person leaving. So, genuine question, because I collect these: what would the sign on that server have needed to say to survive the painters? My candidate: "This box does X. It belongs to Y. If you unplug it, Z stops working within N minutes." Three lines, and your team saves a day. Like comment: Like comment: 1 like Like Comment button Reply Collapse Expand TokenLat TokenLat TokenLat Follow Unified AI gateway — China's best LLMs + OpenAI/Gemini, the OpenAI way. Singapore-hosted (Tencent Cloud SG), PDPA-aligned, up to 90%+ cheaper than OpenAI on China models. Build with AI, Southeast Asia Location Singapore Joined Jul 14, 2026 • Aug 25 Dropdown menu Copy link Hide The 89% number is brutal and probably understated. I'd argue the same blind-trust bug is now repeating with LLM reviewers: most teams wire every call to one frontier model and call it a day, then never feed it a known-bad input either. The fix isn't a bigger model — it's routing by scenario. The mechanical 80% of reviews (format, obvious violations, "does this match the spec") don't need a frontier model at all; a smaller, cheaper one handles them, and you only spend frontier budget on the 20% that needs real judgment. That gets you the negative-control discipline you're describing and a 70%+ cost drop, because the expensive model is finally used where it can actually fail differently. Green checks that never saw a known-bad input are exactly what scenario routing is meant to stress-test. Like comment: Like comment: 2 likes Like Comment button Reply Collapse Expand Heinrich Neb Heinrich Neb Heinrich Neb Follow Building cachly (MCP): your coding agent remembers your codebase and every decision made — not just the chat. Follows you between editors, stays when someone leaves. I post what I learn shipping it. Location Argenthal, Germany Work Founder, cachly.dev Joined Aug 12, 2026 • Aug 25 Dropdown menu Copy link Hide The extension to LLM reviewers is the right next domino: a model-based check that never saw a known-bad input is my 89 % with a bigger invoice. One friendly disagreement, though: routing and falsifiability are orthogonal. Routing changes who reviews; a negative control tests whether the reviewer can fail - and a cheap model that never sees a planted violation is exactly as blind as the frontier one, just cheaper per blind spot. So I'd flip the order: build the known-bad corpus first, run it through every tier, and let the measured catch rates set the routing thresholds - not the task taxonomy. That would also test your most interesting claim, "used where it can actually fail differently": do you have per-tier catch rates on planted violations? If the small and the frontier model miss known-bads in different places, that disagreement is itself a routing signal - and that's the number I'd genuinely love to see. Like comment: Like comment: 2 likes Like Comment button Reply Collapse Expand TokenLat TokenLat TokenLat Follow Unified AI gateway — China's best LLMs + OpenAI/Gemini, the OpenAI way. Singapore-hosted (Tencent Cloud SG), PDPA-aligned, up to 90%+ cheaper than OpenAI on China models. Build with AI, Southeast Asia Location Singapore Joined Jul 14, 2026 • Aug 25 Dropdown menu Copy link Hide Agreed — and that's the part I hadn't fully separated.Routing and falsifiability are orthogonal axes. Routing answers "send the right model to the right task, stop paying frontier prices for mechanical traffic." Falsifiability answers "has this reviewer ever seen a known-bad input." They don't substitute: a router pushing 70% of calls to a cheap model, paired with a cheap reviewer that never saw a known-bad, just trades your 89% for "89% with a smaller invoice." Root cause untouched. The complement I'd want: treat known-bad regression as its own routed stream. Normal calls go through normal routing; a small persistent stream of known-bad traffic is pinned to a reviewer channel that runs regression checks. Routing saves the money, the regression channel keeps proving the reviewer still recognizes the boundary. Two axes, two jobs. (Your line "a model-based check that never saw a known-bad input is my 89% with a bigger invoice" is going straight into the next post's thesis — too good to leave buried.) Like comment: Like comment: 2 likes Like Thread Thread Heinrich Neb Heinrich Neb Heinrich Neb Follow Building cachly (MCP): your coding agent remembers your codebase and every decision made — not just the chat. Follows you between editors, stays when someone leaves. I post what I learn shipping it. Location Argenthal, Germany Work Founder, cachly.dev Joined Aug 12, 2026 • Aug 25 Dropdown menu Copy link Hide The pinned known-bad stream is the right complement - one addition, because there's a third reviewer hiding in your design: the router itself. A misrouted hard call is the new silent failure - "hard, but classified mechanical" produces a cheap answer that looks fine and is quietly wrong, and no per-tier regression stream catches it, because each tier only sees the traffic the router sent it. So the known-bad corpus needs a third slice: inputs that are known-hard-disguised-as-mechanical, pinned through the classifier, scoring its confusion rate. Route the models, regression-test the reviewers, and regression-test the thing that decides who reviews. Like comment: Like comment: 2 likes Like Thread Thread TokenLat TokenLat TokenLat Follow Unified AI gateway — China's best LLMs + OpenAI/Gemini, the OpenAI way. Singapore-hosted (Tencent Cloud SG), PDPA-aligned, up to 90%+ cheaper than OpenAI on China models. Build with AI, Southeast Asia Location Singapore Joined Jul 14, 2026 • Aug 25 Dropdown menu Copy link Hide The "router is the third reviewer" framing is the part most teams miss. The silent failure is real precisely because each tier only ever sees the traffic the router already decided was its's — so a misroute never surfaces as a tier regression, it just becomes a quietly-wrong cheap answer. The fix you're pointing at is making the router's own confusion rate visible: pin an adversarial slice (known-hard-disguised-as-mechanical) and replay it through the classifier every release, the same way you'd regression-test a model. Route the models, regression-test the reviewers, and regression-test the thing deciding who reviews — exactly. The only addition I'd make: log the router's confidence on that slice over time, so drift shows up before it reaches a call. Like comment: Like comment: 2 likes Like Thread Thread Heinrich Neb Heinrich Neb Heinrich Neb Follow Building cachly (MCP): your coding agent remembers your codebase and every decision made — not just the chat. Follows you between editors, stays when someone leaves. I post what I learn shipping it. Location Argenthal, Germany Work Founder, cachly.dev Joined Aug 12, 2026 • Aug 25 Dropdown menu Copy link Hide Logging confidence on the pinned slice - agreed, with one sharpening: track it as a calibration curve per release, not a raw average. The dangerous quadrant is confidence flat while the slice's error rate moves - confidently-wrong is the only failure mode that reaches production without a symptom. A per-release curve on the same pinned slice gives you that drift almost for free, since you're replaying it anyway. Like comment: Like comment: 1 like Like Comment button Reply Collapse Expand kevinbai kevinbai kevinbai Follow Making useful things Joined Mar 23, 2026 • Aug 25 Dropdown menu Copy link Hide This framing is sharp. We invested heavily in 'AI review' but forgot that review is a feedback loop: the reviewer learns the codebase, the team, and the failure modes. If we don't validate that loop, we just speed up the approval of bad changes. Like comment: Like comment: 2 likes Like Comment button Reply Collapse Expand Heinrich Neb Heinrich Neb Heinrich Neb Follow Building cachly (MCP): your coding agent remembers your codebase and every decision made — not just the chat. Follows you between editors, stays when someone leaves. I post what I learn shipping it. Location Argenthal, Germany Work Founder, cachly.dev Joined Aug 12, 2026 • Aug 25 Dropdown menu Copy link Hide The loop framing sharpens something I only half-said: a negative control validates the reviewer at a point in time - it doesn't make the reviewer learn. I counted one week of my own agent-assisted failures and it was the same handful of classes recurring: four imports that silently started a main(), five checks that matched wording instead of meaning. A reviewer that had learned from Monday would have rejected Thursday's change. So the loop needs both halves: falsifiable guards (can it reject?) and accumulated failure modes (does it know what to reject here?). Most setups I've seen have neither wired in. What does validating the loop look like concretely on your team - do you measure whether review findings recur? Like comment: Like comment: 1 like Like Comment button Reply Collapse Expand UnitBuilds UnitBuilds UnitBuilds Follow Founder of UnitBuilds CC Location Swakopmund, Namibia Pronouns He/Him Work Senior software Engineer (day-job), Owner of UnitBuilds (sadly second). Joined May 24, 2026 • Aug 25 Dropdown menu Copy link Hide And that's exactly why I say AI is a senior dev's tool, not a juniors. If you produced garbage code before, AI just makes the pile bigger. Yes, it corrects alot, but if you had a 5% error rate, 5% of 1000 LOC is manageable for a senior to audit, 5% of 10k LOC isnt. The rush to get everyone on AI, skipped the important training stage, where people get taught how to use AI responsibly. Because I bet none of you got training on how to use AI-assistants? And that's exactly the problem. Like comment: Like comment: 2 likes Like Comment button Reply Collapse Expand Heinrich Neb Heinrich Neb Heinrich Neb Follow Building cachly (MCP): your coding agent remembers your codebase and every decision made — not just the chat. Follows you between editors, stays when someone leaves. I post what I learn shipping it. Location Argenthal, Germany Work Founder, cachly.dev Joined Aug 12, 2026 • Aug 25 Dropdown menu Copy link Hide Half agree - and the half I'd push back on is where the fix lives. The 5 %-of-10k problem is real, but seniority doesn't solve it: nobody audits 10k LOC, senior or not. I'm the senior in my own article, and I was the punchline twice in one evening - experience didn't protect me, conventions did. What seniors actually have isn't better eyes; it's habits that shrink what needs eyes. They don't audit the pile, they audit the gates the pile must pass. And that's learnable in a week: the negative-control convention in the post is teachable to a junior on day one, and it scales with LOC in a way eyeballs never will - my audit surface is 204 guard files, not 10,000 lines. On training: agreed that nobody got it - but I'd sharpen what the missing course actually is. Not "how to prompt." It's "how to review" - and that course never existed for humans either. AI didn't create the gap; it promoted everyone into the seat where the gap was always sitting. The syllabus is more or less this comment thread: judge the artifact, not the messenger; every conclusion-bearing check gets shown a known-bad input; a green zero is the most dangerous answer a check can give. Teach that, and a junior with AI is safer than a senior without it - because the junior's checks can prove they're able to fail, and the senior's memory can't. Like comment: Like comment: 2 likes Like Comment button Reply Collapse Expand UnitBuilds UnitBuilds UnitBuilds Follow Founder of UnitBuilds CC Location Swakopmund, Namibia Pronouns He/Him Work Senior software Engineer (day-job), Owner of UnitBuilds (sadly second). Joined May 24, 2026 • Aug 25 Dropdown menu Copy link Hide Exactly, my distinction between senior and junior isnt one of experience, it's role. Senior devs are used to reviewing juniors' work. They know what mistakes a heavy hand makes and know how to course correct over-eagerness. The process of effective AI usage doesnt change, it's still check twice, write once, which is equally effective in a junior's hands as a senior's but the senior has the experience of having reviewed junior work and correcting it, whereas a junior lacks that experience. That's the gap, because a senior was put into the role of reviewing, whereas it's new territory for a junior. The gap grows when the junior produces code with AI, that the senior still needs to review, because the scope of the work grows. That's unavoidable, but the senior cuts out the middleman, they do a T2 audit of the AI's work, before submitting, doesnt mean it doesnt need review, it's just in the adjusting landscape, they've become the bare minimum (junior). That has to self-check before submitting, then their work needs peer-review still. A junior who can barely code, has no way of telling AI slop that compiles apart from clean architecture, they see passing unit tests and clean permissions, thinking that it was enforced properly. Like comment: Like comment: 2 likes Like Thread Thread Heinrich Neb Heinrich Neb Heinrich Neb Follow Building cachly (MCP): your coding agent remembers your codebase and every decision made — not just the chat. Follows you between editors, stays when someone leaves. I post what I learn shipping it. Location Argenthal, Germany Work Founder, cachly.dev Joined Aug 12, 2026 • Aug 25 Dropdown menu Copy link Hide Role, not experience - that's a cleaner cut than mine, and I'll adopt it. The sentence I keep coming back to is your last one: the junior "sees passing unit tests and clean permissions, thinking that it was enforced properly." The half-hopeful thing we've measured: that specific trap is partially fixable with instruments, not only with years of review experience. Our rule is that every check gets a known-bad twin - feed it an input that MUST fail, and if the check stays green, it was decoration, not enforcement. That habit caught a fleet-config guard of ours that would have reported green on a fully broken fleet, because it read a field that was always empty. No amount of "the tests pass" would have surfaced that; one known-bad did, in thirty seconds. It doesn't close the taste gap - telling clean architecture from slop that compiles is still the senior's edge. But "passing tests = enforced" is the most dangerous half of the gap, and it's the mechanical half. Which makes me curious: is your T2 audit a written checklist a junior could run, or is it tacit? If you've written it down, that list might be the most useful artifact in this whole thread. Like comment: Like comment: 2 likes Like Thread Thread UnitBuilds UnitBuilds UnitBuilds Follow Founder of UnitBuilds CC Location Swakopmund, Namibia Pronouns He/Him Work Senior software Engineer (day-job), Owner of UnitBuilds (sadly second). Joined May 24, 2026 • Aug 25 Dropdown menu Copy link Hide Alot of T2 comes down to the nose knows, but essentially it's a scoped check. Are all credentials secured. Are all endpoints guarded. Are granular permissions enforced. Does the implementation match the pattern of the rest of the codebase. Do rules enforced strict scope acceptable values. If visual, does the layout conform with the pattern of the rest of the codebase. Are shared reusables used appropriately. If affecting a shared component, was the blast radius checked for potentially breaking changes. If database was affected, are changes documented in a migration for reproducibility. Does the migration conform with the standards set by previous migrations. Are all sql queries optimized. Are the sql tables indexed, or views created where necessary. Is the code clean of any local paths. Are all URLs verified against the whitelist of company scoped domains. Are all tasks in the scope completed. If dependencies were affected, do all dependents still maintain a working state. If new, is the module properly wired into navigation. Then there's a few more that you can add based on policy, eg. are all new pages listed in the navigation sidebar, do all sql inserts and updates use a transaction, are sql connections properly disposed of, etc. If dependency checks, is it guarded against BOM explosions, such as circular references and diamond BOMs, all endpoints and frontend need to use a URL file entry, etc. Depending on what you're working on, the scope adjusts accordingly, given that a Blazor app and a React app work differently, even the frameworks you use vary what to check, eg. V.A.L.I.D. and CSLA for blazor: V.A.L.I.D. you just need to check your DTO is initialized properly with ValidObjects and properly confined and that your HTML markup looks correct, vs CSLA you need to verify your BO objects, child objects, parent-child handovers, data access layer usage, mappers, rules, etc. Unfortunately it's not a 1 size fits all, it's very dependent on the kind of work being done and the use-case of the system. Like comment: Like comment: 2 likes Like Thread Thread Comment deleted Like comment: Like comment: 2 likes Like Thread Thread UnitBuilds UnitBuilds UnitBuilds Follow Founder of UnitBuilds CC Location Swakopmund, Namibia Pronouns He/Him Work Senior software Engineer (day-job), Owner of UnitBuilds (sadly second). Joined May 24, 2026 • Aug 25 Dropdown menu Copy link Hide Alot of it can be checks, I built them into the IDE I'm developing, but these cover the basics, the more nuanced things come down to when you arent using primitives, you're using parameters. To track the lifecycle of a parameter from start to finish, could in theory be scripted, but that would also mean you need to set a valid state for it at each point in the lifecycle. Take a bool for instance, first it's null, then it's not null, then it's null again. So is it nullable? AI would say yes, but if it's conditionally null during separate processes, with separate fetches, that nullable must at times be treated as a non-nullable. It's a niche situation where you'd need to know what's meant to happen, in order to correctly handle it. Those niche cases are where the context matters and while you can preserve context with memory systems for AI, you cant retroactively teach it from an existing codebase. It's the things that break the mold that would cause an issue in the system. Take your gating, it would pass the gates as a nullable, all the time, it wouldnt even flag it as needing review, because it's something atypical that unfortunately happens far too often in production and knowing the system's intended behavior is the only thing that lets you make an informed decision on it. Sure, you could explicitly state it in the scope, but lets be honest, you dont walk in and get a neatly typed 3 page breakdown of what to do in the morning, you get a 3 sentence instruction and left to wing it till you get stuck and need feedback. Like comment: Like comment: 2 likes Like Thread Thread Heinrich Neb Heinrich Neb Heinrich Neb Follow Building cachly (MCP): your coding agent remembers your codebase and every decision made — not just the chat. Follows you between editors, stays when someone leaves. I post what I learn shipping it. Location Argenthal, Germany Work Founder, cachly.dev Joined Aug 12, 2026 • Aug 25 Dropdown menu Copy link Hide The conditionally-null bool is the best concrete example anyone has given me for this whole discussion, because you're right on every step: the code cannot tell you, a lifecycle script would need the intended state per phase, a gate would wave it through as "nullable", and nobody hands you three typed pages in the morning. Where I'd add one measured data point: we ran exactly this class through a benchmark this week - repo tasks whose governing rule contradicts best practice and appears NOWHERE in the code (our own tasks, so home advantage, full harness published). The agent with no knowledge carrier passed 3/12; with the rule stored as a retrievable lesson it passed 12/12, p=0.0039 paired - and a larger 120-cell run is in progress showing the same shape. The interesting part for your argument: the knowledge was never DERIVED from the codebase. You're completely right that you can't retroactively teach intent from existing code - the code doesn't contain it. What does contain it is the correction moment. Somewhere, once, a senior says "no - after fetch A that bool is guaranteed non-null, treating it as nullable there masks failures." That's your three-sentence morning instruction. If that sentence gets captured at that moment - with the why - it's retrievable the next time anyone touches the parameter. Not retro-learning from code; accretion from corrections. Your IDE tracking parameter lifecycles plus a store for exactly these exception-intentions wouldn't compete - one watches the mold, the other remembers why this piece deliberately breaks it. Like comment: Like comment: 1 like Like Comment button Reply Collapse Expand Vinh Nguyen Vinh Nguyen Vinh Nguyen Follow Solution architect, 16 years in software, last 3 on AI automation. I write about making automated work verifiable: evaluation harnesses, audit trails, and results published even when the answer is no. Location Da Nang, Vietnam Work Solution Architect at FPT Software Joined Jun 22, 2026 • Aug 25 Dropdown menu Copy link Hide My answer to your last question is a publish check that read one field off an API response. The field is not in that endpoint's representation at all: three variants of the request, with the key, without it, and without the vendor accept header, all came back with no published key, and .get() on a missing key hands you None , which the guard scored as "not published". What made it hard to catch with a known-bad input is that there is no bad input to feed, since the guard was reading absence and reporting a value, so both worlds looked identical from inside the test. The control that separated them was asserting key presence separately from key value, and then moving the real check onto a signal that endpoint does carry, an unauthenticated GET returning 200, because a draft fetched by id returns 404 even with the owner's key. Like comment: Like comment: 2 likes Like Comment button Reply Collapse Expand Heinrich Neb Heinrich Neb Heinrich Neb Follow Building cachly (MCP): your coding agent remembers your codebase and every decision made — not just the chat. Follows you between editors, stays when someone leaves. I post what I learn shipping it. Location Argenthal, Germany Work Founder, cachly.dev Joined Aug 12, 2026 • Aug 25 Dropdown menu Copy link Hide This is the sharpest reply the article has received, because it names a class my counter cannot see: guards that read absence. My negative-control definition quietly assumes a known-bad input exists - your check had no bad input to construct, because .get() on a missing key manufactures a value out of absence, and both worlds look identical from inside the test. "Assert key presence separately from key value" is the move that makes the class testable at all: it converts absence back from a value into a state. And relocating the check onto a signal the endpoint actually carries - unauthenticated GET, 200 vs. 404 - is judge-the-artifact in its purest form. I'm adding "absence-readers" as a fourth green-and-blind shape next to the three messenger-readers in the article; with your permission I'll cite this comment when I write it up. One question back: after the fix, did you add a probe asserting the 404-for-drafts behavior stays - or is the vendor's draft semantics now the new untested assumption underneath the guard? Like comment: Like comment: 1 like Like Comment button Reply Collapse Expand Vinh Nguyen Vinh Nguyen Vinh Nguyen Follow Solution architect, 16 years in software, last 3 on AI automation. I write about making automated work verifiable: evaluation harnesses, audit trails, and results published even when the answer is no. Location Da Nang, Vietnam Work Solution Architect at FPT Software Joined Jun 22, 2026 • Aug 26 Dropdown menu Copy link Hide Yes, please cite it. The 404 behavior has a standing negative control: draft 4449977 is intentionally kept unpublished, and GET /api/articles/4449977 returns 404 both anonymously and with the owner's API key, while a public article ID returns 200. The publish path rechecks anonymous 200 every time; keeping the draft means the vendor assumption remains executable rather than becoming prose. Like comment: Like comment: 1 like Like Comment button Reply Collapse Expand Marco Marco Marco Follow Cloud, AI & Security Engineer | Python, AWS & DevSecOps. Building open-source tools, exploring AI agents, breaking things in staging so I can fix them in prod. 🇮🇹 Location Casale Monferrato, 15033, AL, Italy Education Computer Science - Università del Piemonte Orientale Amedeo Avogadro Pronouns he/him Joined Apr 15, 2026 • Aug 25 Dropdown menu Copy link Hide What I find most interesting here is that the hard part isn't making the security check fail when something is obviously broken. It's proving that the check still has a meaningful boundary to test against as the system evolves. A green test can survive a broken invariant, a stale fixture, or even a defence that no longer has anything to protect. That's why I increasingly like the idea of treating negative controls as first-class engineering artifacts: if we can't deliberately cross the failure boundary and make the test go red, we should question what the test is actually proving. The “run, don't trust the comment” conclusion is probably the part I'll take away from this thread. 🔐 Like comment: Like comment: 2 likes Like Comment button Reply Collapse Expand Heinrich Neb Heinrich Neb Heinrich Neb Follow Building cachly (MCP): your coding agent remembers your codebase and every decision made — not just the chat. Follows you between editors, stays when someone leaves. I post what I learn shipping it. Location Argenthal, Germany Work Founder, cachly.dev Joined Aug 12, 2026 • Aug 25 Dropdown menu Copy link Hide The evolution point deserves its own article, because it's the time-axis my count ignores: a negative control proves the guard can fail today - nothing proves the boundary it crosses still exists next quarter. We hit the pure form of this: a guard with a hard-coded threshold that kept validating a world that had moved on. It was green every day, faithfully guarding the past. Two practices that have helped since: (1) generate the planted-bad from the current system at test time instead of storing it as a fixture - a frozen bad input is a frozen boundary, and it rots at exactly the rate of the system around it; (2) pair the crossing-control with a freshness assertion on the boundary itself - my favorite example this week is a test that fails the moment a corpus and its denylist stop matching exactly: it doesn't test the crossing, it tests that there's still a fence where the map says one is. And "run, don't trust the comment" cuts both ways: your prompt-injection piece is the sibling failure - the test passed while the attack worked, which is this thread's 89 % wearing a security label. Like comment: Like comment: 2 likes Like Comment button Reply Collapse Expand Marco Marco Marco Follow Cloud, AI & Security Engineer | Python, AWS & DevSecOps. Building open-source tools, exploring AI agents, breaking things in staging so I can fix them in prod. 🇮🇹 Location Casale Monferrato, 15033, AL, Italy Education Computer Science - Università del Piemonte Orientale Amedeo Avogadro Pronouns he/him Joined Apr 15, 2026 • Aug 25 Dropdown menu Copy link Hide Heinrich, I really like the time-axis framing. I think it adds an important third dimension to the two-point model I was using. A negative control proves “this guard can be falsified against this boundary today”, but it doesn't prove that the boundary, fixture, or predicate still represents the current system. I especially like the combination of generating the bad state from the current system and asserting that the boundary itself is still present. That feels like a natural extension of the same rule: don't just test that the guard can fail — test that it is still guarding the thing you think it is guarding. And yes, the prompt-injection case is basically the same failure wearing a different name: the test existed, the attack existed, and the assertion still proved the wrong thing. 🔐 Like comment: Like comment: 2 likes Like Thread Thread Heinrich Neb Heinrich Neb Heinrich Neb Follow Building cachly (MCP): your coding agent remembers your codebase and every decision made — not just the chat. Follows you between editors, stays when someone leaves. I post what I learn shipping it. Location Argenthal, Germany Work Founder, cachly.dev Joined Aug 12, 2026 • Aug 25 Dropdown menu Copy link Hide "Don't just test that the guard can fail - test that it is still guarding the thing you think it is guarding." That sentence is the whole next article, honestly - may I quote it with your name when I write it? It completes the model: falsifiability (can it go red), liveness (against today's system), and now aim (is the boundary still the one that matters). Your two-point model plus the time axis makes it three orthogonal ways a green check can be lying. Like comment: Like comment: 1 like Like Comment button Reply View full discussion (40 comments) Some comments may only be visible to logged-in visitors. Sign in to view all comments. Code of Conduct • Report abuse Are you sure you want to hide this comment? It will become hidden in your post, but will still be visible via the comment's permalink . Hide child comments as well Confirm For further actions, you may consider blocking this person and/or reporting abuse Heinrich Neb Follow Building cachly (MCP): your coding agent remembers your codebase and every decision made — not just the chat. Follows you between editors, stays when someone leaves. I post what I learn shipping it. Location Argenthal, Germany Work Founder, cachly.dev Joined Aug 12, 2026 More from Heinrich Neb What does your AI assistant remember from yesterday? # ai # productivity # programming # llm The best argument against my MCP server came from Anthropic # ai # mcp # career # programming I built an MCP memory server for one user (me, for six weeks) # ai # mcp # programming # devops DEV Community — A space to discuss and keep up software development and manage your software career Home DEV Challenges DEV++ Videos DEV Education Tracks DEV Help Advertise on DEV Organization Accounts DEV Showcase About Contact Free Postgres Database DEV Shop MLH Code of Conduct Privacy Policy Terms of Use Built on Forem — the open source software that powers DEV and other inclusive communities. Made with love and Ruby on Rails . DEV Community © 2016 - 2026. We're a place where coders share, stay up-to-date and grow their careers. Log in Create account var userSignedIn = false; if (document.readyState === 'complete' || document.readyState === 'interactive') { initAuth(); } else { document.addEventListener('DOMContentLoaded', initAuth); } function initAuth() { var paramToken = new URLSearchParams(window.location.search).get('jwt'); if (paramToken && !userSignedIn) { authenticateUser(paramToken); } else { var iframe = document.createElement('iframe'); iframe.style.display = 'none'; iframe.src = 'https://forem.com/auth_pass/iframe?passed_domain=' + encodeURIComponent(window.location.origin || (window.location.protocol + '//' + window.location.host)); document.body.appendChild(iframe); window.addEventListener('message', function(event) { if (event.origin !== 'https://forem.com' && event.origin !== window.location.origin) { return; } var data = event.data; if (data.authenticated && !userSignedIn) { authenticateUser(data.token); } else if(data.authenticated && window.ReactNativeWebView && window.ReactNativeWebView.postMessage) { window.ReactNativeWebView.postMessage(JSON.stringify({ action: 'login', token: data.token, })); } }); } function authenticateUser(token) { fetch('/auth_pass/token_login', { method: 'POST', credentials: 'include', headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': getMetaContent('csrf-token'), }, body: JSON.stringify({ token: token }), }) .then(function(response) { return response.json(); }) .then(function(data) { if (data.success) { if (document.head.querySelector('meta[name="user-signed-in"][content="false"]')) { // Reload the page to update the user's state location.reload(); } } }) .catch(function(error) { console.error('Error during authentication:', error); }); } function getMetaContent(name) { var element = document.querySelector('meta[name="' + name + '"]'); return element ? element.getAttribute('content') : ''; } }