[{"_1":2,"_7326":-5,"_7327":-5},"loaderData",{"_3":4,"_270":271},"root",{"_5":6,"_7":8,"_252":253},"ua","Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","data",{"_9":10,"_145":146,"_158":159},"footer",{"_11":12,"_13":9,"_14":15,"_16":17,"_23":9,"_24":25,"_26":27,"_34":35,"_107":108},"_createdAt","2022-05-17T11:04:58Z","_id","_rev","6sA5HXzqQ1t7bsnHMaqtNi","_system",{"_18":19},"base",{"_20":9,"_21":22},"id","rev","GthKAURudFYTNgxt0CQU64","_type","_updatedAt","2026-03-18T16:16:29Z","emailSubscription",{"_28":29,"_30":31,"_32":33},"description","Unlock success from technical pioneers in software infrastructure. We send the best strategies, tools, and industry updates directly to your inbox.","targetURL","https://share.hsforms.com/1_KYaR4lAQm-C8_9H7r4MIwqdoyu","title","Stay in the loop","footerColumns",[36,64,88],{"_37":38,"_23":39,"_40":41,"_42":43,"_49":62,"_32":63},"_key","30a87ef6e7c1","footerColumn","heading","About","items",[44,52,57],{"_37":45,"_23":46,"_47":48,"_49":50,"_32":51},"3dfef1283ab5","footerLink","label","Portfolio","link","/portfolio","Navigate to portfolio page",{"_37":53,"_23":46,"_47":54,"_49":55,"_32":56},"42666d0b3987","Team","/team","Navigate to team page",{"_37":58,"_23":46,"_47":59,"_49":60,"_32":61},"398a9b50e214","News","/press","Navigate to press page","/about","Navigate to about page",{"_37":65,"_23":39,"_40":66,"_42":67,"_49":86,"_32":87},"a584a6acf07b","Community",[68,72,77,82],{"_37":69,"_23":46,"_47":70,"_49":71,"_32":70},"4358699f4ca4","Events","/events",{"_37":73,"_23":46,"_47":74,"_49":75,"_32":76},"e2d5641ef7de","DevGuild","https://www.heavybit.com/devguild","Navigate to devguild page",{"_37":78,"_23":46,"_47":79,"_49":80,"_32":81},"feb038ce46a1","Jobs","/jobs","Portfolio Company Jobs",{"_37":83,"_23":46,"_47":84,"_49":85,"_32":84},"e22bfa6b2330","Inclusion","/community/diversity-and-inclusion","https://www.heavybit.com/community","Navigate to community page",{"_37":89,"_23":39,"_40":90,"_42":91,"_49":95,"_32":106},"ea618aa34610","Library",[92,97,102],{"_37":93,"_23":46,"_47":94,"_49":95,"_32":96},"c82a64edc4dd","Blog","https://www.heavybit.com/library/","Navigate to browse on library page",{"_37":98,"_23":46,"_47":99,"_49":100,"_32":101},"e1673433290e","Podcast Network","https://www.heavybit.com/library/podcasts","Navigate to podcasts page",{"_37":103,"_23":46,"_47":104,"_49":105,"_32":104},"96a97ceba3a2","DevToolsDigest","https://www.heavybit.com/devtoolsdigest","Navigate to library page","socialMedia",[109,121,127,133,139],{"_37":110,"_111":112,"_119":120},"ee025c96397c","icon",{"_23":113,"_114":115},"mainImage","asset",{"_116":117,"_23":118},"_ref","image-9d618da0948299428b29e5d499af3fa5369a24d2-20x20-svg","reference","url","https://twitter.com/heavybit",{"_37":122,"_111":123,"_119":126},"5348b028f0d41c3dac62b3dd2b5531ee",{"_23":113,"_114":124},{"_116":125,"_23":118},"image-01bd92eb06873bdc30b1cb5f9c78b1ef5a9559d4-512x512-png","https://www.linkedin.com/company/heavybit",{"_37":128,"_111":129,"_119":132},"fae1cc0df5c6",{"_23":113,"_114":130},{"_116":131,"_23":118},"image-6c548d2e815d1fb3b3077c9b8be1ec251e1bc418-512x512-png","https://www.youtube.com/c/heavybit",{"_37":134,"_111":135,"_119":138},"a100f6448036",{"_23":113,"_114":136},{"_116":137,"_23":118},"image-b38b7ee9ccd05f61ca43d5b29fe7058f822f87eb-512x512-webp","https://www.instagram.com/heavybitindustries/",{"_37":140,"_111":141,"_119":144},"639729864c2d",{"_23":113,"_114":142},{"_116":143,"_23":118},"image-48ac9473e30341c4bee9f6634b77b6f8f1dd6b2f-849x980-webp","https://www.tiktok.com/@heavybitindustries","globalConfig",{"_147":148,"_11":152,"_13":145,"_14":153,"_16":154,"_23":145,"_24":157},"OGimage",{"_23":149,"_114":150},"image",{"_116":151,"_23":118},"image-32fdfae4fd13b8f9a3c5bf6b13a635068cb0ecbb-1332x529-png","2022-05-17T11:15:28Z","RDgKg7yO6ZYtH7Ot8nWSxU",{"_18":155},{"_20":145,"_21":156},"2ZxAYmIihJuAs2TKuKQs3J","2025-08-19T13:26:15Z","header",{"_11":160,"_13":158,"_14":161,"_16":162,"_23":158,"_24":165,"_166":167,"_177":178},"2022-05-17T11:04:33Z","WeSCIjvgoYC0LmYF9BLgMk",{"_18":163},{"_20":158,"_21":164},"rdofR6C9TPA5BJl8cHZaE4","2026-08-03T18:37:30Z","announcementBar",{"_168":169,"_170":171,"_172":173,"_174":175,"_119":176},"buttonText","RSVP","show",true,"start","2026-08-03T18:37:28.492Z","text","🌅 Sep 24: The Long-Horizon Stack feat Hex, Baseten, Glean and more","https://luma.com/long-horizon-stack?utm_source=heavybit_site&utm_medium=banner","headerNavLinks",[179,197,207,220,248],{"_37":180,"_23":181,"_182":171,"_183":184,"_47":41,"_49":62,"_195":196,"_32":63},"a8d641babed2","navLink","addDropdown","dropdown",[185,188,193],{"_37":186,"_23":187,"_47":54,"_49":55,"_32":56},"964627410473","dropdownLink",{"_37":189,"_23":187,"_47":190,"_49":191,"_32":192},"3d279775d73b","FAQ","/faq","Navigate to FAQ page",{"_37":194,"_23":187,"_47":59,"_49":60,"_32":61},"84e68f9f5c33","mobileOnly",false,{"_37":198,"_23":181,"_182":171,"_183":199,"_47":48,"_49":50,"_195":196,"_32":51},"90b68b861c06",[200,205],{"_37":201,"_23":187,"_47":202,"_49":203,"_32":204},"d481fcbbd3af","Spotlights","/portfolio/spotlights","Navigate to spotlights page",{"_37":206,"_23":187,"_47":79,"_49":80,"_32":81},"e50fa43ad2e7",{"_37":208,"_23":181,"_182":171,"_183":209,"_47":66,"_49":219,"_195":196,"_32":87},"716b76469486",[210,212,216],{"_37":211,"_23":187,"_47":70,"_49":71,"_32":70},"3a177efc4649",{"_37":213,"_23":187,"_47":214,"_49":215,"_32":214},"1fb7c9c465d0","Advisors","/community/advisors",{"_37":217,"_23":187,"_47":74,"_49":218,"_32":76},"23077fca0e2f","/devguild","/community",{"_37":221,"_23":181,"_182":171,"_183":222,"_47":90,"_49":247,"_195":196,"_32":106},"4bec0dd665d4",[223,228,233,236,240,244],{"_37":224,"_23":187,"_47":225,"_49":226,"_32":227},"63e48d609548","Articles","/library?filter=articles","Read educational articles in Heavybit's Library",{"_37":229,"_23":187,"_47":230,"_49":231,"_32":232},"6526c69f5260","Videos","/library?filter=videos","Watch educational videos in Heavybit's Library",{"_37":234,"_23":187,"_47":99,"_49":235,"_32":101},"1f3fb936f3b9","/library/podcasts",{"_37":237,"_23":187,"_47":238,"_49":239,"_32":238},"ac06751f0372","Collections","/library/collections",{"_37":241,"_23":187,"_47":242,"_49":243,"_32":242},"2ead3bd14715","Reports","/reports",{"_37":245,"_23":187,"_47":104,"_49":246,"_32":104},"66773779036c","/devtoolsdigest","/library",{"_37":249,"_23":181,"_182":196,"_47":250,"_49":60,"_195":171,"_32":251},"3702498a502c","Contact","Navigate to contact page","ENV",{"_254":255,"_256":257,"_258":259,"_260":261,"_262":263,"_264":265,"_266":267,"_268":255,"_269":255},"SANITY_DATASET","production","ALGOLIA_INDEX","heavybit-library","ALGOLIA_APPLICATION_ID","0QCKBC1FMK","ALGOLIA_SITEWIDE_INDEX","heavybit-sitewide","ALGOLIA_JOBS_INDEX","heavybit-jobs","MUX_ENV_KEY","dRI98bL8ze03gYDL6-Ogle2Js","ALGOLIA_SEARCH_API_KEY","7733fdd199ec7af7ee2923ec39c3f036","MODE","VITE_CONTEXT","routes/library.video.$slug",{"_7":272,"_7324":196,"_7325":-5,"_333":2193},{"_273":274},"routeData",{"_11":275,"_13":276,"_14":277,"_16":278,"_23":282,"_24":283,"_284":285,"_286":171,"_287":-5,"_288":289,"_290":-5,"_291":292,"_293":294,"_310":311,"_312":313,"_2157":-5,"_2158":2159,"_2166":2167,"_2176":2177,"_331":2180,"_2181":2182,"_2190":-5,"_333":2191,"_2194":-5,"_2195":2196,"_2372":292,"_2339":2373,"_32":2399,"_2400":2401,"_282":7251},"2026-06-16T22:22:21Z","c62d9fb9-56c8-4439-8d32-49f5b68c0fb8","Vem3gF5h3LEi9EUb4dhyQk",{"_18":279},{"_20":280,"_21":281},"73c55bad-99db-4c07-a85a-a14c55070868","hrNu3JfIER4D8k6P3PY8kk","video","2026-08-13T22:04:08Z","access","Public","afterEvent","agenda","collections",[],"cta","endDate","2025-09-25T16:00:00.000Z","excerpt",[295],{"_37":296,"_23":297,"_298":299,"_306":307,"_308":309},"216f99acef52","block","children",[300],{"_37":301,"_23":302,"_303":304,"_174":305},"6ed5469b034d","span","marks",[],"Agents can write, review, and ship code — but how accountable can we be for systems we never fully scrutinize? This panel uncovers what it takes to let agents loose on production systems without losing sleep: permissions, observability, and the guardrails keeping it all from imploding.","markDefs",[],"style","normal","filmedOn","2026-05-07","library",{"_42":314,"_370":371},[315,338,354],{"_316":317,"_326":327,"_149":328,"_329":330,"_331":332,"_333":334,"_32":335,"_336":337},"content",[318],{"_37":319,"_23":297,"_298":320,"_306":325,"_308":309},"6eda4afc38ac",[321],{"_37":322,"_23":302,"_303":323,"_174":324},"0a87a5e6833b",[],"On episode 53 of The Kubelist Podcast, Marc Campbell and Benjie De Groot sit down with Anurag Goel. Anurag shares his journey from early employee at Stripe to founder and CEO of Render, one of the fastest-growing application platforms in cloud infrastructure. They discuss Kubernetes, platform engineering, bare metal, AI agents, and what the future of application deployment looks like.",[],"duration","59:55","https://cdn.sanity.io/images/50q6fr1p/production/d43ea2b98686b7418f3d26812b9d9489ffb8fdc2-3000x3000.jpg","objectID","22e3bacf-9e6b-499e-aff9-4eedf21bfc27","publishDate","2026-06-09T07:10:00.000Z","slug","the-kubelist-podcast/ep-53-render-and-the-new-cloud-stack-with-anurag-goel","The Kubelist Podcast Ep. #53, Render and the New Cloud Stack with Anurag Goel","typeName","podcastEpisode",{"_316":339,"_326":348,"_149":349,"_329":350,"_331":351,"_333":352,"_32":353,"_336":337},[340],{"_37":341,"_23":297,"_298":342,"_306":347,"_308":309},"7956c47c09f0",[343],{"_37":344,"_23":302,"_303":345,"_174":346},"a8bd15848226",[],"On episode 91 of o11ycast, Ken Rimple and Jess Kerr sit down with Janaki Vivrekar. Janaki shares how Amplitude is building AI-powered analytics agents, why evaluation frameworks are becoming essential to AI product development, and how teams can use observability techniques to improve agent performance over time. The conversation explores eval-driven development, production feedback loops, and the challenges of helping AI systems reason about complex business data.",[],"42:06","https://cdn.sanity.io/images/50q6fr1p/production/3e9f4710b6abe02488e6cc34e8ff0d0fc29a41a7-3000x3000.png","a3d99e72-8291-4340-b0e3-926efee70139","2026-06-04T07:10:00.000Z","o11ycast/ep-91-every-failure-becomes-an-eval-with-janaki-vivrekar","O11ycast Ep. #91, Every Failure Becomes an Eval with Janaki Vivrekar",{"_316":355,"_326":364,"_149":365,"_329":366,"_331":367,"_333":368,"_32":369,"_336":337},[356],{"_37":357,"_23":297,"_298":358,"_306":363,"_308":309},"3cc9bfb7560b",[359],{"_37":360,"_23":302,"_303":361,"_174":362},"7b3ca11657b8",[],"On episode 11 of High Leverage, Joe Ruscio speaks with David Crawshaw about the shift from traditional developer infrastructure to agent-native computing. David traces lessons from founding Tailscale to now building exe.dev around the idea that “agents need a computer.” The conversation explores why VMs may be the right primitive for AI coding agents, why deployment friction becomes unacceptable when software can be generated in minutes, and how agent-driven development changes code review, production workflows, security, and the economics of personal software.",[],"01:08:47","https://cdn.sanity.io/images/50q6fr1p/production/8f17bb7e2365a02b5673a5eed27be04254c5d5b4-3000x3000.jpg","f5d4fea2-adcc-444c-a4c8-29c60d2d838d","2026-06-03T07:10:00.000Z","high-leverage/ep-11-why-agents-need-computers-with-david-crawshaw","High Leverage Ep. #11, Why Agents Need Computers with David Crawshaw","recent",[372,1107,1656],{"_316":373,"_326":1100,"_149":1101,"_329":1102,"_331":1103,"_333":1104,"_32":1105,"_336":1106},[374,383,414,422,472,476,491,499,518,537,553,561,591,599,607,615,634,653,669,677,685,694,702,710,718,737,772,780,804,823,831,839,855,863,871,886,894,902,910,929,953,979,987,1014,1022,1030,1038,1046,1054,1073,1092],{"_37":375,"_23":297,"_298":376,"_306":381,"_308":382},"7a24ac14d07e",[377],{"_37":378,"_23":302,"_303":379,"_174":380},"48d3443f20a8",[],"How Can Agents Be the Future if They’re not Reliable?",[],"h2",{"_37":384,"_23":297,"_298":385,"_306":408,"_308":309},"ff2d4b14fb40",[386,390,395,399,404],{"_37":387,"_23":302,"_303":388,"_174":389},"4b0676b5544d",[],"Some reports suggest that as many as ",{"_37":391,"_23":302,"_303":392,"_174":394},"7633cd943027",[393],"72307ba313ce","25%",{"_37":396,"_23":302,"_303":397,"_174":398},"103406d76f91",[]," of enterprises have adopted agentic workflows. Isn’t this the era of AI agents? Why isn’t the number higher? Aside from predictable enterprise-level concerns (",{"_37":400,"_23":302,"_303":401,"_174":403},"01325be45d21",[402],"c2f310fcffb7","49%",{"_37":405,"_23":302,"_303":406,"_174":407},"c145585f2175",[]," of enterprises are reportedly holding off due to security concerns, for instance), agents haven’t proven to be reliable workhorses at scale, particularly for long-horizon tasks. Not yet, anyway.",[409,412],{"_37":393,"_23":49,"_410":411},"href","https://snyk.io/blog/agentic-development-lifecycle/",{"_37":402,"_23":49,"_410":413},"https://www.forrester.com/blogs/the-state-of-agentic-ai-in-2026-companies-are-chasing-few-are-catching/",{"_37":415,"_23":297,"_298":416,"_306":421,"_308":309},"72359702785e",[417],{"_37":418,"_23":302,"_303":419,"_174":420},"c805aa2c35cb",[],"Agents still can’t “remember” their learnings from previous successful runs, still hallucinate when their context window becomes full, and still have a number of other issues. Wasn’t the perfect agentic future supposed to feature technical teams commanding fleets of infallible, well-trained agents to do their bidding?",[],{"_37":423,"_23":297,"_298":424,"_306":465,"_308":309},"24ebf08aefeb",[425,429,434,438,443,447,452,456,461],{"_37":426,"_23":302,"_303":427,"_174":428},"0008706ad2a9",[],"Open-source creator ",{"_37":430,"_23":302,"_303":431,"_174":433},"bfecdcdc2d8e",[432],"f4b6e64c3af0","Mike Hostetler",{"_37":435,"_23":302,"_303":436,"_174":437},"c55ab4189631",[]," believes such a future is possible if systems combine strong guardrails with fault-tolerant infrastructure designed for large-scale deployments. Here, he explains why he built his project, ",{"_37":439,"_23":302,"_303":440,"_174":442},"1b8ae31d51e4",[441],"b47c6c24e0ab","Jido",{"_37":444,"_23":302,"_303":445,"_174":446},"4ac8dc73cc44",[],", ",{"_37":448,"_23":302,"_303":449,"_174":451},"2f226b078804",[450],"em","not",{"_37":453,"_23":302,"_303":454,"_174":455},"7c9c7486bda0",[]," using Python, the de facto programming language for AI, but using ",{"_37":457,"_23":302,"_303":458,"_174":460},"be12332c52e1",[459],"f60c98fe6c05","Elixir",{"_37":462,"_23":302,"_303":463,"_174":464},"1edf6210f9e9",[],", which was inherently designed to support maintainability and massive concurrency.",[466,468,470],{"_37":432,"_23":49,"_410":467},"https://www.linkedin.com/in/mikehostetler/",{"_37":441,"_23":49,"_410":469},"https://github.com/agentjido/jido",{"_37":459,"_23":49,"_410":471},"https://elixir-lang.org/",{"_37":473,"_23":149,"_114":474},"4bd56dd08d69",{"_116":475,"_23":118},"image-734e0114ac063680123b3471dd5f4f588375e8fa-1280x591-jpg",{"_37":477,"_23":297,"_298":478,"_306":488,"_308":309},"b03c891ae9f1",[479,483],{"_37":480,"_23":302,"_303":481,"_174":482},"0c2b5b8ee499",[450],"Mike Hostetler discusses how AI is changing the business of building software on the Software Without Borders podcast. Image courtesy ",{"_37":484,"_23":302,"_303":485,"_174":487},"a2615c1217b2",[486,450],"ea2317db32a8","Software \nWithout Borders",[489],{"_37":486,"_23":49,"_410":490},"https://www.youtube.com/watch?v=zVYqx_FDTPU",{"_37":492,"_23":297,"_298":493,"_306":498,"_308":382},"378419cb6538",[494],{"_37":495,"_23":302,"_303":496,"_174":497},"b95d6349db96",[],"Resolving Nondeterministic LLMs and Deterministic Algorithms",[],{"_37":500,"_23":297,"_298":501,"_306":515,"_308":309},"87b25977462d",[502,506,511],{"_37":503,"_23":302,"_303":504,"_174":505},"2cb97e117942",[],"Hostetler, a veteran developer and founder, got his start working on the ",{"_37":507,"_23":302,"_303":508,"_174":510},"5df850edc9eb",[509],"e7748844a8dd","jQuery",{"_37":512,"_23":302,"_303":513,"_174":514},"e6f957aa936c",[]," ecosystem, from which he built a career, a startup, and a lifelong curiosity about technology. He explains that his introduction to Elixir came from a tour of duty working in distributed systems, which inspired him to think through the challenge of being able to run multiple agents with real-world reliability. “I envisioned a future of ‘10,000 agents per human.’”",[516],{"_37":509,"_23":49,"_410":517},"https://jquery.com",{"_37":519,"_23":297,"_298":520,"_306":534,"_308":309},"0609f26003a6",[521,525,530],{"_37":522,"_23":302,"_303":523,"_174":524},"11d3ae480877",[],"The creator notes that his original vision was running 10,000 agents on a single ",{"_37":526,"_23":302,"_303":527,"_174":529},"bd8a2854d8b0",[528],"f82f2e7f077d","Raspberry Pi",{"_37":531,"_23":302,"_303":532,"_174":533},"3efb34af94bf",[],". In other words, in a resource-constrained environment with no direct access to LLMs. Models were getting better, but capital expenditure from foundation labs continued to skyrocket.",[535],{"_37":528,"_23":49,"_410":536},"https://www.raspberrypi.com/",{"_37":538,"_23":297,"_298":539,"_306":552,"_308":309},"c7d5705ac245",[540,544,548],{"_37":541,"_23":302,"_303":542,"_174":543},"22d5ed497f6c",[],"Hostetler suggests his goal was to make the best of both worlds: Capitalize on the improving performance of state-of-the-art models, combined with performant infrastructure that wouldn’t break the bank. “The intersection of these two ideas is: Agents needed to be easy to build, easy to deploy, easy to orchestrate and coordinate. And then, 20% of your agents could use an LLM as the thinking step. But it needed to be just as easy to build 80% of your agents on what I call ",{"_37":545,"_23":302,"_303":546,"_174":547},"ae3aaa793b3f",[450],"classical AI algorithms",{"_37":549,"_23":302,"_303":550,"_174":551},"c8c502a31d12",[],".”",[],{"_37":554,"_23":297,"_298":555,"_306":560,"_308":309},"5e457368ab04",[556],{"_37":557,"_23":302,"_303":558,"_174":559},"ed9ee37193c9",[],"The builder clarifies that ‘classic algorithms’ also include logic that predates LLMs: “Everything from a finite state machine to a behavior tree to hierarchical task networks. Video game AI as well as LLM AI. And those could work in concert and then be easily and inexpensively deployable for a user, which gets into the real pragmatic piece.”",[],{"_37":562,"_23":297,"_298":563,"_306":586,"_308":309},"c843c70bed8d",[564,568,573,577,582],{"_37":565,"_23":302,"_303":566,"_174":567},"9425a3cff2d7",[],"Hostetler clarifies that despite Jido’s explicit “AI optional” clause, his approach to AI is less skeptical and more pragmatic. “I love the nondeterminism of LLMs. I love pushing those boundaries. Just by happenstance, having put my own open-source project out there, I got to know ",{"_37":569,"_23":302,"_303":570,"_174":572},"9a152e0af025",[571],"c4b6a4a077f8","Geoff Huntley",{"_37":574,"_23":302,"_303":575,"_174":576},"f8941b7433c0",[]," and tried ",{"_37":578,"_23":302,"_303":579,"_174":581},"b9826d2e81c5",[580],"cfe50a2150c7","the Ralph Wiggum loop",{"_37":583,"_23":302,"_303":584,"_174":585},"64bca5e784fe",[]," before it was publicly available. Loved it, and had Jido running loops.”",[587,589],{"_37":571,"_23":49,"_410":588},"https://ghuntley.com",{"_37":580,"_23":49,"_410":590},"https://ghuntley.com/loop/",{"_37":592,"_23":297,"_298":593,"_306":598,"_308":309},"103cd73a04dd",[594],{"_37":595,"_23":302,"_303":596,"_174":597},"9d7796132c04",[],"“But I quickly observed that [with successive loops], you end up compounding quality issues and you run into these challenges.” The builder’s observations led him to start experimenting with combinations of agentic loops and classical, deterministic AI algorithms, based on a self-taught course of study.",[],{"_37":600,"_23":297,"_298":601,"_306":606,"_308":309},"0291b7718207",[602],{"_37":603,"_23":302,"_303":604,"_174":605},"094ef0d4310f",[],"“I've used my computer science degree more in the last two years than I have in my entire career, which is just so much fun. So I’m not ‘anti-AI’ at all. What I want to put forward is something that lets you use both modes of tooling in the same context, instead of having a ‘developer mentality’ that requires a hard switch between ‘Here’s an agent that has an LLM as its brain’ versus an agent that has a finite state machine as its brain, and you have to build a bridge between the two. Jido makes both into first-class citizens. I knew as an engineer, as an architect, I would want both.”",[],{"_37":608,"_23":297,"_298":609,"_306":614,"_308":382},"8b84eea66041",[610],{"_37":611,"_23":302,"_303":612,"_174":613},"66b38a3b5c20",[],"How Can Agents Master Long-Horizon Jobs?",[],{"_37":616,"_23":297,"_298":617,"_306":631,"_308":309},"47dc737f6c1e",[618,622,627],{"_37":619,"_23":302,"_303":620,"_174":621},"867bc7833538",[],"Hostetler admits to reading, and speculating about, the news headlines about agents seemingly getting closer to the holy grail of succeeding on complicated, long-horizon tasks in a single, zero-shot run. “Yes, I read the headlines, and I see the ",{"_37":623,"_23":302,"_303":624,"_174":626},"06d1b38bf227",[625],"2ea1dd0e9cf9","METR reports",{"_37":628,"_23":302,"_303":629,"_174":630},"8b4acaa0ac87",[],". I would say I have hypotheses about the topic, but I acknowledge the limits of my hypotheses.”",[632],{"_37":625,"_23":49,"_410":633},"https://metr.org/time-horizons/",{"_37":635,"_23":297,"_298":636,"_306":650,"_308":309},"8d0acac52e79",[637,641,646],{"_37":638,"_23":302,"_303":639,"_174":640},"2c71dc2a8678",[],"“We see new model drops like the [recalled-and-re-released] ",{"_37":642,"_23":302,"_303":643,"_174":645},"45c1bbef0be9",[644],"f0386e204d9f","Fable 5",{"_37":647,"_23":302,"_303":648,"_174":649},"1fbdf1ad0724",[],", which can reportedly do a long-running task for eight hours, uninterrupted.” The creator notes that it can be hard to separate reality from hype, but notes that foundation labs may have different incentives than the rest of us.",[651],{"_37":644,"_23":49,"_410":652},"https://www.anthropic.com/news/fable-mythos-access",{"_37":654,"_23":297,"_298":655,"_306":668,"_308":309},"a2cf64eff7a9",[656,660,664],{"_37":657,"_23":302,"_303":658,"_174":659},"eaf492689fdb",[],"“I would think there is a lot of pressure to embed the capability ",{"_37":661,"_23":302,"_303":662,"_174":663},"92183232c806",[450],"behind the walls",{"_37":665,"_23":302,"_303":666,"_174":667},"8058aab96629",[]," of the model; to make Fable better via academic RL, or to have a lot of infrastructure running between the API call and the model itself. You see them pulling in things like their web API tools, and doing everything to make Anthropic a walled garden for any corporate customer.”",[],{"_37":670,"_23":297,"_298":671,"_306":676,"_308":309},"41e706c870c8",[672],{"_37":673,"_23":302,"_303":674,"_174":675},"14797e507230",[],"Hostetler contrasts the lofty ambitions of foundation labs against his day-to-day work, which requires him to service ERP systems maintained by tech leaders across the country who need reliable performance, day in and day out.",[],{"_37":678,"_23":297,"_298":679,"_306":684,"_308":309},"c04dd4c136d2",[680],{"_37":681,"_23":302,"_303":682,"_174":683},"840ee6f95b58",[],"The creator suggests starting from a strong harness, but also looking into reliability. “A really concrete step is to start with a very high-quality model harness if I'm going to build my own agent. Yes, admittedly, Elixir’s philosophy is ‘let it crash,’ but you don't really want that to happen in practice. Your focus should be: How do you recover from the crash? How do you deliver high-quality software from that?”",[],{"_37":686,"_23":297,"_298":687,"_306":692,"_308":693},"5824b8efe53d",[688],{"_37":689,"_23":302,"_303":690,"_174":691},"16d38849b2d4",[],"“Elixir’s philosophy is ‘let it crash,’ but you don't really want that to happen in practice. Your focus should be: How do you recover from the crash? How do you deliver high-quality software from that?” -Mike Hostetler, Creator/Jido",[],"blockquote",{"_37":695,"_23":297,"_298":696,"_306":701,"_308":382},"5959d9db324a",[697],{"_37":698,"_23":302,"_303":699,"_174":700},"9adc46734820",[],"Squaring the Circle: Autonomous Agents vs. Reliable Uptime",[],{"_37":703,"_23":297,"_298":704,"_306":709,"_308":309},"b4fc4a69d6f3",[705],{"_37":706,"_23":302,"_303":707,"_174":708},"e958458240eb",[],"The builder recognizes the gap between LLM-powered agents that often fall down for a variety of reasons and Elixir-based systems like Discord and Pinterest that are expected to provide continuous uptime for millions of users. “Without getting too technical, this really was a key design decision of Jido itself.”",[],{"_37":711,"_23":297,"_298":712,"_306":717,"_308":309},"40ebc9c9d5ad",[713],{"_37":714,"_23":302,"_303":715,"_174":716},"9b48defee678",[],"Hostetler explains that the project was designed to accommodate both builders of independent, solo agents as well as those building for coordinated, multi-agent use cases. “And also, there’s all the infrastructure concerns for when you go to deploy: Where does it live? How is it persisted? It's persisted with what I consider to be two schools of thought.”",[],{"_37":719,"_23":297,"_298":720,"_306":734,"_308":309},"01a202626a6c",[721,725,730],{"_37":722,"_23":302,"_303":723,"_174":724},"6a69eed46b2a",[],"“The first [persistence story] is that I persist my ",{"_37":726,"_23":302,"_303":727,"_174":729},"be252cd6a312",[728],"e73f4b0caaf3","BEAM system",{"_37":731,"_23":302,"_303":732,"_174":733},"888079506d54",[]," back to a durable store, like a Postgres database which is centralized. No issue there. And I would say that more than half our users use this. The other key story is what I call the fully distributed, fault-tolerant approach for scale that you get into with the really big deployments, such as WhatsApp or Facebook Messenger. Multi-region, multi-clustered...the big stuff.”",[735],{"_37":728,"_23":49,"_410":736},"https://en.wikipedia.org/wiki/BEAM_(Erlang_virtual_machine)",{"_37":738,"_23":297,"_298":739,"_306":769,"_308":309},"52380ad616db",[740,744,748,752,756,760,765],{"_37":741,"_23":302,"_303":742,"_174":743},"04e742e52dfa",[],"“Jido was geared for [large-scale deployments]. We could get into the specifics of how Jido was created to live within the BEAM itself, but it ",{"_37":745,"_23":302,"_303":746,"_174":747},"7f2912ffef49",[450],"is",{"_37":749,"_23":302,"_303":750,"_174":751},"f38721e57e56",[]," possible. And there are nuances here, too. People come to the project and they sometimes feel like it's overbuilt. Case in point, when you send a message into a Jido agent, we use a dedicated message envelope called a ",{"_37":753,"_23":302,"_303":754,"_174":755},"360df35b7e3c",[450],"Jido signal",{"_37":757,"_23":302,"_303":758,"_174":759},"51665ddc50c6",[],", which, when you're in a single-node use case is really and truly overkill. And yet it's the standard built on the ",{"_37":761,"_23":302,"_303":762,"_174":764},"152bbc74099b",[763],"0abd3bbf8418","CloudEvents spec",{"_37":766,"_23":302,"_303":767,"_174":768},"ca903e998961",[]," (which is independent of, and exists outside of Elixir).”",[770],{"_37":763,"_23":49,"_410":771},"https://cloudevents.io/",{"_37":773,"_23":297,"_298":774,"_306":779,"_308":309},"46faea6fef5d",[775],{"_37":776,"_23":302,"_303":777,"_174":778},"e670c10a6d1b",[],"“We do this because you can't have it both ways: You can't have your typical simple JSON payload alongside a robust, enterprise-grade, transport-agnostic CloudEvents envelope and do the same thing. You've got to pick a lane. And so, Jido picked the ‘robust and scalable’ lane. I do work with some teams that are now asking me about the multi-cluster use case. I’m working toward a demo of a multi-region cluster independent of a durable store with resilient agents.”",[],{"_37":781,"_23":297,"_298":782,"_306":803,"_308":309},"8a6207353488",[783,787,791,795,799],{"_37":784,"_23":302,"_303":785,"_174":786},"775967e75006",[],"“To clarify, everything I just discussed was ",{"_37":788,"_23":302,"_303":789,"_174":790},"5cfaaca93e88",[450],"outside",{"_37":792,"_23":302,"_303":793,"_174":794},"97a4f29363fe",[]," the agentic loop. ",{"_37":796,"_23":302,"_303":797,"_174":798},"accaff227757",[450],"Within",{"_37":800,"_23":302,"_303":801,"_174":802},"83672844a0e3",[]," the agentic loop of determinism versus nondeterminism, there are guardrails within which Jido has to operate. It can't just go do everything because it was meant for the Raspberry Pi use case. Because Elixir and BEAM can natively cluster, you could easily have a deploy to support a use case for something like, let’s say, Internet of Things (IoT).”",[],{"_37":805,"_23":297,"_298":806,"_306":820,"_308":309},"36dc5918428c",[807,811,816],{"_37":808,"_23":302,"_303":809,"_174":810},"f9f53a4cd67b",[],"“For example, you’re going to have a central cluster that then works with Jido agents that are deployed in an IoT setting that talk back to the central cluster, and you'd have two agents communicating with one another. You could have that modality in a larger deployment. As an example, one of the big successes in the Elixir world is the robot control project ",{"_37":812,"_23":302,"_303":813,"_174":815},"30ba83fa35f4",[814],"74bfc2d507f3","Beam Bots",{"_37":817,"_23":302,"_303":818,"_174":819},"ac05df5ad6aa",[],", which adopted Jido natively, so we’re seeing use cases for non-clustered low-level IoT all the way up to your typical Claude Code agent harness.”",[821],{"_37":814,"_23":49,"_410":822},"https://github.com/beam-bots",{"_37":824,"_23":297,"_298":825,"_306":830,"_308":382},"6471f5331431",[826],{"_37":827,"_23":302,"_303":828,"_174":829},"88eb6436d108",[],"How Startup Founders Should Think About Agentic Products",[],{"_37":832,"_23":297,"_298":833,"_306":838,"_308":309},"b02dc08a377e",[834],{"_37":835,"_23":302,"_303":836,"_174":837},"71c2612c69e4",[],"When asked how he would advise startup founders thinking of launching agentic products, Hostetler offers a handful of suggestions. “For your first level of planning, I would say: Focus on the default. How you define what an ‘agent’ is could be colored by your Claude Code glasses! Make sure that as you're defining an agent in terms of what you bring into your infrastructure, and that you have a tighter, concrete definition of what that is.”",[],{"_37":840,"_23":297,"_298":841,"_306":854,"_308":309},"322cd96a4729",[842,846,850],{"_37":843,"_23":302,"_303":844,"_174":845},"a7c7ebbd0554",[],"“Second, the other mistake that I've seen (and have made personally) is that everybody wants to let the LLM ‘take the wheel.’ At the point when you're designing infrastructure to build a company on, you have to draw clear boundaries between nondeterminism and determinism, and create gates to bring it back and forth. And this is ",{"_37":847,"_23":302,"_303":848,"_174":849},"1ea4bce42d9f",[450],"outside of",{"_37":851,"_23":302,"_303":852,"_174":853},"9510224427a1",[]," the agentic loop.”",[],{"_37":856,"_23":297,"_298":857,"_306":862,"_308":309},"e80672b9d929",[858],{"_37":859,"_23":302,"_303":860,"_174":861},"61e9e3aed2a8",[],"The creator suggests that customer expectations for agentic projects have grown over time. “After a few years, we’ve seen the patterns that ‘work,’ whether it's natural language or dynamic UI, we know the patterns to create user value and what users expect. The innovation has shifted from, ‘We can do just about anything,’ to ‘The user expectations are pretty much there, so how do you deliver on that?’”",[],{"_37":864,"_23":297,"_298":865,"_306":870,"_308":309},"32013237d35f",[866],{"_37":867,"_23":302,"_303":868,"_174":869},"95b20553d1ab",[],"Hostetler describes an interesting project a friend built in Elixir that effectively replicated Claude Code’s functionality, but for posting on LinkedIn, using a dedicated agent with a specific ID. “Persistence wasn't dynamically allocated, but the agent would persist, and then it would hibernate and rehydrate when you wanted to edit a piece of content.”",[],{"_37":872,"_23":297,"_298":873,"_306":885,"_308":309},"3309765c1194",[874,878,882],{"_37":875,"_23":302,"_303":876,"_174":877},"6b38e3208d9b",[],"The project let users edit posts normally via the LinkedIn GUI, but also supported giving instructions to a coordinator agent. “The key is, he made that persistent throughout the entire experience. So, no matter which modality you were interacting with, there was a kind of predefined team structure for how the agents would interact with your content. And it just struck me as this really innovative mental leap, that any agentic products I see ",{"_37":879,"_23":302,"_303":880,"_174":881},"302d71f6b276",[450],"aren't thinking big enough",{"_37":883,"_23":302,"_303":884,"_174":551},"5e2c8479951c",[],[],{"_37":887,"_23":297,"_298":888,"_306":893,"_308":309},"f9024ac9c07d",[889],{"_37":890,"_23":302,"_303":891,"_174":892},"090d9b48e200",[],"The creator suggests that the main challenge he’d offer to startup founders and product owners is to think bigger. “I don't see a lot of product owners thinking about [larger ideas] because they're really stuck in the mindset of, ‘Claude Code is my agent. It can do everything. I have to always use Claude Code.’ I say: ‘No, you really need to break out of that mental model.’”",[],{"_37":895,"_23":297,"_298":896,"_306":901,"_308":693},"d279075f7c24",[897],{"_37":898,"_23":302,"_303":899,"_174":900},"1407b21e0ca6",[],"“It just struck me...that any agentic products I see aren't thinking big enough. I don't see a lot of product owners thinking about [larger ideas] because they're really stuck in the mindset of, ‘Claude Code is my agent. I say: ‘No, you really need to break out of that mental model.’”",[],{"_37":903,"_23":297,"_298":904,"_306":909,"_308":382},"8bd22d228d02",[905],{"_37":906,"_23":302,"_303":907,"_174":908},"febccbe3dad6",[],"How to Think About Enterprise Adoption",[],{"_37":911,"_23":297,"_298":912,"_306":926,"_308":309},"55d44d50b054",[913,917,922],{"_37":914,"_23":302,"_303":915,"_174":916},"140634326dd3",[],"When asked how he would advise ambitious startup founders looking to sell AI products to enterprises, Hostetler draws upon his own day-to-day experience. “I would say that we're early. I would acknowledge the timeline displacement, ",{"_37":918,"_23":302,"_303":919,"_174":921},"60c8c804bc83",[920],"395e4914b63e","Ray Kurzweil’s",{"_37":923,"_23":302,"_303":924,"_174":925},"940cd7aa0849",[]," riff on the William Gibson quote about the future being here, just not evenly distributed.”",[927],{"_37":920,"_23":49,"_410":928},"https://www.writingsbyraykurzweil.com/the-law-of-accelerating-returns",{"_37":930,"_23":297,"_298":931,"_306":952,"_308":309},"69e3ce55443f",[932,936,940,944,948],{"_37":933,"_23":302,"_303":934,"_174":935},"c2cbce688847",[],"“What's happening in enterprises is that there are engineering teams who ",{"_37":937,"_23":302,"_303":938,"_174":939},"8f43979ee05a",[450],"are ",{"_37":941,"_23":302,"_303":942,"_174":943},"7386ba0066ca",[],"using the latest tools, and they're on that curve somewhere. But ",{"_37":945,"_23":302,"_303":946,"_174":947},"80dac42e9497",[450],"the operations teams",{"_37":949,"_23":302,"_303":950,"_174":951},"f3f8661263be",[]," are just starting.” The creator notes that he works with operations teams that are just starting to encounter the same challenges that engineering teams faced 18 months ago: Worrying whether AI will replace them, realizing AI likely won’t replace them, then trying to figure out how to fit into this brave new world.",[],{"_37":954,"_23":297,"_298":955,"_306":976,"_308":309},"d6bdea3ba045",[956,960,965,969,973],{"_37":957,"_23":302,"_303":958,"_174":959},"6dbb3fee6f18",[],"“We're early in that phase [where people are trying to find] pragmatic use cases. The issue isn't so much the tools themselves. What's interesting to me is the layer above them, which I wrote about in ",{"_37":961,"_23":302,"_303":962,"_174":964},"e44831398a0a",[963],"ba5da2cbf968","my blog on orchestration",{"_37":966,"_23":302,"_303":967,"_174":968},"3c012b7e05ac",[],". To me, orchestration is the cornerstone, because there's nothing actually orchestrating ",{"_37":970,"_23":302,"_303":971,"_174":972},"c493d94c2769",[450],"the tools",{"_37":974,"_23":302,"_303":975,"_174":551},"0ddedf4c18ed",[],[977],{"_37":963,"_23":49,"_410":978},"https://mike-hostetler.com/blog/orchestrating-intelligence-why-every-company-will-have-a-chief-agent-officer-y4EQpjr9",{"_37":980,"_23":297,"_298":981,"_306":986,"_308":309},"ffdaf5b669b5",[982],{"_37":983,"_23":302,"_303":984,"_174":985},"d9fbf1dd4f20",[],"“I think that corporations are set up in silos. Sales isn't talking to marketing, who isn't talking to ops. Ops isn't talking to product. So, the pragmatic things that we need are places to deploy AI experiments, because the thing we've emphasized internally isn't so much AI as an end state, but rather, AI as iteration.”",[],{"_37":988,"_23":297,"_298":989,"_306":1011,"_308":309},"86f97ec55b10",[990,994,999,1003,1007],{"_37":991,"_23":302,"_303":992,"_174":993},"9aad627e9643",[],"The creator recounts a story of a team he works with using Claude Code to build a dashboard. In a completely unintended consequence, Claude Code used the Google Workspace connect to deploy the dashboard to Google Sheets via ",{"_37":995,"_23":302,"_303":996,"_174":998},"865280795f71",[997],"2ed51292aa7f","Apps Script",{"_37":1000,"_23":302,"_303":1001,"_174":1002},"4e0143f6f2a2",[]," to create a public URL via, of all things, a Google Sheets backdoor. “I burst out laughing. There's an entire product ",{"_37":1004,"_23":302,"_303":1005,"_174":1006},"25776a76f3df",[450],"right there,",{"_37":1008,"_23":302,"_303":1009,"_174":1010},"8a472903f3c6",[]," in that space between a non-technical user and a model’s instincts to propose unexpected solutions.”",[1012],{"_37":997,"_23":49,"_410":1013},"https://developers.google.com/apps-script/overview",{"_37":1015,"_23":297,"_298":1016,"_306":1021,"_308":309},"1afc27d42096",[1017],{"_37":1018,"_23":302,"_303":1019,"_174":1020},"19bdb5bfad0f",[],"Hostetler also points out a strong business opportunity in governance. “During my tour of duty in corporate, I spent two years as a VP of enterprise technology. I managed all the IT and compliance stuff for a publicly-traded company.” The creator notes that “AI governance” could easily go beyond basic compliance frameworks like SOC 2.",[],{"_37":1023,"_23":297,"_298":1024,"_306":1029,"_308":309},"8fe6ad610c6c",[1025],{"_37":1026,"_23":302,"_303":1027,"_174":1028},"9e450b53590e",[],"“Every LLM call in your organization will eventually be proxied and logged. But nobody's doing this at the moment. Think about just capturing it as one problem. Analyzing and fingerprinting.” The creator predicts that sooner than we think, organizations may need to be “AI compliant” with whatever new version of the SOC2 standard that arises.",[],{"_37":1031,"_23":297,"_298":1032,"_306":1037,"_308":309},"ef3eab6b09a3",[1033],{"_37":1034,"_23":302,"_303":1035,"_174":1036},"856e44ea47e3",[],"“Another [enterprise use case] I’d look at is insurance. Cybersecurity insurance with AI, mark my words, in the next two years, that will be a ‘thing.’ And this won’t just be your CrowdStrike-style, on-device stuff. It's all the things that get exfiltrated from what you build. Nobody's talking about that. The strategy there is very immature. Admittedly, this isn’t a very sexy space, but I think there's a lot of money to be made there.”",[],{"_37":1039,"_23":297,"_298":1040,"_306":1045,"_308":382},"8c5c46a705f3",[1041],{"_37":1042,"_23":302,"_303":1043,"_174":1044},"e55bcf804130",[],"The Future of Elixir and Durable Systems",[],{"_37":1047,"_23":297,"_298":1048,"_306":1053,"_308":309},"44f07555baa8",[1049],{"_37":1050,"_23":302,"_303":1051,"_174":1052},"83aa3f705283",[],"The creator is realistic about how his project and the greater Elixir ecosystem figure into the future of software development. “A non-Elixir shop is going to have a hard time adopting Elixir. So I can see Jido becoming an agentic sidecar that eventually makes it easy for TypeScript devs to spin up an agent with Elixir-level durability.”",[],{"_37":1055,"_23":297,"_298":1056,"_306":1070,"_308":309},"dbad934515e7",[1057,1061,1066],{"_37":1058,"_23":302,"_303":1059,"_174":1060},"ddf9d15e994c",[],"“This might be for a market segment that could be ‘one step below’ the ",{"_37":1062,"_23":302,"_303":1063,"_174":1065},"7929d2ffed1a",[1064],"f82cbe3cdbf8","Temporals",{"_37":1067,"_23":302,"_303":1068,"_174":1069},"82c9fc0d6074",[]," of the world. To clarify, I’ve used Temporal before, and it’s fantastic, though expensive. But thinking pragmatically, there are a lot more TypeScript devs in the world. Python will be its own thing. TypeScript devs are going to need an SDK to easily call Jido agents, and then Jido agents will operate async, and it'll just be self-contained with a dashboard to spin it up.”",[1071],{"_37":1064,"_23":49,"_410":1072},"https://temporal.io/",{"_37":1074,"_23":297,"_298":1075,"_306":1089,"_308":309},"b3f069861ce0",[1076,1080,1085],{"_37":1077,"_23":302,"_303":1078,"_174":1079},"7dc862f60a3b",[],"The creator acknowledges that popularly-used languages and frameworks like TypeScript and ",{"_37":1081,"_23":302,"_303":1082,"_174":1084},"a5ab9089e32a",[1083],"cc95782eb602","Node.js",{"_37":1086,"_23":302,"_303":1087,"_174":1088},"05d94911aeac",[]," are excellent for what they are, and that they solve specific problems. “But you need to be able to spin things up inexpensively that companies built on TypeScript can solve either by running in Docker via their infrastructure, or run out of a cloud where they can spin up platforms of agents, with one agent per customer (or dozens of agents per customer).”",[1090],{"_37":1083,"_23":49,"_410":1091},"http://node.js",{"_37":1093,"_23":297,"_298":1094,"_306":1099,"_308":309},"1c459bacc235",[1095],{"_37":1096,"_23":302,"_303":1097,"_174":1098},"4773952f0d8b",[],"“And to be able to do that practically is just going to be overwhelmingly expensive, even with Cloud costs. And you need to be able to stitch it all together, potentially using non-deterministic agents with deterministic agents to deliver a strong product experience. It’s a very early picture, but that's where I see the market now.”",[],19,"https://cdn.sanity.io/images/50q6fr1p/production/a119969ab98a18b0937559865cbc6ee4dee53c2d-1200x630.png","39be2ceb-4600-40c4-80a9-dc0c7b9504dd","2026-08-04T16:24:00.000Z","how-to-make-agents-durable-for-concurrent-systems","How to Make Agents Durable for Concurrent Systems","blogArticle",{"_316":1108,"_326":1650,"_149":1651,"_329":1652,"_331":1653,"_333":1654,"_32":1655,"_336":1106},[1109,1161,1169,1177,1185,1193,1201,1209,1217,1225,1244,1251,1260,1267,1275,1310,1318,1374,1382,1390,1398,1440,1482,1513,1545,1607,1615,1623,1642],{"_37":1110,"_23":297,"_298":1111,"_306":1152,"_308":309},"dafff4cce2a3",[1112,1116,1121,1125,1130,1134,1139,1143,1148],{"_37":1113,"_23":302,"_303":1114,"_174":1115},"00ea291ee56d",[],"Not every employee is a superstar. In large organizations, you will encounter order takers. Despite targeted coaching, repeated instruction, and close evaluation, order takers require constant supervision, learn slowly, and fail to internalize the goals (org, team, and personal) that they’re meant to carry to their daily tasks. Powered by language models and the harnesses around them, AI agents are touted as a way to replace order takers. Companies like ",{"_37":1117,"_23":302,"_303":1118,"_174":1120},"5fef21b3f6d6",[1119],"50828daca8fb","Oracle",{"_37":1122,"_23":302,"_303":1123,"_174":1124},"4afa264f877b",[],",",{"_37":1126,"_23":302,"_303":1127,"_174":1129},"df4a5d5afd5a",[1128],"5f61fc51253e"," GitLab",{"_37":1131,"_23":302,"_303":1132,"_174":1133},"b6287376d76e",[],", and ",{"_37":1135,"_23":302,"_303":1136,"_174":1138},"fa056cb9bd29",[1137],"5a27ccc6caa1","Elastic",{"_37":1140,"_23":302,"_303":1141,"_174":1142},"24cb5eb1e249",[]," are enacting huge AI-related layoffs and the numbers ",{"_37":1144,"_23":302,"_303":1145,"_174":1147},"a498f882385b",[1146],"5de35999d553","just keep growing",{"_37":1149,"_23":302,"_303":1150,"_174":1151},"169c35076cdf",[],".",[1153,1155,1157,1159],{"_37":1119,"_23":49,"_410":1154},"https://www.cnbc.com/2026/06/23/oracle-ai-job-cuts-layoffs-21000.html",{"_37":1128,"_23":49,"_410":1156},"https://www.wsj.com/business/earnings/gitlab-to-cut-14-of-workforce-as-part-of-ai-pivot-493b9813",{"_37":1137,"_23":49,"_410":1158},"https://www.elastic.co/blog/ceo-ash-kulkarni-announcement-to-elastic-employees",{"_37":1146,"_23":49,"_410":1160},"https://layoffs.fyi/",{"_37":1162,"_23":297,"_298":1163,"_306":1168,"_308":309},"1b3c23662662",[1164],{"_37":1165,"_23":302,"_303":1166,"_174":1167},"3ef0a1311459",[],"",[],{"_37":1170,"_23":297,"_298":1171,"_306":1176,"_308":309},"d10a347bc4ea",[1172],{"_37":1173,"_23":302,"_303":1174,"_174":1175},"dff6b4741678",[],"But despite the promises of a pure AI utopia, we still can’t trust agents to work towards goals. Most agents require an order-taker’s level of supervision, coaxing, and constant prompt-response. A human manager must still own a goal, break it into tasks, give the models context for each task, and triple check outputs and next steps. Even when an agent uses tools and skills across multiple tasks, a human for the most part, is still responsible for goals and outcomes.",[],{"_37":1178,"_23":297,"_298":1179,"_306":1184,"_308":382},"d104f15573fe",[1180],{"_37":1181,"_23":302,"_303":1182,"_174":1183},"dcbbac58dc27",[],"From Order Takers to Outcome Owners",[],{"_37":1186,"_23":297,"_298":1187,"_306":1192,"_308":309},"4ce2d7e4753e",[1188],{"_37":1189,"_23":302,"_303":1190,"_174":1191},"2e670174faee",[],"A task has a clear input and output. A tool can help complete it. But intelligent humans are not hired simply to complete a collection of tasks. They are given an area of responsibility and expected to deliver against goals or outcomes.",[],{"_37":1194,"_23":297,"_298":1195,"_306":1200,"_308":309},"6b88bc55d5c0",[1196],{"_37":1197,"_23":302,"_303":1198,"_174":1199},"6372aa4941fb",[],"Take a software engineer. Refactoring a module or fixing compile-time errors are tasks. Making sure the backend scales with seasonal traffic patterns while minimizing cloud spend is an outcome. The engineer needs to understand the architecture, decide what needs to change (if anything), weigh tradeoffs, sequence the work, then monitor outcomes and adjust the plan when they find something unexpected. To understand seasonality, they might study historical trends or ask others in the organization. As new patterns emerge, they update their understanding and remember it to avoid future pain.",[],{"_37":1202,"_23":297,"_298":1203,"_306":1208,"_308":309},"485a1e312f8a",[1204],{"_37":1205,"_23":302,"_303":1206,"_174":1207},"7ff860a00209",[],"Consider a sales rep. Writing a follow-up email after a sales call and logging it in the CRM are tasks. Generating $500K in qualified pipeline is an outcome. The rep needs to understand the customer needs and product value, identify and target accounts, personalize outreach, handle objections and find patterns towards a quick close. If the pipeline runs dry, they need to diagnose issues and determine whether it’s their own missteps, or an issue with another team’s outputs, such as product or engineering.",[],{"_37":1210,"_23":297,"_298":1211,"_306":1216,"_308":309},"e119119cdf54",[1212],{"_37":1213,"_23":302,"_303":1214,"_174":1215},"ae3c0219ad08",[],"Current agents are very good at completing tasks, especially when the output can be verified immediately. But they are surprisingly bad at making informed decisions over long time horizons.",[],{"_37":1218,"_23":297,"_298":1219,"_306":1224,"_308":382},"9658f3be24c9",[1220],{"_37":1221,"_23":302,"_303":1222,"_174":1223},"ddb2a9924c0d",[],"Longer Context Is not Enough",[],{"_37":1226,"_23":297,"_298":1227,"_306":1241,"_308":309},"5da59480b4f2",[1228,1232,1237],{"_37":1229,"_23":302,"_303":1230,"_174":1231},"3ca1621e6a86",[],"Most agents implement some variation of ",{"_37":1233,"_23":302,"_303":1234,"_174":1236},"94850813aa1c",[1235],"e2235117d2d9","ReAct",{"_37":1238,"_23":302,"_303":1239,"_174":1240},"56d750040c30",[],": the model reasons about what to do, takes an action, observes how the world responds, and reasons again. For a while, practitioners believed the path toward long-horizon agents looked straightforward:",[1242],{"_37":1235,"_23":49,"_410":1243},"https://arxiv.org/abs/2210.03629",{"_37":1245,"_23":297,"_298":1246,"_306":1250,"_308":309},"b6d0ee05c960",[1247],{"_37":1248,"_23":302,"_303":1249,"_174":1167},"646e76908063",[],[],{"_37":1252,"_23":297,"_298":1253,"_306":1258,"_308":1259},"241c35d79b95",[1254],{"_37":1255,"_23":302,"_303":1256,"_174":1257},"4d575ebedbc0",[450],"model + tools + loop + infinite context = autonomous agent",[],"h6",{"_37":1261,"_23":297,"_298":1262,"_306":1266,"_308":1259},"8eaac035be12",[1263],{"_37":1264,"_23":302,"_303":1265,"_174":1167},"bc5f0a12e5ca",[],[],{"_37":1268,"_23":297,"_298":1269,"_306":1274,"_308":309},"f9a296aca6d2",[1270],{"_37":1271,"_23":302,"_303":1272,"_174":1273},"1dcd8ef751fb",[],"The corollary was that if the context window became large enough, the agent could keep every prior thought, action, and result in view, loop forever, and learn in context.",[],{"_37":1276,"_23":297,"_298":1277,"_306":1305,"_308":309},"f4cb1575bf9c",[1278,1283,1287,1292,1296,1301],{"_37":1279,"_23":302,"_303":1280,"_174":1282},"11338ed82451",[1281],"strong","This is dead wrong. More time = more mistakes.",{"_37":1284,"_23":302,"_303":1285,"_174":1286},"76aab952ad27",[]," Recent studies find a ",{"_37":1288,"_23":302,"_303":1289,"_174":1291},"47944c12a591",[1290],"73c0eba6c540","30% accuracy drop",{"_37":1293,"_23":302,"_303":1294,"_174":1295},"ce67468cd0b0",[]," when long-context models are asked to retain information across sustained interactions. Over longer horizons, agents also suffer from what researchers call ",{"_37":1297,"_23":302,"_303":1298,"_174":1300},"02037c5debdd",[1299],"200d2886c5d2","\"behavioral state decay\"",{"_37":1302,"_23":302,"_303":1303,"_174":1304},"a8fe1f37c2a0",[],": requirements, prior attempts, diagnoses and open sub-goals begin as priorities, but at some point stop influencing the agent's decisions.",[1306,1308],{"_37":1290,"_23":49,"_410":1307},"https://arxiv.org/abs/2410.10813",{"_37":1299,"_23":49,"_410":1309},"https://arxiv.org/abs/2607.08716",{"_37":1311,"_23":297,"_298":1312,"_306":1317,"_308":309},"5655df795471",[1313],{"_37":1314,"_23":302,"_303":1315,"_174":1316},"5b7375946540",[],"Long context gives an agent a larger transcript. It does not decide what remains important, which beliefs should be revised, or whether the current plan still serves the original goal. Essentially, long context just offers an agent more time to make mistakes and lose the plot.",[],{"_37":1319,"_23":297,"_298":1320,"_306":1365,"_308":309},"3111ae9b30a1",[1321,1325,1329,1334,1338,1343,1347,1352,1356,1361],{"_37":1322,"_23":302,"_303":1323,"_174":1324},"124c073eac6a",[1281],"This forgetfulness is expensive. ",{"_37":1326,"_23":302,"_303":1327,"_174":1328},"c7ed9382db06",[],"Repeatedly looping agent jobs also incurs higher token costs. We’re seeing increasing concerns from orgs like ",{"_37":1330,"_23":302,"_303":1331,"_174":1333},"ac43738a1fd3",[1332],"41d2e7a50a6e","Uber",{"_37":1335,"_23":302,"_303":1336,"_174":1337},"a1fddaaffe44",[]," and ",{"_37":1339,"_23":302,"_303":1340,"_174":1342},"39c4a622e066",[1341],"8abe03c81caf","ServiceNow",{"_37":1344,"_23":302,"_303":1345,"_174":1346},"86dd0f0abcec",[]," that have burned through annual token budgets in a matter of months as the ",{"_37":1348,"_23":302,"_303":1349,"_174":1351},"cbceefb2e48e",[1350],"7ff16473a6bd","“free lunch” effect from tokenmaxxing",{"_37":1353,"_23":302,"_303":1354,"_174":1355},"3372bcf7910f",[]," has come to an end. Token costs have gotten so bad, that there’s a growing trend of ",{"_37":1357,"_23":302,"_303":1358,"_174":1360},"d45fe71a3bb8",[1359],"6a719821562f","inference theft",{"_37":1362,"_23":302,"_303":1363,"_174":1364},"63d5e691f87c",[]," leading to eye-opening losses via public gateways. What organizations need to make agents live up to the hype is the ability to build and run them over long-horizon jobs without breaking the bank.",[1366,1368,1370,1372],{"_37":1332,"_23":49,"_410":1367},"https://www.forbes.com/sites/janakirammsv/2026/05/17/uber-burns-its-2026-ai-budget-in-four-months-on-claude-code/",{"_37":1341,"_23":49,"_410":1369},"https://x.com/LauraBratton5/status/2054558503436755350",{"_37":1350,"_23":49,"_410":1371},"https://apnews.com/article/ai-token-openai-anthropic-corporate-31bb80ac1cd7862d05f6397177d826b1",{"_37":1359,"_23":49,"_410":1373},"https://vercel.com/blog/protecting-against-token-theft",{"_37":1375,"_23":297,"_298":1376,"_306":1381,"_308":382},"7ff082431161",[1377],{"_37":1378,"_23":302,"_303":1379,"_174":1380},"802c220994fe",[],"The Five Missing Systems",[],{"_37":1383,"_23":297,"_298":1384,"_306":1389,"_308":309},"df1fbd802629",[1385],{"_37":1386,"_23":302,"_303":1387,"_174":1388},"ed09e8ee172e",[],"Long-horizon agency isn’t as simple as a model running in a loop for a long time. We’re missing critical systems that provide continuity of intelligent behavior across changing evidence, context boundaries, process failures and learning cycles. We’re just not there yet, but I’m hopeful.",[],{"_37":1391,"_23":297,"_298":1392,"_306":1397,"_308":309},"25b7051401c6",[1393],{"_37":1394,"_23":302,"_303":1395,"_174":1396},"62657fd39dbc",[],"We believe continuity depends on five systems:",[],{"_37":1399,"_23":297,"_298":1400,"_306":1433,"_308":309},"5ec1eef1502a",[1401,1405,1409,1414,1417,1422,1425,1430],{"_37":1402,"_23":302,"_303":1403,"_174":1404},"7692a01e007d",[1281],"Organizational memory.",{"_37":1406,"_23":302,"_303":1407,"_174":1408},"9083d47bde42",[]," Most agent memory is scoped to a single session. Long-horizon agents need useful knowledge to survive beyond that session, evolve as facts change and become available to other agents in the organization. Examples of companies working here include ",{"_37":1410,"_23":302,"_303":1411,"_174":1413},"3e6e27c22a37",[1412],"d6ace36b3275","Mem0",{"_37":1415,"_23":302,"_303":1416,"_174":446},"5b230b30b1dd",[],{"_37":1418,"_23":302,"_303":1419,"_174":1421},"eb1e6d2c6028",[1420],"9cd93ba548dc","Mubit",{"_37":1423,"_23":302,"_303":1424,"_174":1133},"e48e505667b3",[],{"_37":1426,"_23":302,"_303":1427,"_174":1429},"637d49e4e96b",[1428],"382328d5fa57","Antfly",{"_37":1431,"_23":302,"_303":1432,"_174":1151},"0a8e1b752568",[],[1434,1436,1438],{"_37":1412,"_23":49,"_410":1435},"https://mem0.ai/",{"_37":1420,"_23":49,"_410":1437},"https://mubit.ai/",{"_37":1428,"_23":49,"_410":1439},"https://antfly.io/",{"_37":1441,"_23":297,"_298":1442,"_306":1475,"_308":309},"4b79d32309d4",[1443,1447,1451,1456,1459,1464,1467,1472],{"_37":1444,"_23":302,"_303":1445,"_174":1446},"8a6db538a407",[1281],"Goal orchestration. ",{"_37":1448,"_23":302,"_303":1449,"_174":1450},"63fce0cd84e4",[],"Distant outcomes do not provide enough feedback to guide each decision. Agents need to break them into intermediate goals that make progress measurable without losing the relationship between a local reward and the original objective. Examples of companies working here include ",{"_37":1452,"_23":302,"_303":1453,"_174":1455},"76e48f1d325d",[1454],"e9993843b07c","LangChain",{"_37":1457,"_23":302,"_303":1458,"_174":446},"2b89322b3f2c",[],{"_37":1460,"_23":302,"_303":1461,"_174":1463},"45a2a098c81e",[1462],"471fc2c4e7cd","CrewAI",{"_37":1465,"_23":302,"_303":1466,"_174":1133},"d6f913931aca",[],{"_37":1468,"_23":302,"_303":1469,"_174":1471},"1daf6c53fefe",[1470],"77074ddff25e","Paperclip",{"_37":1473,"_23":302,"_303":1474,"_174":1151},"98475dc5c095",[],[1476,1478,1480],{"_37":1454,"_23":49,"_410":1477},"https://www.langchain.com/",{"_37":1462,"_23":49,"_410":1479},"https://crewai.com/",{"_37":1470,"_23":49,"_410":1481},"https://paperclip.ing/",{"_37":1483,"_23":297,"_298":1484,"_306":1509,"_308":309},"a7490dd45d74",[1485,1489,1493,1498,1501,1506],{"_37":1486,"_23":302,"_303":1487,"_174":1488},"ab25aa899584",[1281],"Durable execution.",{"_37":1490,"_23":302,"_303":1491,"_174":1492},"d0f11e59179b",[]," Long-running work must survive crashes, expired credentials, delayed approvals and irreversible interactions with the outside world. The execution layer must preserve state and know whether a failed operation should be retried, reversed, escalated or stopped. Examples of companies working here include ",{"_37":1494,"_23":302,"_303":1495,"_174":1497},"db6feb64f54d",[1496],"8b581fe7cbbb","Temporal",{"_37":1499,"_23":302,"_303":1500,"_174":1337},"b4df513fced3",[],{"_37":1502,"_23":302,"_303":1503,"_174":1505},"677202be818a",[1504],"9fb992f291c6","Diagrid",{"_37":1507,"_23":302,"_303":1508,"_174":1151},"fb036805c041",[],[1510,1511],{"_37":1496,"_23":49,"_410":1072},{"_37":1504,"_23":49,"_410":1512},"https://www.diagrid.io/",{"_37":1514,"_23":297,"_298":1515,"_306":1540,"_308":309},"2a8dfd5e2da8",[1516,1520,1524,1529,1532,1537],{"_37":1517,"_23":302,"_303":1518,"_174":1519},"62d6a9c3d362",[1281],"Learning.",{"_37":1521,"_23":302,"_303":1522,"_174":1523},"ddb56b77adc0",[]," A transcript records what happened; learning changes what happens next. Agents need a way to turn outcomes into better playbooks, instructions, memory and, potentially, model behavior without allowing every noisy experience to rewrite the system. Examples of companies working here include: ",{"_37":1525,"_23":302,"_303":1526,"_174":1528},"1f6099116696",[1527],"365223e6abca","Letta",{"_37":1530,"_23":302,"_303":1531,"_174":1337},"1e4a94a69683",[],{"_37":1533,"_23":302,"_303":1534,"_174":1536},"f611e46d1634",[1535],"4ae5abc514f6","Modiqo",{"_37":1538,"_23":302,"_303":1539,"_174":1151},"347bb5d7b9c7",[],[1541,1543],{"_37":1527,"_23":49,"_410":1542},"https://www.letta.com/",{"_37":1535,"_23":49,"_410":1544},"https://modiqo.ai/",{"_37":1546,"_23":297,"_298":1547,"_306":1596,"_308":309},"80c9fbd2d18c",[1548,1552,1556,1561,1564,1569,1572,1577,1580,1585,1588,1593],{"_37":1549,"_23":302,"_303":1550,"_174":1551},"5f0ddbdd77d8",[1281],"Toolchain Ecosystem.",{"_37":1553,"_23":302,"_303":1554,"_174":1555},"af999511f199",[]," Agents need additional tooling to extend their capabilities into real-world use cases where they can genuinely move the needle. Among others, a competitive ecosystem of agentic tooling that offers better harnesses, secure sandbox environments and verification, comprehensive monitoring, the ability to support bespoke, custom builds for individual use cases, and large-scale governance and security need to emerge. Examples of companies working here include: ",{"_37":1557,"_23":302,"_303":1558,"_174":1560},"8bf2fe652287",[1559],"d27bdd03882f","Galileo",{"_37":1562,"_23":302,"_303":1563,"_174":446},"b0b1eba96f71",[],{"_37":1565,"_23":302,"_303":1566,"_174":1568},"65dcffc95280",[1567],"8d18392d95fa","AgentOps",{"_37":1570,"_23":302,"_303":1571,"_174":446},"c99edb73daf6",[],{"_37":1573,"_23":302,"_303":1574,"_174":1576},"a041c21dbcb2",[1575],"392239aab680","Halluminate",{"_37":1578,"_23":302,"_303":1579,"_174":446},"eb43cd65895d",[],{"_37":1581,"_23":302,"_303":1582,"_174":1584},"06c84b130f5f",[1583],"22c12974453f","Bronto",{"_37":1586,"_23":302,"_303":1587,"_174":1133},"708d1abf6030",[],{"_37":1589,"_23":302,"_303":1590,"_174":1592},"00900a7e41f7",[1591],"905afc1ab029","Strix",{"_37":1594,"_23":302,"_303":1595,"_174":1151},"9f25c1fc17a6",[],[1597,1599,1601,1603,1605],{"_37":1559,"_23":49,"_410":1598},"https://galileo.ai/",{"_37":1567,"_23":49,"_410":1600},"https://www.agentops.ai/",{"_37":1575,"_23":49,"_410":1602},"https://www.halluminate.ai/",{"_37":1583,"_23":49,"_410":1604},"https://bronto.io/",{"_37":1591,"_23":49,"_410":1606},"https://www.strix.ai/",{"_37":1608,"_23":297,"_298":1609,"_306":1614,"_308":309},"ad7d74561ff6",[1610],{"_37":1611,"_23":302,"_303":1612,"_174":1613},"e5e9aa7a039c",[],"Delineations between these system categories will blur. Memory, for example, may eventually absorb some of what we now call learning. Some capabilities may become native to models. Others will remain properties of the harness, runtime, or organization around the model. The primitives powering these systems are still being designed and implemented.",[],{"_37":1616,"_23":297,"_298":1617,"_306":1622,"_308":382},"5d4c4d98f641",[1618],{"_37":1619,"_23":302,"_303":1620,"_174":1621},"0d5fa68a405a",[],"Got Opinions or Want to Hear More?",[],{"_37":1624,"_23":297,"_298":1625,"_306":1639,"_308":309},"127bd5410ecf",[1626,1630,1635],{"_37":1627,"_23":302,"_303":1628,"_174":1629},"2bd9073f277b",[],"If you’re interested in exploring these further or you’re already building in one or more of these categories, join us on Sept 24 in SF alongside Glean, Hex, and Baseten at ",{"_37":1631,"_23":302,"_303":1632,"_174":1634},"d77b082ef13e",[1633],"6aa2b457ed68","DevGuild: Long-Horizon Stack",{"_37":1636,"_23":302,"_303":1637,"_174":1638},"53b783478968",[],". In future posts, I’ll explore each of these systems and how orgs can transform agents from order takers to outcome owners.",[1640],{"_37":1633,"_23":49,"_410":1641},"https://luma.com/long-horizon-stack?utm_source=library&utm_medium=referral&utm_content=amir_post",{"_37":1643,"_23":297,"_298":1644,"_306":1649,"_308":309},"e20bd4bf8f74",[1645],{"_37":1646,"_23":302,"_303":1647,"_174":1648},"befa8cc290f5",[],"\n",[],8,"https://cdn.sanity.io/images/50q6fr1p/production/41d00bf7d5503991f5f14c7ca50e0f39ab635499-1200x630.png","87ce419b-74da-4384-9e37-22234194cec6","2026-07-31T05:39:00.000Z","long-horizon-agents-from-order-takers-to-outcome-owners","Long-Horizon Agents: From Order-Takers to Outcome Owners",{"_316":1657,"_326":2151,"_149":2152,"_329":2153,"_331":2154,"_333":2155,"_32":2156,"_336":1106},[1658,1666,1685,1715,1723,1742,1785,1793,1801,1809,1817,1825,1833,1841,1860,1879,1887,1925,1964,1972,1980,1988,1996,2011,2019,2027,2045,2053,2061,2080,2096,2104,2112,2120,2136,2144],{"_37":1659,"_23":297,"_298":1660,"_306":1665,"_308":382},"26529dc9c5de",[1661],{"_37":1662,"_23":302,"_303":1663,"_174":1664},"fa8054e9342a",[],"Can You Make AI Infrastructure Invisible?",[],{"_37":1667,"_23":297,"_298":1668,"_306":1682,"_308":309},"f2d60f9923e1",[1669,1673,1678],{"_37":1670,"_23":302,"_303":1671,"_174":1672},"64011ed9736c",[],"Depending on which people you talk to, individual developers spend hundreds (to thousands) of dollars monthly on AI services, with some predicting that ",{"_37":1674,"_23":302,"_303":1675,"_174":1677},"5090b4e72b4a",[1676],"a946eda66f90","AI tool costs will simply exceed the cost of human developers",{"_37":1679,"_23":302,"_303":1680,"_174":1681},"77be8bc77101",[]," soon.",[1683],{"_37":1676,"_23":49,"_410":1684},"https://www.gartner.com/en/newsroom/press-releases/2026-06-24-gartner-predicts-ai-coding-costs-will-surpass-average-developer-salary-by-2028-as-token-consumption-surges",{"_37":1686,"_23":297,"_298":1687,"_306":1710,"_308":309},"0855751c3f10",[1688,1692,1697,1701,1706],{"_37":1689,"_23":302,"_303":1690,"_174":1691},"52907f0c8150",[],"What if the infrastructure and tooling needed to build with AI were free of charge, and wasn’t locked behind a dozen different vendors’ paid plans? That was the question that led developer ",{"_37":1693,"_23":302,"_303":1694,"_174":1696},"80688d3902b7",[1695],"0356687bc521","Mike Kuykendall",{"_37":1698,"_23":302,"_303":1699,"_174":1700},"33b8056f3ce8",[]," to create the open-source project ",{"_37":1702,"_23":302,"_303":1703,"_174":1705},"7a3994a58daa",[1704],"88e723a78af6","Shimmy",{"_37":1707,"_23":302,"_303":1708,"_174":1709},"3ca2a6606a55",[],", a lightweight API replacement for local LLMs. Below, he explains his thought process and his vision for AI software built entirely on free infrastructure.",[1711,1713],{"_37":1695,"_23":49,"_410":1712},"https://www.linkedin.com/in/makuykendall/",{"_37":1704,"_23":49,"_410":1714},"https://github.com/Michael-A-Kuykendall/shimmy",{"_37":1716,"_23":297,"_298":1717,"_306":1722,"_308":382},"4592ad6df254",[1718],{"_37":1719,"_23":302,"_303":1720,"_174":1721},"3a1537e8d179",[],"From Frustration to Lightweight Infrastructure",[],{"_37":1724,"_23":297,"_298":1725,"_306":1739,"_308":309},"5fd9de2e5ac4",[1726,1730,1735],{"_37":1727,"_23":302,"_303":1728,"_174":1729},"99a4067d4313",[],"The start of Kuykendall’s journey was his own frustrations with working across the many different AI-powered software tools he had been using. “It honestly started as a simple ",{"_37":1731,"_23":302,"_303":1732,"_174":1734},"defe0d26f299",[1733],"1c1e796c8ce8","shim",{"_37":1736,"_23":302,"_303":1737,"_174":1738},"e243b5153cc7",[],", because I was getting sick of all the ways to connect with OpenAPI protocols.” The creator was surprised at the project’s popularity, which appears to have come from many other developers also being tired of having to log into dozens of different paid services.",[1740],{"_37":1733,"_23":49,"_410":1741},"https://en.wikipedia.org/wiki/Shim_(computing)",{"_37":1743,"_23":297,"_298":1744,"_306":1782,"_308":309},"c8538982f90c",[1745,1749,1754,1758,1762,1766,1770,1774,1778],{"_37":1746,"_23":302,"_303":1747,"_174":1748},"3583a19bf04c",[],"“I just wanted something that was completely private and offline, where you can control your own inputs, outputs and everything else.” Since then, the creator has moved the project to his own Rust-based library ",{"_37":1750,"_23":302,"_303":1751,"_174":1753},"a36ae61765f0",[1752],"655a455c95a2","Airframe",{"_37":1755,"_23":302,"_303":1756,"_174":1757},"d40bac0b8395",[]," to avoid having to depend on other vendors’ command-line interfaces. “Every time I turned around, I would try to do something else and immediately say, ‘Oh wait, ",{"_37":1759,"_23":302,"_303":1760,"_174":1761},"ff2797f22639",[450],"this",{"_37":1763,"_23":302,"_303":1764,"_174":1765},"fce1c9e1efc5",[]," other thing is owned by ",{"_37":1767,"_23":302,"_303":1768,"_174":1769},"f8eb5fa31c7e",[450],"someone else",{"_37":1771,"_23":302,"_303":1772,"_174":1773},"a92539bc0796",[]," at ",{"_37":1775,"_23":302,"_303":1776,"_174":1777},"e13463e1d2bf",[450],"that",{"_37":1779,"_23":302,"_303":1780,"_174":1781},"5ba470632200",[]," level.”",[1783],{"_37":1752,"_23":49,"_410":1784},"https://github.com/Michael-A-Kuykendall/airframe/blob/master/docs/architecture-map.md",{"_37":1786,"_23":297,"_298":1787,"_306":1792,"_308":309},"d0b0f9748499",[1788],{"_37":1789,"_23":302,"_303":1790,"_174":1791},"dcfb640fd1c2",[],"“I never really was looking for anything out of it other than just to make a good space for people. That's why I’m trying to make it a ‘free forever’ concept. If I can build products that use [this foundation] that would help people, I could sell [those products]. But the underlying service should be a place where we could all come together and just use AI for free. The other thing is, if you look at every service out there, everybody is just completely throttled on tokens. That's an enormous cost for people who want to do development now.”",[],{"_37":1794,"_23":297,"_298":1795,"_306":1800,"_308":309},"4a800f6c6d0b",[1796],{"_37":1797,"_23":302,"_303":1798,"_174":1799},"ed82fed30636",[],"“I think that companies want everyone to pay for inference, and you shouldn't have to. The thing is, everybody's already got bundled costs.” The creator points out that the cost of utilizing high-end foundation models continues to increase, while Chinese AI labs are turning out increasingly competitive open models...which aren’t always supported natively within mainstream AI development toolsets.",[],{"_37":1802,"_23":297,"_298":1803,"_306":1808,"_308":309},"dea88bd57928",[1804],{"_37":1805,"_23":302,"_303":1806,"_174":1807},"1b4fb1e6f304",[],"“Now the users are left with this mess. It was getting to a point where I was paying four different vendors and I wasn't even maximizing all the stuff I had. I asked myself, ‘Why can't we just have an agreed-upon substrate we just all go to?’”",[],{"_37":1810,"_23":297,"_298":1811,"_306":1816,"_308":309},"28fe2c5d871a",[1812],{"_37":1813,"_23":302,"_303":1814,"_174":1815},"6e2a9747f792",[],"“Why not make the substrate free? Shimmy is free, always will be free. I'm not going to charge you. You have it. You use your model, I make sure it's up to date and your model helps you succeed.” Kuykendall suggests his primary audience is software developers, who could eventually use tools built on a free infrastructure substrate.",[],{"_37":1818,"_23":297,"_298":1819,"_306":1824,"_308":693},"a37fc10e7cc9",[1820],{"_37":1821,"_23":302,"_303":1822,"_174":1823},"4db4e6b40e89",[],"If you look at every service out there, everybody is just completely throttled on tokens. That's an enormous cost for people who want to do development now.” -Mike Kuykendall, Creator/Shimmy",[],{"_37":1826,"_23":297,"_298":1827,"_306":1832,"_308":382},"a7520fdacac8",[1828],{"_37":1829,"_23":302,"_303":1830,"_174":1831},"52b20128f977",[],"Freeing AI Infrastructure from Garbage Collection",[],{"_37":1834,"_23":297,"_298":1835,"_306":1840,"_308":309},"739788adaa0a",[1836],{"_37":1837,"_23":302,"_303":1838,"_174":1839},"b785ad9c3e1a",[],"Kuykendall makes the emphatic point that despite the domineering popularity of Python for AI use cases, he designed his project to be 100% Rust-based for a reason. “Garbage collection [in Python] is ridiculous. It’s bad engineering.”",[],{"_37":1842,"_23":297,"_298":1843,"_306":1857,"_308":309},"d43a10427b0d",[1844,1848,1853],{"_37":1845,"_23":302,"_303":1846,"_174":1847},"64dfa91afcb9",[],"The creator explains that he began learning Rust to challenge himself, but found it to be an excellent working language whose ",{"_37":1849,"_23":302,"_303":1850,"_174":1852},"38c36a70c8e3",[1851],"2cbe051ccdd9","ownership and borrowing",{"_37":1854,"_23":302,"_303":1855,"_174":1856},"8a9094d444ed",[]," system inherently forces developers to manage memory safely at compile time, but also matched AI software development well by minimizing non-repeatable, nondeterministic outcomes.",[1858],{"_37":1851,"_23":49,"_410":1859},"https://doc.rust-lang.org/book/ch04-00-understanding-ownership.html",{"_37":1861,"_23":297,"_298":1862,"_306":1876,"_308":309},"09c2e644eec9",[1863,1867,1872],{"_37":1864,"_23":302,"_303":1865,"_174":1866},"40abccc499f8",[],"“I also found that the garbage collection brought along with it a bunch of stuff you don't need. If you change your thinking around whether you need to retire some variables or let them gather in heaps, you can just use them and borrow them appropriately.” The creator notes that Rust’s ",{"_37":1868,"_23":302,"_303":1869,"_174":1871},"fb73d36dfd1e",[1870],"606966e22cb9","static type system",{"_37":1873,"_23":302,"_303":1874,"_174":1875},"6a47bc41c228",[],", which verifies every variable at compile time, was both more sensical and helped him create invariant pipelines that reduced unpredictable outcomes as he built.",[1877],{"_37":1870,"_23":49,"_410":1878},"https://en.wikipedia.org/wiki/Type_system#STATIC",{"_37":1880,"_23":297,"_298":1881,"_306":1886,"_308":382},"fa4e453f8ffb",[1882],{"_37":1883,"_23":302,"_303":1884,"_174":1885},"17d5610728e0",[],"How Free Infrastructure Cures Tokenmaxxing",[],{"_37":1888,"_23":297,"_298":1889,"_306":1918,"_308":309},"77a74cdcd61c",[1890,1894,1898,1901,1905,1909,1914],{"_37":1891,"_23":302,"_303":1892,"_174":1893},"c1d4179c805b",[],"Kuykendall suggests that a largely free AI infrastructure substrate should be the solution for headline-grabbing token budget problems from orgs like ",{"_37":1895,"_23":302,"_303":1896,"_174":1333},"6956f660bcc5",[1897],"f74274218a16",{"_37":1899,"_23":302,"_303":1900,"_174":1337},"b3c179bda066",[],{"_37":1902,"_23":302,"_303":1903,"_174":1342},"dba715781e04",[1904],"d15ce3ecf70c",{"_37":1906,"_23":302,"_303":1907,"_174":1908},"7f00a359d649",[],". “Just recently, almost every AI company cranked up their rates,” notes the creator, in reference to how major AI vendors recently ",{"_37":1910,"_23":302,"_303":1911,"_174":1913},"d4b75be75e15",[1912],"53cfd64a369d","repriced tokens",{"_37":1915,"_23":302,"_303":1916,"_174":1917},"7271af2c0792",[]," that caused users everywhere to burn through their token budgets early.",[1919,1921,1923],{"_37":1897,"_23":49,"_410":1920},"https://fortune.com/2026/05/26/uber-coo-ai-spending-tokens-claude-code/",{"_37":1904,"_23":49,"_410":1922},"https://x.com/laurabratton5/status/2054558503436755350",{"_37":1912,"_23":49,"_410":1924},"https://www.investing.com/analysis/the-ai-token-pricing-crisis-behind-openai-and-anthropics-revenue-race-200680777",{"_37":1926,"_23":297,"_298":1927,"_306":1957,"_308":309},"fd64b7edec35",[1928,1932,1937,1940,1945,1948,1953],{"_37":1929,"_23":302,"_303":1930,"_174":1931},"73c5a60e5841",[],"“Last month, I ran out of tokens for every one of my top services on day 15. And I was staring down at the remaining two weeks, thinking to myself, ‘What am I supposed to do now?’ I was forced to go try to ‘do it cheap.’” For his own purposes, the creator went off and did what he imagined many other developers are doing: Trying a list of free alternatives, including ",{"_37":1933,"_23":302,"_303":1934,"_174":1936},"986ca549a796",[1935],"5e03a202a084","OpenCode",{"_37":1938,"_23":302,"_303":1939,"_174":446},"ba2ccf513f53",[],{"_37":1941,"_23":302,"_303":1942,"_174":1944},"bb8757025006",[1943],"e2c462342b10","Pi",{"_37":1946,"_23":302,"_303":1947,"_174":446},"78c40617341a",[],{"_37":1949,"_23":302,"_303":1950,"_174":1952},"e3a3359c81f6",[1951],"674622a84064","Hermes Agent",{"_37":1954,"_23":302,"_303":1955,"_174":1956},"a2435627bb91",[],", and others.",[1958,1960,1962],{"_37":1935,"_23":49,"_410":1959},"https://opencode.ai/",{"_37":1943,"_23":49,"_410":1961},"https://pi.dev/",{"_37":1951,"_23":49,"_410":1963},"https://hermesagent.agency/",{"_37":1965,"_23":297,"_298":1966,"_306":1971,"_308":309},"e9871fa1f8fc",[1967],{"_37":1968,"_23":302,"_303":1969,"_174":1970},"114996d67558",[],"“I think basically the corporations right now are trying to recoup their money. They realize that they're just going to have to take it from the people already using it. That's when I realized if I'm going to be a developer and I'm going to use this service, I need a more-economical way.”",[],{"_37":1973,"_23":297,"_298":1974,"_306":1979,"_308":309},"7ec3c3b943a8",[1975],{"_37":1976,"_23":302,"_303":1977,"_174":1978},"3e48df6c56f5",[],"Having already canceled many of his pricey AI subscription services, Kuykendall reflects on how other developers like him may be making the same choices...and ending up with the extra hundreds of dollars saved from those canceled services, perhaps ready to spend elsewhere. “I’m working on other products on top of shimmy. Because that's where the money should be, in my opinion. Application should be where we put the money, not the inference.”",[],{"_37":1981,"_23":297,"_298":1982,"_306":1987,"_308":382},"8407533ebf07",[1983],{"_37":1984,"_23":302,"_303":1985,"_174":1986},"65e2c0c1d5d1",[],"An Inference Tipping Point for Engineering Orgs, Coming Soon?",[],{"_37":1989,"_23":297,"_298":1990,"_306":1995,"_308":309},"59f0368952cd",[1991],{"_37":1992,"_23":302,"_303":1993,"_174":1994},"7c37320bdb49",[],"Kuykendall foresees a not-to-distant future where engineering teams take a hard look at their AI bills, come to the same realization he did, and look to make a change. “I think the [largest AI vendors] understand that they overinvested.”",[],{"_37":1997,"_23":297,"_298":1998,"_306":2010,"_308":309},"5fd0bd7e9815",[1999,2003,2006],{"_37":2000,"_23":302,"_303":2001,"_174":2002},"13de0ed48aeb",[],"“In the meantime, if I had a young up-and-comer startup right now, I would use an inference stack that was free. I’d try to get the best compute I could get for my dollar. I think there ",{"_37":2004,"_23":302,"_303":2005,"_174":747},"a774b1f7d0d5",[450],{"_37":2007,"_23":302,"_303":2008,"_174":2009},"c6e4eab459af",[]," going to be an ‘API apocalypse,’ or whatever they’re calling it on social media, when everybody starts seeing viable alternatives they can use instead. And I think we're already getting there.”",[],{"_37":2012,"_23":297,"_298":2013,"_306":2018,"_308":309},"2fdcdbfc3d6d",[2014],{"_37":2015,"_23":302,"_303":2016,"_174":2017},"aada6eb3f5f3",[],"The creator notes that open models like Alibaba’s Qwen family and Z.ai’s GLM models can already perform well on software development when coupled with high-end hardware, a stark contrast to some of his previous experiences. “I worked at the IRS for several years, with no AI available. We had to do everything ‘the hard way.’ There were times when I’d think: ‘I could save three hours if I could just use AI right now!’”",[],{"_37":2020,"_23":297,"_298":2021,"_306":2026,"_308":309},"88e2c04c93f9",[2022],{"_37":2023,"_23":302,"_303":2024,"_174":2025},"9b9b207217a7",[],"According to Kuykendall, we live in very different times now, with highly performant open models and high-end hardware both within reach of individual developers. “Long story short: What I wanted was for nerds like us to lay the groundwork so that it was just invisible infrastructure, for AI work, development, or anything else.”",[],{"_37":2028,"_23":297,"_298":2029,"_306":2042,"_308":309},"67f4203d099e",[2030,2034,2039],{"_37":2031,"_23":302,"_303":2032,"_174":2033},"d2f914659680",[],"“It was also important to build the project offline. There are capable builders in Asia, some of the biggest drivers of growth on my GitHub repos, because I think they’re under the ",{"_37":2035,"_23":302,"_303":2036,"_174":2038},"e7ad924dbde1",[2037],"bae39e222a2e","Great Firewall",{"_37":2040,"_23":302,"_303":2041,"_174":551},"0b802225f345",[],[2043],{"_37":2037,"_23":49,"_410":2044},"https://en.wikipedia.org/wiki/Great_Firewall",{"_37":2046,"_23":297,"_298":2047,"_306":2052,"_308":693},"486ea7c3b94d",[2048],{"_37":2049,"_23":302,"_303":2050,"_174":2051},"2201bd929675",[],"What I wanted was for nerds like us to lay the groundwork so that it was just invisible infrastructure, for AI work, development, or anything else.”",[],{"_37":2054,"_23":297,"_298":2055,"_306":2060,"_308":382},"53749e95bb4e",[2056],{"_37":2057,"_23":302,"_303":2058,"_174":2059},"0310b232625d",[],"Lessons from Building Many Projects with AI",[],{"_37":2062,"_23":297,"_298":2063,"_306":2077,"_308":309},"c8b2d2381460",[2064,2068,2073],{"_37":2065,"_23":302,"_303":2066,"_174":2067},"1d9af94ea5c0",[],"While AI can be a powerful assistant, Kuykendall warns other builders to avoid lulled into a false sense of security from ",{"_37":2069,"_23":302,"_303":2070,"_174":2072},"a2aaf9fd4b37",[2071],"e12cad35cf8c","sycophantic models",{"_37":2074,"_23":302,"_303":2075,"_174":2076},"e5e63a879bd9",[]," that agree with you a little too much. “You have to separate yourself from this stuff, or at least watch for AI having an emotional impact on you. Try to get away from that and make it a source of data instead.”",[2078],{"_37":2071,"_23":49,"_410":2079},"https://en.wikipedia.org/wiki/Sycophancy_(artificial_intelligence)",{"_37":2081,"_23":297,"_298":2082,"_306":2095,"_308":309},"8b1ffd549782",[2083,2087,2091],{"_37":2084,"_23":302,"_303":2085,"_174":2086},"ee3e7aee0636",[],"“I've been in AI development pretty much from the moment it started. And I noticed while doing testing that tests would just ",{"_37":2088,"_23":302,"_303":2089,"_174":2090},"e0d714f4e6d7",[450],"happen",{"_37":2092,"_23":302,"_303":2093,"_174":2094},"d2e6fb2bfd8d",[]," to go the way I wanted them to, by a point or two. Always. The AI was quietly influencing things. So for founders, I would recommend that first of all, they should spread out their compute and find many points of view. Every one of these [models] is differently trained and they're going to give you a different answer. And a lot of times you have to watch your logic, because it'll tell you exactly what you want to hear.”",[],{"_37":2097,"_23":297,"_298":2098,"_306":2103,"_308":309},"9078a52ba64c",[2099],{"_37":2100,"_23":302,"_303":2101,"_174":2102},"45fa152bf8da",[],"The creator recommends putting models through their paces, at minimum asking why ideas might be good and why they might be bad, and potentially pitting other models against each other. “You have to try to look at it as an empirical and scientific process, and try to take the human emotion out of it.”",[],{"_37":2105,"_23":297,"_298":2106,"_306":2111,"_308":382},"79614730978e",[2107],{"_37":2108,"_23":302,"_303":2109,"_174":2110},"c2ee84a941c8",[],"What AI Infra Will Look Like in the Future",[],{"_37":2113,"_23":297,"_298":2114,"_306":2119,"_308":309},"934f5116fe06",[2115],{"_37":2116,"_23":302,"_303":2117,"_174":2118},"8392b1017d92",[],"Kuykendall envisions a future AI infrastructure without the many layers of interface that block users from getting things done. “When I first started really spitballing it early on, I remember thinking: Why am I wasting my time with a computer, that's running a browser, that's running a window, that's running a thing that I connect to a server, that runs the AI? What's all this intervening stuff about?”",[],{"_37":2121,"_23":297,"_298":2122,"_306":2135,"_308":309},"bbcd02b13f08",[2123,2127,2131],{"_37":2124,"_23":302,"_303":2125,"_174":2126},"3bad3c3c0ef8",[],"“Some of my early, kind of crazy experiments were running AI on hot metal and then having an operating system that's just an AI with tools. Because I found that I would rather have that than have to poke around for that one thing, or that one file. Why don't we just change the idea of an operating system to basically be an ",{"_37":2128,"_23":302,"_303":2129,"_174":2130},"0da0ebd6f719",[450],"intention system",{"_37":2132,"_23":302,"_303":2133,"_174":2134},"1d4a5ea25430",[],"? Why not take the AI, surround it with tools, and give it the ability to talk to me and others?”",[],{"_37":2137,"_23":297,"_298":2138,"_306":2143,"_308":309},"d63b7dcb7393",[2139],{"_37":2140,"_23":302,"_303":2141,"_174":2142},"1073724587bf",[],"“I think that's the next thing that would be really transformative. Taking out all the shell layers I don't need between myself and an AI with compute. I need to be able to talk to it. I need to have rewrite ability. I know this because I've already built more than a dozen AI tools. Then you put that in a stack and run on hot metal. I’d be saving a lot of money just doing that.”",[],{"_37":2145,"_23":297,"_298":2146,"_306":2150,"_308":309},"38200e1a51a6",[2147],{"_37":2148,"_23":302,"_303":2149,"_174":1167},"ca88a52ab9af",[],[],11,"https://cdn.sanity.io/images/50q6fr1p/production/0017fa2c5f2372892542ff219e62bd4b155672ed-1200x630.png","3efff6ce-8204-46cb-a76d-3cf6b9473edf","2026-07-28T16:48:00.000Z","can-you-make-ai-infrastructure-free-forever","Can You Make AI Infrastructure Free Forever?","modules","moreFromLibrary",[2160,2162,2164],{"_37":2161,"_116":330,"_23":118},"cGhKi2Jf7quM",{"_37":2163,"_116":350,"_23":118},"glLVQJwaXCh3",{"_37":2165,"_116":366,"_23":118},"CKKWKVTNMOpm","outlines",[2168],{"_37":2169,"_23":297,"_298":2170,"_306":2175,"_308":309},"47115bc6b21e",[2171],{"_37":2172,"_23":302,"_303":2173,"_174":2174},"2e9c077f9fcd",[],"

Agent-Led Trust and Safety\n
- Is the Agent a User or a New Persona?
- Where Does the Agent Live?
- Tabletop Exercise: The Agent That Deleted Everything
- Define the Constraint or Control the Experience
- Discerning Attackers from the Vicious Loop
- Traces, Sampling, and What Comes Next
\n

",[],"posterFrame",{"_23":149,"_114":2178},{"_116":2179,"_23":118},"image-47f2212c90164e54941f815753e3511c3f6d2f22-1920x1080-jpg","2026-06-16T21:55:32.272Z","seo",{"_147":2183,"_23":2181,"_28":2186,"_2187":196,"_2188":196,"_2189":305},{"_23":149,"_114":2184},{"_116":2185,"_23":118},"image-3e55eb64e7510be61d9e90d42f222a30a6b86c35-1920x1080-jpg","This panel uncovers what it takes to let agents loose on production systems without losing sleep: permissions, observability, and the guardrails.","nofollow","noindex","OGdescription","slideDeck",{"_23":333,"_2192":2193},"current","agent-led-trust-and-safety","speakers","speakersWithOrg",[2197,2253,2276,2350],{"_37":2198,"_23":2199,"_2200":-5,"_2201":2202},"320d0b80d9af","speaker","organization","person",{"_11":2203,"_13":2204,"_14":2205,"_16":2206,"_23":2201,"_24":2209,"_2210":2211,"_2220":196,"_2221":2222,"_2200":2223,"_2231":2232,"_2238":2239,"_333":2240,"_2242":2243,"_2252":196},"2021-09-21T21:21:21Z","person-8285","0B0sCzJ8ywifPyVwtbGQMb",{"_18":2207},{"_20":2204,"_21":2208},"aeN0todGvIjkWyTywFzxJp","2025-08-19T13:26:33Z","bio",[2212],{"_37":2213,"_23":297,"_298":2214,"_306":2219,"_308":309},"fe5cd992aa26",[2215],{"_37":2216,"_23":302,"_303":2217,"_174":2218},"fe5cd992aa260",[],"CTO & Co-founder, Honeycomb.io",[],"generate","name","Charity Majors",{"_11":2224,"_13":2225,"_14":2226,"_23":2200,"_24":2224,"_28":2227,"_2220":196,"_2221":2228,"_2229":196,"_2181":2230},"2022-08-22T17:52:12Z","0f8d6695-82d8-43dd-8e8c-9843a43df108","aeN0todGvIjkWyTywFzWPh","Honeycomb is a fast analysis tool that reveals the truth about how users experience your code in complex and unpredictable environments. Within seconds, find patterns and outliers across billions of rows of data to definitively solve problems.","Honeycomb","portfolioCompany",{"_23":2181,"_28":2227,"_2187":196,"_2188":196,"_32":2228},"photo",{"_23":113,"_2233":2234,"_114":2235,"_2237":2234},"alt","Charity Majors Headshot",{"_116":2236,"_23":118},"image-bfcdf6774488b3d95a532a596ab0211b328a2b09-400x400-jpg","caption","position","CTO",{"_23":333,"_2192":2241},"charity-majors-old","socials",[2244,2248],{"_37":2245,"_23":107,"_49":2246,"_2221":2247},"20c12aa0ea4f","https://www.linkedin.com/in/charity-majors/","linkedin",{"_37":2249,"_23":107,"_49":2250,"_2221":2251},"a92011e37e13","https://twitter.com/mipsytipsy","twitter","teamMember",{"_37":2254,"_23":2199,"_2200":-5,"_2201":2255},"8561def8a7f0",{"_11":2256,"_13":2257,"_14":2258,"_16":2259,"_23":2201,"_24":2262,"_2220":196,"_2221":2263,"_2200":2264,"_2270":196,"_2231":2271,"_2238":2274,"_2181":2275,"_2252":196},"2026-04-06T21:48:22Z","c3085b45-e7be-4581-9837-9635a76e513a","Vht0I1MDD4rKgReZcoy8Up",{"_18":2260},{"_20":2257,"_21":2261},"Vht0I1MDD4rKgReZcoxeER","2026-04-06T21:49:37Z","Sarah Sachs",{"_11":2265,"_13":2266,"_14":2267,"_23":2200,"_24":2268,"_2220":196,"_2221":2269,"_2229":196},"2022-09-16T02:41:33Z","oTug8j9bB2lUwM6o7Izfon","TYN2nJHhdg1RiOg5ta1HWG","2022-09-16T23:14:43Z","Notion","partner",{"_23":113,"_114":2272},{"_116":2273,"_23":118},"image-23fbea7f5e09c39e2505d33821d3bd7bd907ec55-400x400-jpg","Head of AI",{"_23":2181,"_2187":196,"_2188":196},{"_37":2277,"_23":2199,"_2200":-5,"_2201":2278},"19f133bd4fad",{"_11":2279,"_13":2280,"_14":2281,"_16":2282,"_23":2201,"_24":2285,"_2220":196,"_2221":2286,"_2200":2287,"_2270":196,"_2231":2345,"_2238":2348,"_2181":2349,"_2252":196},"2026-04-06T21:51:27Z","d30c576e-7098-42cd-b9fa-8e54cc3bc115","JKqyaTxoFmjftwQTrhPnca",{"_18":2283},{"_20":2280,"_21":2284},"JKqyaTxoFmjftwQTrhPnWG","2026-04-06T21:51:52Z","Ben Woskow",{"_11":2288,"_13":2289,"_14":2290,"_16":2291,"_23":2200,"_24":2294,"_2295":2296,"_2303":2304,"_28":2309,"_2220":171,"_2310":2311,"_49":2312,"_2313":2314,"_2221":2317,"_2229":171,"_2318":2319,"_2181":2332,"_333":2333,"_2335":2336,"_2337":2338,"_2339":2340},"2022-04-29T20:41:45Z","recOesppkhmpvVfk7","rdofR6C9TPA5BJl8cKEph1",{"_18":2292},{"_20":2289,"_21":2293},"0B0sCzJ8ywifPyVwtbGQZ5","2026-07-22T22:04:04Z","about",[2297],{"_37":2295,"_23":297,"_298":2298,"_306":2302,"_308":309},[2299],{"_37":2295,"_23":302,"_303":2300,"_174":2301},[],"LaunchDarkly is a feature management platform that allows software development teams to deliver to their customers. By decoupling feature rollout and code deployment, LaunchDarkly enables developers to test their code live in production, gradually release features to groups of users, and manage flags throughout their entire lifecycle. This allows developers to release better software, with less risk. \n",[],"altLogo",{"_23":113,"_2233":2305,"_114":2306,"_2308":1167,"_2237":2305},"Logo of LaunchDarkly",{"_116":2307,"_23":118},"image-b764655c80d33e514b20e0e953e345f4a2c8dd0f-587x91-svg","attribution","Feature Flag Management","hiringPage","https://boards.greenhouse.io/launchdarkly","https://www.launchdarkly.com","logo",{"_23":113,"_2233":2305,"_114":2315,"_2308":1167,"_2237":2305},{"_116":2316,"_23":118},"image-05a866a4ae36a150ec37c4c7ca319504eeb69f68-587x91-svg","LaunchDarkly","resources",[2320,2324,2328],{"_37":2321,"_23":2322,"_47":2323,"_49":2312,"_32":2323},"website","resource","Website",{"_37":2325,"_23":2322,"_47":2326,"_49":2327,"_32":2326},"02de1f4900d7","Portfolio Spotlight","https://www.heavybit.com/portfolio/spotlights/launchdarkly",{"_37":2329,"_23":2322,"_47":2330,"_49":2331,"_32":2330},"11e5bf64aac1","Work at LaunchDarkly","https://launchdarkly.com/careers/",{"_23":2181,"_28":2317,"_2187":196,"_2188":196,"_32":2317},{"_23":333,"_2192":2334},"launchdarkly","stage","series C","status","active","tags",[2341,2343],{"_37":2342,"_116":2342,"_23":118},"rec9kFoUpbhvhQGMh",{"_37":2344,"_116":2344,"_23":118},"recfPmuQyTJq61F3S",{"_23":113,"_114":2346},{"_116":2347,"_23":118},"image-214e267bc7d01e69b03d0e583dae2588bdb6b0e5-1008x1008-jpg","Senior Director of Engineering",{"_23":2181,"_2187":196,"_2188":196},{"_37":2351,"_23":2199,"_2200":-5,"_2201":2352},"6fcb071f1a3c",{"_11":2353,"_13":2354,"_14":2355,"_16":2356,"_23":2201,"_24":2359,"_2220":196,"_2221":2360,"_2200":2361,"_2270":196,"_2231":2367,"_2238":2370,"_2181":2371,"_2252":196},"2026-04-06T21:52:56Z","97fbf4dc-380b-4f8a-b4c3-a9514472ae74","9uVHPft3lgVHpgLxC6Pogw",{"_18":2357},{"_20":2354,"_21":2358},"LGJIrDmN0ZiOuKpBNWucOp","2026-04-29T21:16:51Z","Chetan Conikee",{"_11":2362,"_13":2363,"_14":2364,"_23":2200,"_24":2365,"_2220":196,"_2221":1536,"_2229":196,"_2181":2366},"2026-04-06T21:53:03Z","626ac00a-c76e-45be-922b-374748ad9c2b","JKqyaTxoFmjftwQTrhPp36","2026-04-06T21:53:05Z",{"_23":2181,"_2187":196,"_2188":196},{"_23":113,"_114":2368},{"_116":2369,"_23":118},"image-99e5cd557ec8b45836aa0dd2bc9d047c3e0761f7-880x880-png","Founder/CEO",{"_23":2181,"_2187":196,"_2188":196},"startDate",[2374,2383,2391],{"_11":2375,"_13":2376,"_14":2377,"_23":2378,"_24":2375,"_28":2379,"_2221":2380,"_333":2381},"2024-09-05T17:31:35Z","017fa9be-32e2-46ee-bddf-f13275fbdf00","aeN0todGvIjkWyTywFzV6F","tag","Pertaining to the technology underpinning AI and machine learning operations.","AI Infrastructure",{"_23":333,"_2192":2382},"ai-infrastructure",{"_11":2384,"_13":2385,"_14":2386,"_23":2378,"_24":2384,"_28":2387,"_2221":2388,"_333":2389},"2025-04-15T17:19:22Z","80867d7d-9222-41f5-a465-915aa791bc97","YlxgmKTLxstUdop23VNy4f","AI agents are autonomous systems built to independently take action on business or technical tasks.","AI Agents",{"_23":333,"_2192":2390},"ai-agents",{"_11":2392,"_13":2393,"_14":2394,"_23":2378,"_24":2392,"_28":2395,"_2221":2396,"_333":2397},"2023-07-17T23:23:49Z","eec559a9-10e7-4405-82e4-e0736495ec36","TfMkf5gVJ7UWv1vMi5Qxor","Artificial intelligence, including AI-powered coding assistants built on large language models.","AI",{"_23":333,"_2192":2398},"ai","Agent-Led Trust and Safety","transcript",[2402,2410,2448,2479,2490,2525,2556,2597,2652,2683,2704,2735,2776,2807,2852,2883,2914,2945,2976,3011,3019,3054,3085,3116,3137,3162,3193,3224,3275,3316,3337,3378,3409,3440,3481,3535,3586,3640,3664,3705,3736,3771,3822,3873,3897,3938,3969,4023,4084,4125,4149,4180,4214,4245,4276,4284,4338,4369,4410,4451,4505,4516,4530,4574,4595,4646,4667,4708,4739,4753,4774,4825,4866,4917,4971,5012,5063,5094,5125,5156,5180,5188,5232,5263,5284,5335,5376,5410,5441,5452,5503,5544,5588,5629,5650,5711,5752,5813,5854,5895,5916,5960,6001,6042,6093,6124,6165,6206,6214,6258,6289,6310,6364,6375,6399,6440,6461,6505,6556,6590,6611,6635,6643,6677,6718,6732,6746,6780,6814,6878,6919,6930,6971,7001,7032,7066,7097,7141,7172,7213,7227],{"_37":2403,"_23":297,"_298":2404,"_306":2409,"_308":382},"4c0f44c09c61",[2405],{"_37":2406,"_23":302,"_303":2407,"_174":2408},"1f943e0ede13",[],"Is the Agent a User or a New Persona?",[],{"_37":2411,"_23":297,"_298":2412,"_306":2439,"_308":309},"beff13b122a2",[2413,2418,2422,2426,2431,2434],{"_37":2414,"_23":302,"_303":2415,"_174":2417},"bb294e6b21fb",[2416,1281],"7ea35a4","Chetan Conikee:",{"_37":2419,"_23":302,"_303":2420,"_174":2421},"dba37af640a1",[2416]," We are in a phase shift where the persona that we have been catering our products to were users.",{"_37":2423,"_23":302,"_303":2424,"_174":2425},"d0e68bfac499",[]," ",{"_37":2427,"_23":302,"_303":2428,"_174":2430},"2fac267ec3da",[2429],"6d5cd8a","And now we're dealing with these agents operating at level one, two, three autonomy, orchestrated by users.",{"_37":2432,"_23":302,"_303":2433,"_174":2425},"a95625dcd6fd",[],{"_37":2435,"_23":302,"_303":2436,"_174":2438},"b0aa2fed532d",[2437],"0a3b542","So I'd like to start with maybe framing a question and then taking it to the panel and letting them introduce themselves.",[2440,2444,2446],{"_37":2416,"_23":2441,"_2442":2443},"timestamp","time","00:00:04.640",{"_37":2429,"_23":2441,"_2442":2445},"00:00:12.620",{"_37":2437,"_23":2441,"_2442":2447},"00:00:20.340",{"_37":2449,"_23":297,"_298":2450,"_306":2472,"_308":309},"8fbc4e462c5d",[2451,2456,2459,2464,2467],{"_37":2452,"_23":302,"_303":2453,"_174":2455},"e0b8c7b73cd1",[2454],"8f0a146","The question is, often when we perceive security or discuss security, we ask ourselves: is the user allowed to do this action?",{"_37":2457,"_23":302,"_303":2458,"_174":2425},"621b07337ed1",[],{"_37":2460,"_23":302,"_303":2461,"_174":2463},"0434f6415d42",[2462],"c5761af","And now with the phase shift where we're dealing with agents, users express intent, and that intent converts to a set of actions.",{"_37":2465,"_23":302,"_303":2466,"_174":2425},"78d8b3ee8fcc",[],{"_37":2468,"_23":302,"_303":2469,"_174":2471},"c0e6cce9bac7",[2470],"ee15b6a","So from a perception standpoint, we can either treat the agent as a user or the agent as a brand new persona that we have to level-set security with.",[2473,2475,2477],{"_37":2454,"_23":2441,"_2442":2474},"00:00:28.880",{"_37":2462,"_23":2441,"_2442":2476},"00:00:41.140",{"_37":2470,"_23":2441,"_2442":2478},"00:00:51.820",{"_37":2480,"_23":297,"_298":2481,"_306":2487,"_308":309},"aab1f594f7f5",[2482],{"_37":2483,"_23":302,"_303":2484,"_174":2486},"95808015ea12",[2485],"ed45ddc","So in 30 seconds, can each of you introduce yourselves, tell us what you're building, and what is your opinion of this question?",[2488],{"_37":2485,"_23":2441,"_2442":2489},"00:01:02.680",{"_37":2491,"_23":297,"_298":2492,"_306":2518,"_308":309},"6af9c5a31a91",[2493,2498,2502,2505,2510,2513],{"_37":2494,"_23":302,"_303":2495,"_174":2497},"851548129850",[2496,1281],"6ddd217","Ben Woskow:",{"_37":2499,"_23":302,"_303":2500,"_174":2501},"74052eaa3708",[2496]," Hi, my name is Ben Woskow, and I am a senior director of engineering at LaunchDarkly.",{"_37":2503,"_23":302,"_303":2504,"_174":2425},"d6d24433c67a",[],{"_37":2506,"_23":302,"_303":2507,"_174":2509},"078d0af08bfe",[2508],"b63bf77","I'm responsible for building an AI product named AI Configs.",{"_37":2511,"_23":302,"_303":2512,"_174":2425},"3caf694d3f9e",[],{"_37":2514,"_23":302,"_303":2515,"_174":2517},"278c9f5b70e9",[2516],"f620b4a","And we have a very big launch coming out in a couple weeks, so keep an eye out for some announcements coming from us soon.",[2519,2521,2523],{"_37":2496,"_23":2441,"_2442":2520},"00:01:15.880",{"_37":2508,"_23":2441,"_2442":2522},"00:01:22.200",{"_37":2516,"_23":2441,"_2442":2524},"00:01:27.540",{"_37":2526,"_23":297,"_298":2527,"_306":2549,"_308":309},"554620fd8bba",[2528,2533,2536,2541,2544],{"_37":2529,"_23":302,"_303":2530,"_174":2532},"1c989d9702d3",[2531],"a685917","In response to your question, we have",{"_37":2534,"_23":302,"_303":2535,"_174":2425},"48bc7697540e",[],{"_37":2537,"_23":302,"_303":2538,"_174":2540},"c87021bacaab",[2539],"c3f9676","human users, we have agent users, and they're not mutually exclusive.",{"_37":2542,"_23":302,"_303":2543,"_174":2425},"77406dce0a50",[],{"_37":2545,"_23":302,"_303":2546,"_174":2548},"0a8c936972bf",[2547],"ebbbcaa","Both worlds very much exist.",[2550,2552,2554],{"_37":2531,"_23":2441,"_2442":2551},"00:01:33.780",{"_37":2539,"_23":2441,"_2442":2553},"00:01:37.200",{"_37":2547,"_23":2441,"_2442":2555},"00:01:42.000",{"_37":2557,"_23":297,"_298":2558,"_306":2588,"_308":309},"eb2cc6ce789c",[2559,2564,2567,2572,2575,2580,2583],{"_37":2560,"_23":302,"_303":2561,"_174":2563},"bbec3fd5cc0c",[2562],"a9b57ca","With coding assistants today, we have some coding assistants that are behaving on",{"_37":2565,"_23":302,"_303":2566,"_174":2425},"d9f493e7f016",[],{"_37":2568,"_23":302,"_303":2569,"_174":2571},"be0aec0aad57",[2570],"c2e7326","behalf of users, and they're somewhat transactional, having specific actions, creating a feature flag as part of some larger workflow where each individual step is acted upon",{"_37":2573,"_23":302,"_303":2574,"_174":2425},"7bbd334d8c19",[],{"_37":2576,"_23":302,"_303":2577,"_174":2579},"dd085bb542b6",[2578],"a9835f8","with conjunction of the user.",{"_37":2581,"_23":302,"_303":2582,"_174":2425},"ac0e147e6c61",[],{"_37":2584,"_23":302,"_303":2585,"_174":2587},"15ce274fa393",[2586],"a9b6c09","And then we have some fairly autonomous agents that are going through the entire SDLC, in which case creating and rolling out a feature flag is just one step of many others.",[2589,2591,2593,2595],{"_37":2562,"_23":2441,"_2442":2590},"00:01:44.240",{"_37":2570,"_23":2441,"_2442":2592},"00:01:51.720",{"_37":2578,"_23":2441,"_2442":2594},"00:02:03.420",{"_37":2586,"_23":2441,"_2442":2596},"00:02:05.100",{"_37":2598,"_23":297,"_298":2599,"_306":2641,"_308":309},"c01ef180bb10",[2600,2605,2609,2612,2617,2620,2625,2628,2633,2636],{"_37":2601,"_23":302,"_303":2602,"_174":2604},"a57aafe6ae8c",[2603,1281],"7d88e8f","Charity Majors:",{"_37":2606,"_23":302,"_303":2607,"_174":2608},"c6791e813005",[2603]," My name is Charity Majors.",{"_37":2610,"_23":302,"_303":2611,"_174":2425},"c4d7d94d037d",[],{"_37":2613,"_23":302,"_303":2614,"_174":2616},"b4a7c0afe6b5",[2615],"a91bfed","I'm co-founder, CTO of Honeycomb.",{"_37":2618,"_23":302,"_303":2619,"_174":2425},"4862514706c0",[],{"_37":2621,"_23":302,"_303":2622,"_174":2624},"ee9b21d0b2ca",[2623],"45d6dc1","We also have an enormous launch of many things coming out next week.",{"_37":2626,"_23":302,"_303":2627,"_174":2425},"8ad8e8099350",[],{"_37":2629,"_23":302,"_303":2630,"_174":2632},"046a357e86af",[2631],"c23d17b","This is awkwardly timed.",{"_37":2634,"_23":302,"_303":2635,"_174":2425},"8775d65e05d3",[],{"_37":2637,"_23":302,"_303":2638,"_174":2640},"fbfaec94f78a",[2639],"bd1cef7","But if I zoom out, I would say:",[2642,2644,2646,2648,2650],{"_37":2603,"_23":2441,"_2442":2643},"00:02:20.640",{"_37":2615,"_23":2441,"_2442":2645},"00:02:22.959",{"_37":2623,"_23":2441,"_2442":2647},"00:02:26.240",{"_37":2631,"_23":2441,"_2442":2649},"00:02:31.520",{"_37":2639,"_23":2441,"_2442":2651},"00:02:35.040",{"_37":2653,"_23":297,"_298":2654,"_306":2676,"_308":693},"2ccbdae2a82b",[2655,2660,2663,2668,2671],{"_37":2656,"_23":302,"_303":2657,"_174":2659},"28376e47446e",[2658],"a580029","The whole point of the 20 years of DevOps arguably was to create one feedback loop that contained both developers writing their code and that code in production.",{"_37":2661,"_23":302,"_303":2662,"_174":2425},"ed61fe632ecf",[],{"_37":2664,"_23":302,"_303":2665,"_174":2667},"a2cb658609a0",[2666],"c073561","And on that count, before AI, it had failed.",{"_37":2669,"_23":302,"_303":2670,"_174":2425},"d3c8b2013c7d",[],{"_37":2672,"_23":302,"_303":2673,"_174":2675},"c577c730d844",[2674],"2a8f651","It 100% failed.",[2677,2679,2681],{"_37":2658,"_23":2441,"_2442":2678},"00:02:39.860",{"_37":2666,"_23":2441,"_2442":2680},"00:02:52.400",{"_37":2674,"_23":2441,"_2442":2682},"00:02:56.160",{"_37":2684,"_23":297,"_298":2685,"_306":2699,"_308":309},"881d2b81fa79",[2686,2691,2694],{"_37":2687,"_23":302,"_303":2688,"_174":2690},"84f4c02b43b2",[2689],"3ba34fd","There was one feedback loop that contained developers who are writing code and testing it and writing code and testing it and then hitting merge.",{"_37":2692,"_23":302,"_303":2693,"_174":2425},"673fb057e97f",[],{"_37":2695,"_23":302,"_303":2696,"_174":2698},"c8dee73911f4",[2697],"8ff0e47","They represent change.",[2700,2702],{"_37":2689,"_23":2441,"_2442":2701},"00:02:57.760",{"_37":2697,"_23":2441,"_2442":2703},"00:03:07.380",{"_37":2705,"_23":297,"_298":2706,"_306":2728,"_308":309},"36dc8c292c92",[2707,2712,2715,2720,2723],{"_37":2708,"_23":302,"_303":2709,"_174":2711},"c623029c8cc4",[2710],"65aee56","And then there was another feedback loop that represents stability, which was you get an alert.",{"_37":2713,"_23":302,"_303":2714,"_174":2425},"eccad2668f5a",[],{"_37":2716,"_23":302,"_303":2717,"_174":2719},"c40f415e9a25",[2718],"f32bde7","You debug it, you change it, you get an alert, you debug it, you change it.",{"_37":2721,"_23":302,"_303":2722,"_174":2425},"92affab7c567",[],{"_37":2724,"_23":302,"_303":2725,"_174":2727},"2b6c8f6cf292",[2726],"c7370ac","And never the twain shall meet.",[2729,2731,2733],{"_37":2710,"_23":2441,"_2442":2730},"00:03:08.820",{"_37":2718,"_23":2441,"_2442":2732},"00:03:15.140",{"_37":2726,"_23":2441,"_2442":2734},"00:03:19.060",{"_37":2736,"_23":297,"_298":2737,"_306":2767,"_308":309},"cf7d34f9ece5",[2738,2743,2746,2751,2754,2759,2762],{"_37":2739,"_23":302,"_303":2740,"_174":2742},"96276c366f03",[2741],"53cfc5b","It has taken such a high level of engineering discipline",{"_37":2744,"_23":302,"_303":2745,"_174":2425},"c219f7606034",[],{"_37":2747,"_23":302,"_303":2748,"_174":2750},"9a0bbceeb203",[2749],"55b523c","and excellence to make a feedback loop that contains people writing the code and them understanding it in production, and one fluid, that it just hasn't been accessible for",{"_37":2752,"_23":302,"_303":2753,"_174":2425},"302915703bae",[],{"_37":2755,"_23":302,"_303":2756,"_174":2758},"d7327a12e77d",[2757],"b999398","the median, let alone the median engineering team. I would say the top five percent of engineering teams have been able to achieve this.",{"_37":2760,"_23":302,"_303":2761,"_174":2425},"695e554dbff7",[],{"_37":2763,"_23":302,"_303":2764,"_174":2766},"9daeb87c4be2",[2765],"4c2478b","And it is so exciting to me.",[2768,2770,2772,2774],{"_37":2741,"_23":2441,"_2442":2769},"00:03:21.780",{"_37":2749,"_23":2441,"_2442":2771},"00:03:28.379",{"_37":2757,"_23":2441,"_2442":2773},"00:03:41.680",{"_37":2765,"_23":2441,"_2442":2775},"00:03:47.680",{"_37":2777,"_23":297,"_298":2778,"_306":2800,"_308":309},"c06cb52c6663",[2779,2784,2787,2792,2795],{"_37":2780,"_23":302,"_303":2781,"_174":2783},"a3fdd798c264",[2782],"726bf67","Now, on so many ways, at so many levels, AI is attacking the status quo and it's making it so that it is easier to instrument your code, easier to get stuff out.",{"_37":2785,"_23":302,"_303":2786,"_174":2425},"78b86910633b",[],{"_37":2788,"_23":302,"_303":2789,"_174":2791},"2012ec7131b3",[2790],"d5e2b3d","Easier to understand, easier to find the starting place.",{"_37":2793,"_23":302,"_303":2794,"_174":2425},"2cbcead246ed",[],{"_37":2796,"_23":302,"_303":2797,"_174":2799},"e048174142db",[2798],"9ffc719","It is easily within the grasp of the median engineer, engineering team, or it is becoming that, and this gives me breath and life every day.",[2801,2803,2805],{"_37":2782,"_23":2441,"_2442":2802},"00:03:50.720",{"_37":2790,"_23":2441,"_2442":2804},"00:04:00.340",{"_37":2798,"_23":2441,"_2442":2806},"00:04:03.220",{"_37":2808,"_23":297,"_298":2809,"_306":2843,"_308":309},"27e6365c9566",[2810,2815,2819,2822,2827,2830,2835,2838],{"_37":2811,"_23":302,"_303":2812,"_174":2814},"a8dd9a82d1cc",[2813,1281],"3e56b3f","Sarah Sachs:",{"_37":2816,"_23":302,"_303":2817,"_174":2818},"3d2231460aa2",[2813]," Hi, I'm Sarah.",{"_37":2820,"_23":302,"_303":2821,"_174":2425},"777c660d97e4",[],{"_37":2823,"_23":302,"_303":2824,"_174":2826},"e486cd61a1f8",[2825],"0688d2f","I lead our AI teams at Notion.",{"_37":2828,"_23":302,"_303":2829,"_174":2425},"d7c499fea033",[],{"_37":2831,"_23":302,"_303":2832,"_174":2834},"04fcd0914a5c",[2833],"57596ef","Notion's a little bit on a higher level in terms of the application space,",{"_37":2836,"_23":302,"_303":2837,"_174":2425},"f8e49bf2cd45",[],{"_37":2839,"_23":302,"_303":2840,"_174":2842},"333dc32eb131",[2841],"6cfdba0","an enterprise collaboration tool.",[2844,2846,2848,2850],{"_37":2813,"_23":2441,"_2442":2845},"00:04:13.200",{"_37":2825,"_23":2441,"_2442":2847},"00:04:14.400",{"_37":2833,"_23":2441,"_2442":2849},"00:04:16.799",{"_37":2841,"_23":2441,"_2442":2851},"00:04:20.940",{"_37":2853,"_23":297,"_298":2854,"_306":2876,"_308":309},"de50467c0178",[2855,2860,2863,2868,2871],{"_37":2856,"_23":302,"_303":2857,"_174":2859},"753ae6ca6c72",[2858],"216a12e","But I think in the past probably two years have heavily transitioned to actually being an AI-first collaboration tool.",{"_37":2861,"_23":302,"_303":2862,"_174":2425},"409eb9163493",[],{"_37":2864,"_23":302,"_303":2865,"_174":2867},"5eb8b327958f",[2866],"c61d33c","And that's been quite successful for us.",{"_37":2869,"_23":302,"_303":2870,"_174":2425},"0be0f415c3eb",[],{"_37":2872,"_23":302,"_303":2873,"_174":2875},"733bd827dd45",[2874],"ba2237d","Big actual users of both of these products in my life and quite big fans.",[2877,2879,2881],{"_37":2858,"_23":2441,"_2442":2878},"00:04:23.020",{"_37":2866,"_23":2441,"_2442":2880},"00:04:31.080",{"_37":2874,"_23":2441,"_2442":2882},"00:04:33.800",{"_37":2884,"_23":297,"_298":2885,"_306":2907,"_308":309},"9a037e06adfc",[2886,2891,2894,2899,2902],{"_37":2887,"_23":302,"_303":2888,"_174":2890},"2326666eeed7",[2889],"42a3362","I think what's interesting about collaborative AI is that it's almost like a misnomer.",{"_37":2892,"_23":302,"_303":2893,"_174":2425},"ecc6d2c65d07",[],{"_37":2895,"_23":302,"_303":2896,"_174":2898},"0f1eefd96ad7",[2897],"badec01","Figuring out how to have AI that understands your whole context, is self-healing based on what you want, and is also secure with the information that you have but at the same time can be shared",{"_37":2900,"_23":302,"_303":2901,"_174":2425},"fb79b6936420",[],{"_37":2903,"_23":302,"_303":2904,"_174":2906},"9def44ecada1",[2905],"67e4281","and live across different sources that all have their own permission models is a very difficult problem. It all comes back to RBAC eventually, right?",[2908,2910,2912],{"_37":2889,"_23":2441,"_2442":2909},"00:04:38.760",{"_37":2897,"_23":2441,"_2442":2911},"00:04:45.000",{"_37":2905,"_23":2441,"_2442":2913},"00:04:56.960",{"_37":2915,"_23":297,"_298":2916,"_306":2938,"_308":309},"c00534c4bcb8",[2917,2922,2925,2930,2933],{"_37":2918,"_23":302,"_303":2919,"_174":2921},"080d9697a3f6",[2920],"c2719f4","And that's something that Notion has been building.",{"_37":2923,"_23":302,"_303":2924,"_174":2425},"50454b3c78b2",[],{"_37":2926,"_23":302,"_303":2927,"_174":2929},"d3129c142db9",[2928],"1e71cdb","We launched in February our custom agents, which have been probably our most successful product ever, which has been really fun, which is about configuring those agents to do tasks and automate processes.",{"_37":2931,"_23":302,"_303":2932,"_174":2425},"964f605a1fcb",[],{"_37":2934,"_23":302,"_303":2935,"_174":2937},"7f9cc763787d",[2936],"68f9a50","And then we co-launched Managed Agents with Claude.",[2939,2941,2943],{"_37":2920,"_23":2441,"_2442":2940},"00:05:04.660",{"_37":2928,"_23":2441,"_2442":2942},"00:05:07.380",{"_37":2936,"_23":2441,"_2442":2944},"00:05:18.840",{"_37":2946,"_23":297,"_298":2947,"_306":2969,"_308":309},"79198f18b8dd",[2948,2953,2956,2961,2964],{"_37":2949,"_23":302,"_303":2950,"_174":2952},"cb538177ffc0",[2951],"e7d95b1","And then I think next Thursday we have our big Make with Notion conference again, where I don't think it's as secret what we'll talk about.",{"_37":2954,"_23":302,"_303":2955,"_174":2425},"eb437464f2d6",[],{"_37":2957,"_23":302,"_303":2958,"_174":2960},"12f1861b5897",[2959],"38ad5f4","Hopefully, which is how you can actually create a software factory that's model-agnostic inside Notion.",{"_37":2962,"_23":302,"_303":2963,"_174":2425},"e37ec69da43a",[],{"_37":2965,"_23":302,"_303":2966,"_174":2968},"a27ec227e1d4",[2967],"33f4ee5","So all of those things have to be done appropriately with good observation.",[2970,2972,2974],{"_37":2951,"_23":2441,"_2442":2971},"00:05:23.080",{"_37":2959,"_23":2441,"_2442":2973},"00:05:29.560",{"_37":2967,"_23":2441,"_2442":2975},"00:05:36.759",{"_37":2977,"_23":297,"_298":2978,"_306":3004,"_308":309},"9c20e9abf205",[2979,2984,2988,2991,2996,2999],{"_37":2980,"_23":302,"_303":2981,"_174":2983},"3b1006a5353c",[2982,1281],"50a8e2b","Charity:",{"_37":2985,"_23":302,"_303":2986,"_174":2987},"9dc4d8bfe9a4",[2982]," Oh, I should have said Honeycomb is an observability tool.",{"_37":2989,"_23":302,"_303":2990,"_174":2425},"557f397a18ec",[],{"_37":2992,"_23":302,"_303":2993,"_174":2995},"b1b25ad0f3ec",[2994],"06963eb","It was made for this era.",{"_37":2997,"_23":302,"_303":2998,"_174":2425},"c60e87ace3e3",[],{"_37":3000,"_23":302,"_303":3001,"_174":3003},"2e9839f8b7cd",[3002],"59ce2dc","If you haven't tried it, you should.",[3005,3007,3009],{"_37":2982,"_23":2441,"_2442":3006},"00:05:44.640",{"_37":2994,"_23":2441,"_2442":3008},"00:05:48.080",{"_37":3002,"_23":2441,"_2442":3010},"00:05:49.360",{"_37":3012,"_23":297,"_298":3013,"_306":3018,"_308":382},"fcf50c46ef1e",[3014],{"_37":3015,"_23":302,"_303":3016,"_174":3017},"8d6403b35976",[],"Where Does the Agent Live?",[],{"_37":3020,"_23":297,"_298":3021,"_306":3047,"_308":309},"2ce71cd34674",[3022,3027,3031,3034,3039,3042],{"_37":3023,"_23":302,"_303":3024,"_174":3026},"cea78aadab82",[3025,1281],"ed2ad0d","Chetan:",{"_37":3028,"_23":302,"_303":3029,"_174":3030},"c0aaec960089",[3025]," So Ben, Sarah, and Charity, much of all of us when we create product and use product, we'll end up touching each of your products in some sense.",{"_37":3032,"_23":302,"_303":3033,"_174":2425},"176fce75c0cf",[],{"_37":3035,"_23":302,"_303":3036,"_174":3038},"a258a900aba7",[3037],"67aced5","So, speaking of the phase shift, agents are living somewhere and we are utilizing these agents to touch your products.",{"_37":3040,"_23":302,"_303":3041,"_174":2425},"c79ec381aae4",[],{"_37":3043,"_23":302,"_303":3044,"_174":3046},"888c21a1db6d",[3045],"ce7636f","So obviously from a product standpoint, a primitive and a concept is created, and then a certain company owns it because they do a fantastic job in representing it.",[3048,3050,3052],{"_37":3025,"_23":2441,"_2442":3049},"00:05:52.460",{"_37":3037,"_23":2441,"_2442":3051},"00:06:04.600",{"_37":3045,"_23":2441,"_2442":3053},"00:06:15.420",{"_37":3055,"_23":297,"_298":3056,"_306":3078,"_308":309},"6190cecc0737",[3057,3062,3065,3070,3073],{"_37":3058,"_23":302,"_303":3059,"_174":3061},"bc757892d128",[3060],"de122f2","So Ben, you own flags.",{"_37":3063,"_23":302,"_303":3064,"_174":2425},"cb9185a60ff0",[],{"_37":3066,"_23":302,"_303":3067,"_174":3069},"404acdffd75c",[3068],"1450774","Charity, you want traces.",{"_37":3071,"_23":302,"_303":3072,"_174":2425},"db27b2e3b2c9",[],{"_37":3074,"_23":302,"_303":3075,"_174":3077},"49e79d2f9e75",[3076],"87f8025","Sarah, you want documents and extended brain hypothesis, because that's how we materialize our information.",[3079,3081,3083],{"_37":3060,"_23":2441,"_2442":3080},"00:06:24.940",{"_37":3068,"_23":2441,"_2442":3082},"00:06:27.120",{"_37":3076,"_23":2441,"_2442":3084},"00:06:29.840",{"_37":3086,"_23":297,"_298":3087,"_306":3109,"_308":309},"84ec0faab5fb",[3088,3093,3096,3101,3104],{"_37":3089,"_23":302,"_303":3090,"_174":3092},"15e35a8dff85",[3091],"818e000","How do you see homing that agent?",{"_37":3094,"_23":302,"_303":3095,"_174":2425},"7bc19ec65bbe",[],{"_37":3097,"_23":302,"_303":3098,"_174":3100},"43ac9b023cb5",[3099],"367e3b6","Which means the agent can stay outside your product.",{"_37":3102,"_23":302,"_303":3103,"_174":2425},"f9c59f2cae08",[],{"_37":3105,"_23":302,"_303":3106,"_174":3108},"eb2018a54e5f",[3107],"7e35004","We as users express our intent and then proxy through that agent to your products.",[3110,3112,3114],{"_37":3091,"_23":2441,"_2442":3111},"00:06:40.220",{"_37":3099,"_23":2441,"_2442":3113},"00:06:42.460",{"_37":3107,"_23":2441,"_2442":3115},"00:06:46.620",{"_37":3117,"_23":297,"_298":3118,"_306":3132,"_308":309},"a9e4a7591f7b",[3119,3124,3127],{"_37":3120,"_23":302,"_303":3121,"_174":3123},"47123c004fc6",[3122],"761adcb","Or perhaps you're going to home agents within your product, and then our agents are talking to your agents.",{"_37":3125,"_23":302,"_303":3126,"_174":2425},"e1769cdb61fc",[],{"_37":3128,"_23":302,"_303":3129,"_174":3131},"6f035f6c15f8",[3130],"42381f7","So tell us more about how you're thinking through this.",[3133,3135],{"_37":3122,"_23":2441,"_2442":3134},"00:06:53.240",{"_37":3130,"_23":2441,"_2442":3136},"00:06:59.400",{"_37":3138,"_23":297,"_298":3139,"_306":3157,"_308":309},"780f3bb7d823",[3140,3145,3149,3152],{"_37":3141,"_23":302,"_303":3142,"_174":3144},"817d9c705309",[3143,1281],"d325f0c","Sarah:",{"_37":3146,"_23":302,"_303":3147,"_174":3148},"7592ad5c4061",[3143]," Yeah, I can take a first stab.",{"_37":3150,"_23":302,"_303":3151,"_174":2425},"501c06427a09",[],{"_37":3153,"_23":302,"_303":3154,"_174":3156},"7bb289183924",[3155],"dc2e531","That was not obvious to us initially.",[3158,3160],{"_37":3143,"_23":2441,"_2442":3159},"00:07:03.320",{"_37":3155,"_23":2441,"_2442":3161},"00:07:05.320",{"_37":3163,"_23":297,"_298":3164,"_306":3186,"_308":309},"474ca6268ff8",[3165,3170,3173,3178,3181],{"_37":3166,"_23":302,"_303":3167,"_174":3169},"d52cba3c6df0",[3168],"0e62fbf","I feel like Notion probably in the past year we found a pretty good rhythm and understanding, which is that",{"_37":3171,"_23":302,"_303":3172,"_174":2425},"cb168e218042",[],{"_37":3174,"_23":302,"_303":3175,"_174":3177},"bc533db5ebf9",[3176],"2d54364","you have to focus on two things.",{"_37":3179,"_23":302,"_303":3180,"_174":2425},"1cb67e11ff11",[],{"_37":3182,"_23":302,"_303":3183,"_174":3185},"8d3eef38b574",[3184],"73cc7d2","You have to be the best system of record, whether it be for humans or agents, and you need to be the best place where those agents collaborate, whether they be native or not.",[3187,3189,3191],{"_37":3168,"_23":2441,"_2442":3188},"00:07:09.240",{"_37":3176,"_23":2441,"_2442":3190},"00:07:15.539",{"_37":3184,"_23":2441,"_2442":3192},"00:07:17.220",{"_37":3194,"_23":297,"_298":3195,"_306":3217,"_308":309},"9d76f5ebcb31",[3196,3201,3204,3209,3212],{"_37":3197,"_23":302,"_303":3198,"_174":3200},"89e9edd9cb82",[3199],"ef81441","And at the same time, we want to be the best agents for doing enterprise work in Notion, right?",{"_37":3202,"_23":302,"_303":3203,"_174":2425},"c7d6a8af650e",[],{"_37":3205,"_23":302,"_303":3206,"_174":3208},"01a1a6ae4f48",[3207],"10ecccd","And ultimately those two things aren't disagreeable with each other.",{"_37":3210,"_23":302,"_303":3211,"_174":2425},"0703d013e6bc",[],{"_37":3213,"_23":302,"_303":3214,"_174":3216},"b595690c81c4",[3215],"f95b745","It's quite nice to work on both.",[3218,3220,3222],{"_37":3199,"_23":2441,"_2442":3219},"00:07:25.660",{"_37":3207,"_23":2441,"_2442":3221},"00:07:30.060",{"_37":3215,"_23":2441,"_2442":3223},"00:07:33.760",{"_37":3225,"_23":297,"_298":3226,"_306":3264,"_308":309},"ffcf899c3ab8",[3227,3232,3235,3240,3243,3248,3251,3256,3259],{"_37":3228,"_23":302,"_303":3229,"_174":3231},"ee509188fbbb",[3230],"dd0fd37","So for instance, the team that owns offline mode and collaboration inside Notion, CRDT is the technology they're using.",{"_37":3233,"_23":302,"_303":3234,"_174":2425},"3dedda91f8b2",[],{"_37":3236,"_23":302,"_303":3237,"_174":3239},"06904335953d",[3238],"88737fc","That's a huge area of investment for agents to be editing blocks in Notion, right?",{"_37":3241,"_23":302,"_303":3242,"_174":2425},"bb18aa57103d",[],{"_37":3244,"_23":302,"_303":3245,"_174":3247},"52e1fc5166e9",[3246],"5637826","If you have 2,000 employees that are all editing the same task database, how do you do",{"_37":3249,"_23":302,"_303":3250,"_174":2425},"138e4006002f",[],{"_37":3252,"_23":302,"_303":3253,"_174":3255},"705515178b7f",[3254],"d8d8225","conflict management, right?",{"_37":3257,"_23":302,"_303":3258,"_174":2425},"0245fff9c14d",[],{"_37":3260,"_23":302,"_303":3261,"_174":3263},"2b721db132d5",[3262],"8109221","That system of record's really important, and we've seen that with Notion usage.",[3265,3267,3269,3271,3273],{"_37":3230,"_23":2441,"_2442":3266},"00:07:35.520",{"_37":3238,"_23":2441,"_2442":3268},"00:07:45.720",{"_37":3246,"_23":2441,"_2442":3270},"00:07:50.840",{"_37":3254,"_23":2441,"_2442":3272},"00:07:55.960",{"_37":3262,"_23":2441,"_2442":3274},"00:07:57.080",{"_37":3276,"_23":297,"_298":3277,"_306":3307,"_308":309},"719b68440f19",[3278,3283,3286,3291,3294,3299,3302],{"_37":3279,"_23":302,"_303":3280,"_174":3282},"4e8363874043",[3281],"bd2fa1b","So I think it's something around 15% of all enterprise Notion content is written or augmented by AI now.",{"_37":3284,"_23":302,"_303":3285,"_174":2425},"ab21e17c19cc",[],{"_37":3287,"_23":302,"_303":3288,"_174":3290},"5815115b6773",[3289],"237569e","And the search traffic load is closer to 30% being done by agents.",{"_37":3292,"_23":302,"_303":3293,"_174":2425},"66dfd5139d68",[],{"_37":3295,"_23":302,"_303":3296,"_174":3298},"3dc58b20f838",[3297],"93f7a15","So it's already happening.",{"_37":3300,"_23":302,"_303":3301,"_174":2425},"b6403b292d48",[],{"_37":3303,"_23":302,"_303":3304,"_174":3306},"88ea7c0e58f2",[3305],"b34148d","Agents are doing work.",[3308,3310,3312,3314],{"_37":3281,"_23":2441,"_2442":3309},"00:08:00.520",{"_37":3289,"_23":2441,"_2442":3311},"00:08:08.560",{"_37":3297,"_23":2441,"_2442":3313},"00:08:12.639",{"_37":3305,"_23":2441,"_2442":3315},"00:08:14.720",{"_37":3317,"_23":297,"_298":3318,"_306":3332,"_308":309},"65b05e5624d6",[3319,3324,3327],{"_37":3320,"_23":302,"_303":3321,"_174":3323},"aab50e61647d",[3322],"840a716","We still have to be the best place where they collaborate, because we still believe in the fundamentals that",{"_37":3325,"_23":302,"_303":3326,"_174":2425},"d7a90020e279",[],{"_37":3328,"_23":302,"_303":3329,"_174":3331},"7604671d1c00",[3330],"0b5f51b","Notion is for collaboration and it doesn't necessarily need to be from humans.",[3333,3335],{"_37":3322,"_23":2441,"_2442":3334},"00:08:16.160",{"_37":3330,"_23":2441,"_2442":3336},"00:08:21.320",{"_37":3338,"_23":297,"_298":3339,"_306":3369,"_308":309},"ca8bceb3056a",[3340,3345,3348,3353,3356,3361,3364],{"_37":3341,"_23":302,"_303":3342,"_174":3344},"65214731c14f",[3343],"244da7c","At the same time, there are definitions that we believe",{"_37":3346,"_23":302,"_303":3347,"_174":2425},"e6002e6fda5b",[],{"_37":3349,"_23":302,"_303":3350,"_174":3352},"4ffd19f8c742",[3351],"74c4fcd","that makes enterprise knowledge work good.",{"_37":3354,"_23":302,"_303":3355,"_174":2425},"8a28e4c1e7c4",[],{"_37":3357,"_23":302,"_303":3358,"_174":3360},"8177d4da3765",[3359],"9b080ee","There's accountability, there's rigor, understanding escalations, understanding kind of",{"_37":3362,"_23":302,"_303":3363,"_174":2425},"bc024274d6de",[],{"_37":3365,"_23":302,"_303":3366,"_174":3368},"008e3ca93b58",[3367],"9714c62","the whole connected workspace and how it interacts.",[3370,3372,3374,3376],{"_37":3343,"_23":2441,"_2442":3371},"00:08:25.800",{"_37":3351,"_23":2441,"_2442":3373},"00:08:29.440",{"_37":3359,"_23":2441,"_2442":3375},"00:08:32.080",{"_37":3367,"_23":2441,"_2442":3377},"00:08:37.440",{"_37":3379,"_23":297,"_298":3380,"_306":3402,"_308":309},"4ae52fb7ebad",[3381,3386,3389,3394,3397],{"_37":3382,"_23":302,"_303":3383,"_174":3385},"80ec11ad1290",[3384],"406f788","And we invest a whole team.",{"_37":3387,"_23":302,"_303":3388,"_174":2425},"5d68f216639c",[],{"_37":3390,"_23":302,"_303":3391,"_174":3393},"7336b8af6488",[3392],"583ad9a","Some of them are here today that make that work really, really well.",{"_37":3395,"_23":302,"_303":3396,"_174":2425},"9eb321f2dbfa",[],{"_37":3398,"_23":302,"_303":3399,"_174":3401},"d35828fd0418",[3400],"af62415","And also make it work agnostic to the model.",[3403,3405,3407],{"_37":3384,"_23":2441,"_2442":3404},"00:08:39.600",{"_37":3392,"_23":2441,"_2442":3406},"00:08:42.480",{"_37":3400,"_23":2441,"_2442":3408},"00:08:47.160",{"_37":3410,"_23":297,"_298":3411,"_306":3433,"_308":309},"66df63b91324",[3412,3417,3420,3425,3428],{"_37":3413,"_23":302,"_303":3414,"_174":3416},"8f8cd3c294b4",[3415],"a5afbb3","Because we believe that we call it the Fortune 5 million.",{"_37":3418,"_23":302,"_303":3419,"_174":2425},"3432ce9ee96a",[],{"_37":3421,"_23":302,"_303":3422,"_174":3424},"923340b5c800",[3423],"b7f523e","The non-Fortune 500 companies that don't have Palantir or I guess now Accenture or whoever OpenAI is",{"_37":3426,"_23":302,"_303":3427,"_174":2425},"9db1ef3e56ae",[],{"_37":3429,"_23":302,"_303":3430,"_174":3432},"87c64b4b658b",[3431],"10aebde","partnered with to consult and set it up in their workspace, you know, the other five million companies want these workflows as well.",[3434,3436,3438],{"_37":3415,"_23":2441,"_2442":3435},"00:08:49.640",{"_37":3423,"_23":2441,"_2442":3437},"00:08:52.840",{"_37":3431,"_23":2441,"_2442":3439},"00:08:59.740",{"_37":3441,"_23":297,"_298":3442,"_306":3472,"_308":309},"625015d53739",[3443,3448,3451,3456,3459,3464,3467],{"_37":3444,"_23":302,"_303":3445,"_174":3447},"3427f393d4f6",[3446],"3c566c6","And they can't negotiate 15% discounts and they can't negotiate token max for their team.",{"_37":3449,"_23":302,"_303":3450,"_174":2425},"f432c898b808",[],{"_37":3452,"_23":302,"_303":3453,"_174":3455},"966e342ac193",[3454],"aed0237","And how do we make a product where everyone can be doing this type of work?",{"_37":3457,"_23":302,"_303":3458,"_174":2425},"acb45c654cf2",[],{"_37":3460,"_23":302,"_303":3461,"_174":3463},"04c7c1ecfec8",[3462],"38fb5a4","So we just, we do both.",{"_37":3465,"_23":302,"_303":3466,"_174":2425},"0caab5e371cd",[],{"_37":3468,"_23":302,"_303":3469,"_174":3471},"c98c46352ece",[3470],"8181d94","And it helps us that we do both at the same time.",[3473,3475,3477,3479],{"_37":3446,"_23":2441,"_2442":3474},"00:09:05.900",{"_37":3454,"_23":2441,"_2442":3476},"00:09:12.060",{"_37":3462,"_23":2441,"_2442":3478},"00:09:16.140",{"_37":3470,"_23":2441,"_2442":3480},"00:09:17.660",{"_37":3482,"_23":297,"_298":3483,"_306":3524,"_308":309},"8e0af5192c84",[3484,3488,3492,3495,3500,3503,3508,3511,3516,3519],{"_37":3485,"_23":302,"_303":3486,"_174":2983},"998c9b1eaea1",[3487,1281],"86c5eec",{"_37":3489,"_23":302,"_303":3490,"_174":3491},"706f0e5670fd",[3487]," Yeah, one of the hardest things when it comes to AI is disambiguating language.",{"_37":3493,"_23":302,"_303":3494,"_174":2425},"9f64c8c50b1e",[],{"_37":3496,"_23":302,"_303":3497,"_174":3499},"4f5f6a7eff01",[3498],"3cbb631","Oh cool, you have AI and you want observability.",{"_37":3501,"_23":302,"_303":3502,"_174":2425},"fc7ad519636a",[],{"_37":3504,"_23":302,"_303":3505,"_174":3507},"92fe806aebdb",[3506],"6e660ca","Is it AI",{"_37":3509,"_23":302,"_303":3510,"_174":2425},"c49e7247d919",[],{"_37":3512,"_23":302,"_303":3513,"_174":3515},"e3b682e911ff",[3514],"98c2011","for observability, or do you need observability for AI?",{"_37":3517,"_23":302,"_303":3518,"_174":2425},"4efccbc6ecaf",[],{"_37":3520,"_23":302,"_303":3521,"_174":3523},"94b9d8da8620",[3522],"5e52e29","And there's just like so many different categories there.",[3525,3527,3529,3531,3533],{"_37":3487,"_23":2441,"_2442":3526},"00:09:21.400",{"_37":3498,"_23":2441,"_2442":3528},"00:09:27.320",{"_37":3506,"_23":2441,"_2442":3530},"00:09:30.760",{"_37":3514,"_23":2441,"_2442":3532},"00:09:32.339",{"_37":3522,"_23":2441,"_2442":3534},"00:09:35.540",{"_37":3536,"_23":297,"_298":3537,"_306":3575,"_308":309},"49c7dc916dab",[3538,3543,3546,3551,3554,3559,3562,3567,3570],{"_37":3539,"_23":302,"_303":3540,"_174":3542},"c74bbadeb6d1",[3541],"1a25e59","Are you talking about evals?",{"_37":3544,"_23":302,"_303":3545,"_174":2425},"8e5d3dad1708",[],{"_37":3547,"_23":302,"_303":3548,"_174":3550},"077ae11bdb6e",[3549],"bd34352","Are you talking about instrumentation?",{"_37":3552,"_23":302,"_303":3553,"_174":2425},"1bb8472d82f4",[],{"_37":3555,"_23":302,"_303":3556,"_174":3558},"b018be1e3315",[3557],"10fe873","Are you talking about",{"_37":3560,"_23":302,"_303":3561,"_174":2425},"905110fbc59c",[],{"_37":3563,"_23":302,"_303":3564,"_174":3566},"0910ff1f9a6e",[3565],"566a24d","And it's just kind of a giant mess right now.",{"_37":3568,"_23":302,"_303":3569,"_174":2425},"b93d3bcca039",[],{"_37":3571,"_23":302,"_303":3572,"_174":3574},"d5204d7c456e",[3573],"15e93af","But I'm not sure I understand the question, to be honest.",[3576,3578,3580,3582,3584],{"_37":3541,"_23":2441,"_2442":3577},"00:09:39.060",{"_37":3549,"_23":2441,"_2442":3579},"00:09:40.899",{"_37":3557,"_23":2441,"_2442":3581},"00:09:42.500",{"_37":3565,"_23":2441,"_2442":3583},"00:09:44.540",{"_37":3573,"_23":2441,"_2442":3585},"00:09:48.460",{"_37":3587,"_23":297,"_298":3588,"_306":3629,"_308":309},"ebea88a19018",[3589,3593,3597,3600,3605,3608,3613,3616,3621,3624],{"_37":3590,"_23":302,"_303":3591,"_174":3026},"fe659933c3a4",[3592,1281],"c384796",{"_37":3594,"_23":302,"_303":3595,"_174":3596},"53e7e0ca6599",[3592]," Where does the agent live?",{"_37":3598,"_23":302,"_303":3599,"_174":2425},"366db133f9b1",[],{"_37":3601,"_23":302,"_303":3602,"_174":3604},"931e4498d5fa",[3603],"ebdb697","As you think through your product evolution,",{"_37":3606,"_23":302,"_303":3607,"_174":2425},"46b82b7011cb",[],{"_37":3609,"_23":302,"_303":3610,"_174":3612},"0e9b25815543",[3611],"33f132e","today we are interacting perhaps with your product from an observability standpoint through an agent.",{"_37":3614,"_23":302,"_303":3615,"_174":2425},"358850672ce6",[],{"_37":3617,"_23":302,"_303":3618,"_174":3620},"97889b348561",[3619],"cb0348c","And eventually at some point you're going to create an agent within your ecosystem.",{"_37":3622,"_23":302,"_303":3623,"_174":2425},"89b503f9571a",[],{"_37":3625,"_23":302,"_303":3626,"_174":3628},"419421d9461d",[3627],"19ec128","So how are you thinking through that?",[3630,3632,3634,3636,3638],{"_37":3592,"_23":2441,"_2442":3631},"00:09:54.660",{"_37":3603,"_23":2441,"_2442":3633},"00:09:58.260",{"_37":3611,"_23":2441,"_2442":3635},"00:10:01.339",{"_37":3619,"_23":2441,"_2442":3637},"00:10:07.259",{"_37":3627,"_23":2441,"_2442":3639},"00:10:11.660",{"_37":3641,"_23":297,"_298":3642,"_306":3659,"_308":309},"a8ec9eed0a28",[3643,3647,3651,3654],{"_37":3644,"_23":302,"_303":3645,"_174":2983},"9e3b33274e97",[3646,1281],"59e1175",{"_37":3648,"_23":302,"_303":3649,"_174":3650},"2e8cdfd86777",[3646]," Like Sarah said, you need both.",{"_37":3652,"_23":302,"_303":3653,"_174":2425},"65f7cc204e59",[],{"_37":3655,"_23":302,"_303":3656,"_174":3658},"66ac4a69ef48",[3657],"a6c6512","And not everything is agent-shaped.",[3660,3662],{"_37":3646,"_23":2441,"_2442":3661},"00:10:14.880",{"_37":3657,"_23":2441,"_2442":3663},"00:10:19.200",{"_37":3665,"_23":297,"_298":3666,"_306":3696,"_308":309},"9dbd4f0afc9e",[3667,3672,3675,3680,3683,3688,3691],{"_37":3668,"_23":302,"_303":3669,"_174":3671},"4d8fa14125cb",[3670],"a1c2858","I think that we've had several years of robust enthusiasm for non-determinism, and I feel like 2026 is the year when we're all like, oh wait, state should be sustained.",{"_37":3673,"_23":302,"_303":3674,"_174":2425},"47ce236d5b58",[],{"_37":3676,"_23":302,"_303":3677,"_174":3679},"f5086e8518d9",[3678],"92ba222","It's persistent sometimes.",{"_37":3681,"_23":302,"_303":3682,"_174":2425},"918f8c5e40e7",[],{"_37":3684,"_23":302,"_303":3685,"_174":3687},"8187b39b5153",[3686],"0fc8454","Like, oh, we need fast feedback loops.",{"_37":3689,"_23":302,"_303":3690,"_174":2425},"9e7b98c69dcc",[],{"_37":3692,"_23":302,"_303":3693,"_174":3695},"2891107d9a1a",[3694],"d93b306","All of these first principles of software development were kind of like, oh right, that still matters.",[3697,3699,3701,3703],{"_37":3670,"_23":2441,"_2442":3698},"00:10:24.639",{"_37":3678,"_23":2441,"_2442":3700},"00:10:37.540",{"_37":3686,"_23":2441,"_2442":3702},"00:10:39.060",{"_37":3694,"_23":2441,"_2442":3704},"00:10:41.380",{"_37":3706,"_23":297,"_298":3707,"_306":3729,"_308":309},"e8ff0802ec08",[3708,3713,3716,3721,3724],{"_37":3709,"_23":302,"_303":3710,"_174":3712},"3f68ce2a3b14",[3711],"a1e4f6a","And it's less about",{"_37":3714,"_23":302,"_303":3715,"_174":2425},"0b546319b5a4",[],{"_37":3717,"_23":302,"_303":3718,"_174":3720},"cf194b99c5d0",[3719],"8b5e2d9","the new thing can solve everything.",{"_37":3722,"_23":302,"_303":3723,"_174":2425},"ef81082bb784",[],{"_37":3725,"_23":302,"_303":3726,"_174":3728},"d715c00b2787",[3727],"e356727","It's where do you apply each discipline?",[3730,3732,3734],{"_37":3711,"_23":2441,"_2442":3731},"00:10:45.940",{"_37":3719,"_23":2441,"_2442":3733},"00:10:47.980",{"_37":3727,"_23":2441,"_2442":3735},"00:10:49.660",{"_37":3737,"_23":297,"_298":3738,"_306":3764,"_308":309},"485dda43f18f",[3739,3744,3748,3751,3756,3759],{"_37":3740,"_23":302,"_303":3741,"_174":3743},"63f52519d89c",[3742,1281],"66aef65","Ben:",{"_37":3745,"_23":302,"_303":3746,"_174":3747},"127373df1aff",[3742]," I agree with everything that's been said so far, so I won't repeat that.",{"_37":3749,"_23":302,"_303":3750,"_174":2425},"e94683974bb8",[],{"_37":3752,"_23":302,"_303":3753,"_174":3755},"cd70d23e2eb6",[3754],"798685c","The thing I want to add is how important it is that your system of record and your data is intermingled and that it's connected.",{"_37":3757,"_23":302,"_303":3758,"_174":2425},"366dd9c97a6a",[],{"_37":3760,"_23":302,"_303":3761,"_174":3763},"93af408e7006",[3762],"b052702","As a concrete example, for LaunchDarkly, we have feature flags, that's well understood.",[3765,3767,3769],{"_37":3742,"_23":2441,"_2442":3766},"00:10:55.820",{"_37":3754,"_23":2441,"_2442":3768},"00:10:58.940",{"_37":3762,"_23":2441,"_2442":3770},"00:11:07.940",{"_37":3772,"_23":297,"_298":3773,"_306":3811,"_308":309},"78b410772066",[3774,3779,3782,3787,3790,3795,3798,3803,3806],{"_37":3775,"_23":302,"_303":3776,"_174":3778},"d28ae1988473",[3777],"4b7b4bb","We also have this concept of guarded releases, where",{"_37":3780,"_23":302,"_303":3781,"_174":2425},"9e1909f29080",[],{"_37":3783,"_23":302,"_303":3784,"_174":3786},"aef2fb7d9738",[3785],"767135a","when a change gets rolled out through a percentage rollout, it can automatically get rolled back if the observability data shows that",{"_37":3788,"_23":302,"_303":3789,"_174":2425},"c014c74f77bb",[],{"_37":3791,"_23":302,"_303":3792,"_174":3794},"c28ef59734ba",[3793],"8d92850","whatever business or observability operational metric is important to you has tripped beyond your threshold.",{"_37":3796,"_23":302,"_303":3797,"_174":2425},"6a9650e4aa5a",[],{"_37":3799,"_23":302,"_303":3800,"_174":3802},"0cc6331ac98b",[3801],"9e1249e","And so",{"_37":3804,"_23":302,"_303":3805,"_174":2425},"f27d83e684d9",[],{"_37":3807,"_23":302,"_303":3808,"_174":3810},"ac90aa066607",[3809],"1f2e25e","our observability data is connected to our feature flag data.",[3812,3814,3816,3818,3820],{"_37":3777,"_23":2441,"_2442":3813},"00:11:13.300",{"_37":3785,"_23":2441,"_2442":3815},"00:11:16.820",{"_37":3793,"_23":2441,"_2442":3817},"00:11:25.220",{"_37":3801,"_23":2441,"_2442":3819},"00:11:32.820",{"_37":3809,"_23":2441,"_2442":3821},"00:11:34.120",{"_37":3823,"_23":297,"_298":3824,"_306":3862,"_308":309},"4e716396014b",[3825,3830,3833,3838,3841,3846,3849,3854,3857],{"_37":3826,"_23":302,"_303":3827,"_174":3829},"8bcc8ef0cb0c",[3828],"8c4c9a7","And every different feature or product within LaunchDarkly is",{"_37":3831,"_23":302,"_303":3832,"_174":2425},"b3a02ef1a7dd",[],{"_37":3834,"_23":302,"_303":3835,"_174":3837},"e965ce62a809",[3836],"81fefab","interconnected.",{"_37":3839,"_23":302,"_303":3840,"_174":2425},"6405a2253c33",[],{"_37":3842,"_23":302,"_303":3843,"_174":3845},"6a542eb25824",[3844],"fb0cb05","And so I think having interconnected data like that, whether it's within a single consolidated tool or across multiple platforms,",{"_37":3847,"_23":302,"_303":3848,"_174":2425},"5df589e097e0",[],{"_37":3850,"_23":302,"_303":3851,"_174":3853},"a5c019fa3d68",[3852],"327e1c8","through a deep set of integrations.",{"_37":3855,"_23":302,"_303":3856,"_174":2425},"92bb84a42e04",[],{"_37":3858,"_23":302,"_303":3859,"_174":3861},"4b8279581cce",[3860],"7159bf5","I think that is critical towards our agents being able to be truly effective operators of our system.",[3863,3865,3867,3869,3871],{"_37":3828,"_23":2441,"_2442":3864},"00:11:37.320",{"_37":3836,"_23":2441,"_2442":3866},"00:11:43.060",{"_37":3844,"_23":2441,"_2442":3868},"00:11:43.620",{"_37":3852,"_23":2441,"_2442":3870},"00:11:53.240",{"_37":3860,"_23":2441,"_2442":3872},"00:11:56.120",{"_37":3874,"_23":297,"_298":3875,"_306":3892,"_308":309},"f84fab38695d",[3876,3880,3884,3887],{"_37":3877,"_23":302,"_303":3878,"_174":2983},"a8117ecc29ba",[3879,1281],"b80e61d",{"_37":3881,"_23":302,"_303":3882,"_174":3883},"849a7b4f6cb7",[3879]," Context is everything.",{"_37":3885,"_23":302,"_303":3886,"_174":2425},"f58ed9fc9a1e",[],{"_37":3888,"_23":302,"_303":3889,"_174":3891},"e5d9f4f105c9",[3890],"47ebd1c","Your model is only as good as the shit you trained it on.",[3893,3895],{"_37":3879,"_23":2441,"_2442":3894},"00:12:03.660",{"_37":3890,"_23":2441,"_2442":3896},"00:12:05.740",{"_37":3898,"_23":297,"_298":3899,"_306":3929,"_308":309},"7ed798e697bd",[3900,3905,3908,3913,3916,3921,3924],{"_37":3901,"_23":302,"_303":3902,"_174":3904},"f6654972e54c",[3903],"329f576","And I feel like one of the most interesting things I've seen in the past",{"_37":3906,"_23":302,"_303":3907,"_174":2425},"15c62102cb19",[],{"_37":3909,"_23":302,"_303":3910,"_174":3912},"52a9412b1caa",[3911],"49cfa2b","couple of months was some of the AI SRE startups that have been talking about how this canned, like, metrics, logs,",{"_37":3914,"_23":302,"_303":3915,"_174":2425},"af01f470feb1",[],{"_37":3917,"_23":302,"_303":3918,"_174":3920},"a6927fe9c4a7",[3919],"a61f84e","and traces data that we give our agents, they bypass it 70% of the time.",{"_37":3922,"_23":302,"_303":3923,"_174":2425},"b136bcfd6655",[],{"_37":3925,"_23":302,"_303":3926,"_174":3928},"c9dd0cc2dc78",[3927],"9ca4f79","And they look for richer data sources upstream.",[3930,3932,3934,3936],{"_37":3903,"_23":2441,"_2442":3931},"00:12:09.259",{"_37":3911,"_23":2441,"_2442":3933},"00:12:13.980",{"_37":3919,"_23":2441,"_2442":3935},"00:12:21.540",{"_37":3927,"_23":2441,"_2442":3937},"00:12:26.580",{"_37":3939,"_23":297,"_298":3940,"_306":3962,"_308":693},"47a1a7e50427",[3941,3946,3949,3954,3957],{"_37":3942,"_23":302,"_303":3943,"_174":3945},"5d2d09c6c80f",[3944],"e7d19dc","I feel like the whole metrics, logs, and trace, it's a thing that needs to die.",{"_37":3947,"_23":302,"_303":3948,"_174":2425},"726d600d9f09",[],{"_37":3950,"_23":302,"_303":3951,"_174":3953},"d694822afb8c",[3952],"453de67","Metrics and logs are infrastructure tools.",{"_37":3955,"_23":302,"_303":3956,"_174":2425},"08a834418357",[],{"_37":3958,"_23":302,"_303":3959,"_174":3961},"d78b9dfd6f1f",[3960],"5290648","Trace data is what you need for AI, period.",[3963,3965,3967],{"_37":3944,"_23":2441,"_2442":3964},"00:12:29.259",{"_37":3952,"_23":2441,"_2442":3966},"00:12:33.579",{"_37":3960,"_23":2441,"_2442":3968},"00:12:35.899",{"_37":3970,"_23":297,"_298":3971,"_306":4012,"_308":309},"2cffcca98504",[3972,3976,3980,3983,3988,3991,3996,3999,4004,4007],{"_37":3973,"_23":302,"_303":3974,"_174":3144},"94e987726575",[3975,1281],"d842d36",{"_37":3977,"_23":302,"_303":3978,"_174":3979},"3752097cd500",[3975]," Yeah, I can speak maybe to how we build on top of that for our agent workflow.",{"_37":3981,"_23":302,"_303":3982,"_174":2425},"9e046e7c243d",[],{"_37":3984,"_23":302,"_303":3985,"_174":3987},"030d4ff71d98",[3986],"caa3ecd","So number one, we have a bunch of Honeycomb dashboards that alert when tools error out.",{"_37":3989,"_23":302,"_303":3990,"_174":2425},"b162b587e0d6",[],{"_37":3992,"_23":302,"_303":3993,"_174":3995},"b4db4ad49cc2",[3994],"064f050","And if our SQL query tool dies, you better believe a ton of people are getting paged.",{"_37":3997,"_23":302,"_303":3998,"_174":2425},"6cf9dac6fe7f",[],{"_37":4000,"_23":302,"_303":4001,"_174":4003},"814a2858adb5",[4002],"e522fc0","And they're not getting paged because an LLM is deciding that these traces aren't performing SQL well, right?",{"_37":4005,"_23":302,"_303":4006,"_174":2425},"a9e3f97266d6",[],{"_37":4008,"_23":302,"_303":4009,"_174":4011},"242d8e12b2d8",[4010],"9667b18","And so it's still really important.",[4013,4015,4017,4019,4021],{"_37":3975,"_23":2441,"_2442":4014},"00:12:39.720",{"_37":3986,"_23":2441,"_2442":4016},"00:12:43.720",{"_37":3994,"_23":2441,"_2442":4018},"00:12:49.160",{"_37":4002,"_23":2441,"_2442":4020},"00:12:54.339",{"_37":4010,"_23":2441,"_2442":4022},"00:13:00.980",{"_37":4024,"_23":297,"_298":4025,"_306":4071,"_308":309},"a9c90545ce43",[4026,4031,4034,4039,4042,4047,4050,4055,4058,4063,4066],{"_37":4027,"_23":302,"_303":4028,"_174":4030},"454969d6e9c6",[4029],"bec13be","What I think is interesting is that",{"_37":4032,"_23":302,"_303":4033,"_174":2425},"75e03526a55b",[],{"_37":4035,"_23":302,"_303":4036,"_174":4038},"45e63d534cfc",[4037],"fc290a7","we kind of spoke about evals in a very qualitative sense for a very long time.",{"_37":4040,"_23":302,"_303":4041,"_174":2425},"8efbc6525cec",[],{"_37":4043,"_23":302,"_303":4044,"_174":4046},"eb1309a05e78",[4045],"b28767f","And actually with coding agents, they're getting quite good at observing and",{"_37":4048,"_23":302,"_303":4049,"_174":2425},"28d3cb298674",[],{"_37":4051,"_23":302,"_303":4052,"_174":4054},"d4fcae74a791",[4053],"a09641a","creating deterministic signals out of themselves.",{"_37":4056,"_23":302,"_303":4057,"_174":2425},"6950d5a08a3c",[],{"_37":4059,"_23":302,"_303":4060,"_174":4062},"ca9be839dfe0",[4061],"d11c14c","So you can imagine taking all of your thumbs-down feedback or any errored log, depending on your observability permissions with your customers, and having an LLM",{"_37":4064,"_23":302,"_303":4065,"_174":2425},"015ea30adf21",[],{"_37":4067,"_23":302,"_303":4068,"_174":4070},"f30511aa9d9a",[4069],"ef301e1","actually analyze and summarize what those issues are.",[4072,4074,4076,4078,4080,4082],{"_37":4029,"_23":2441,"_2442":4073},"00:13:03.540",{"_37":4037,"_23":2441,"_2442":4075},"00:13:06.180",{"_37":4045,"_23":2441,"_2442":4077},"00:13:12.020",{"_37":4053,"_23":2441,"_2442":4079},"00:13:16.540",{"_37":4061,"_23":2441,"_2442":4081},"00:13:18.940",{"_37":4069,"_23":2441,"_2442":4083},"00:13:28.320",{"_37":4085,"_23":297,"_298":4086,"_306":4116,"_308":309},"38a00e5eded0",[4087,4092,4095,4100,4103,4108,4111],{"_37":4088,"_23":302,"_303":4089,"_174":4091},"9bb232b22217",[4090],"00574f6","But at the end of the day, our team every Friday,",{"_37":4093,"_23":302,"_303":4094,"_174":2425},"62249da90b47",[],{"_37":4096,"_23":302,"_303":4097,"_174":4099},"1a7e18cf3be5",[4098],"3b35a8a","we all sit in a room and we take the most expensive Notion interaction of the week and we go step by step, tool call by tool call, compaction by compaction, and discuss for 45 minutes why it took so long.",{"_37":4101,"_23":302,"_303":4102,"_174":2425},"e8474d375f52",[],{"_37":4104,"_23":302,"_303":4105,"_174":4107},"f03ff3c26a89",[4106],"555a9b1","And we cut like 13 bugs a week.",{"_37":4109,"_23":302,"_303":4110,"_174":2425},"d309dfc34ca8",[],{"_37":4112,"_23":302,"_303":4113,"_174":4115},"23c49fa43012",[4114],"b35ece3","And you still have to do that, at least today.",[4117,4119,4121,4123],{"_37":4090,"_23":2441,"_2442":4118},"00:13:32.160",{"_37":4098,"_23":2441,"_2442":4120},"00:13:36.200",{"_37":4106,"_23":2441,"_2442":4122},"00:13:49.079",{"_37":4114,"_23":2441,"_2442":4124},"00:13:52.279",{"_37":4126,"_23":297,"_298":4127,"_306":4144,"_308":309},"0dfc67bbaf9a",[4128,4132,4136,4139],{"_37":4129,"_23":302,"_303":4130,"_174":3743},"db7f440f2949",[4131,1281],"98faa11",{"_37":4133,"_23":302,"_303":4134,"_174":4135},"f736ccba7724",[4131]," By the way, you mentioned qualitative signals.",{"_37":4137,"_23":302,"_303":4138,"_174":2425},"fddb2afecad7",[],{"_37":4140,"_23":302,"_303":4141,"_174":4143},"172d00f2149a",[4142],"077625a","I think it's so interesting how agents are evolving today.",[4145,4147],{"_37":4131,"_23":2441,"_2442":4146},"00:13:57.360",{"_37":4142,"_23":2441,"_2442":4148},"00:13:59.839",{"_37":4150,"_23":297,"_298":4151,"_306":4173,"_308":309},"0e9acad3d21c",[4152,4157,4160,4165,4168],{"_37":4153,"_23":302,"_303":4154,"_174":4156},"85f77b17bdc0",[4155],"f67c070","Maybe a year ago, which",{"_37":4158,"_23":302,"_303":4159,"_174":2425},"69ea43be66f2",[],{"_37":4161,"_23":302,"_303":4162,"_174":4164},"ccc7ce6efb72",[4163],"d1a9f86","really isn't that long ago, but in agent-speak a very long time ago, we felt like it was far more necessary to have quantitative data that was sent to our agents because qualitative data couldn't be understood well enough.",{"_37":4166,"_23":302,"_303":4167,"_174":2425},"c38c265d6995",[],{"_37":4169,"_23":302,"_303":4170,"_174":4172},"f62fd9e4aa49",[4171],"7f141db","Whereas these days, quantitative or qualitative can end up with the same decisions.",[4174,4176,4178],{"_37":4155,"_23":2441,"_2442":4175},"00:14:03.519",{"_37":4163,"_23":2441,"_2442":4177},"00:14:05.220",{"_37":4171,"_23":2441,"_2442":4179},"00:14:18.200",{"_37":4181,"_23":297,"_298":4182,"_306":4207,"_308":309},"6722129e0481",[4183,4187,4191,4194,4199,4202],{"_37":4184,"_23":302,"_303":4185,"_174":3144},"6313ea851f28",[4186,1281],"1448dd2",{"_37":4188,"_23":302,"_303":4189,"_174":4190},"7ded1c857766",[4186]," And it's also how you eval. So again, this is all contingent on the usage that we have visibility into.",{"_37":4192,"_23":302,"_303":4193,"_174":2425},"9c3278b31f60",[],{"_37":4195,"_23":302,"_303":4196,"_174":4198},"7df46d922164",[4197],"36bf481","I wanna",{"_37":4200,"_23":302,"_303":4201,"_174":2425},"d16a2ce64a5a",[],{"_37":4203,"_23":302,"_303":4204,"_174":4206},"2618b8bb4d69",[4205],"820ebe0","make sure I state, but we'll cluster all of the agent trajectories to understand what is the most expensive, the most interesting, the most value-driven work that customers do, and then we use that to synthetically generate our eval set.",[4208,4210,4212],{"_37":4186,"_23":2441,"_2442":4209},"00:14:23.640",{"_37":4197,"_23":2441,"_2442":4211},"00:14:31.000",{"_37":4205,"_23":2441,"_2442":4213},"00:14:31.540",{"_37":4215,"_23":297,"_298":4216,"_306":4238,"_308":309},"b7da8088150b",[4217,4222,4225,4230,4233],{"_37":4218,"_23":302,"_303":4219,"_174":4221},"62f63d2e649a",[4220],"b761e00","Right, so there's different types of eval sets.",{"_37":4223,"_23":302,"_303":4224,"_174":2425},"b230d4b07959",[],{"_37":4226,"_23":302,"_303":4227,"_174":4229},"fb0886a5b9ab",[4228],"bdff651","There's like your hill-climbing eval set to make sure that you don't have too many emojis in titles anymore.",{"_37":4231,"_23":302,"_303":4232,"_174":2425},"0b6f04095a41",[],{"_37":4234,"_23":302,"_303":4235,"_174":4237},"5e29eef07738",[4236],"0010eb0","You have your regression set to make sure that you don't keep erring out on SQL-ing.",[4239,4241,4243],{"_37":4220,"_23":2441,"_2442":4240},"00:14:44.320",{"_37":4228,"_23":2441,"_2442":4242},"00:14:46.960",{"_37":4236,"_23":2441,"_2442":4244},"00:14:51.840",{"_37":4246,"_23":297,"_298":4247,"_306":4269,"_308":309},"fb77ec25b435",[4248,4253,4256,4261,4264],{"_37":4249,"_23":302,"_303":4250,"_174":4252},"7f42a77c9486",[4251],"87e3f12","But then you also have like your scorecard, right?",{"_37":4254,"_23":302,"_303":4255,"_174":2425},"8846d71e8395",[],{"_37":4257,"_23":302,"_303":4258,"_174":4260},"c901399b2e53",[4259],"bc6da0c","Your pass rate.",{"_37":4262,"_23":302,"_303":4263,"_174":2425},"177be2edaf42",[],{"_37":4265,"_23":302,"_303":4266,"_174":4268},"5efc9248580f",[4267],"5486bf0","And we actually use clustering to define what the distribution of that eval set should be.",[4270,4272,4274],{"_37":4251,"_23":2441,"_2442":4271},"00:14:57.060",{"_37":4259,"_23":2441,"_2442":4273},"00:14:59.780",{"_37":4267,"_23":2441,"_2442":4275},"00:15:00.820",{"_37":4277,"_23":297,"_298":4278,"_306":4283,"_308":382},"92cb0f65a007",[4279],{"_37":4280,"_23":302,"_303":4281,"_174":4282},"9ad962d062e3",[],"Tabletop Exercise: The Agent That Deleted Everything",[],{"_37":4285,"_23":297,"_298":4286,"_306":4327,"_308":309},"ec2976e82128",[4287,4291,4295,4298,4303,4306,4311,4314,4319,4322],{"_37":4288,"_23":302,"_303":4289,"_174":3026},"c0f39c07e6f0",[4290,1281],"17d4517",{"_37":4292,"_23":302,"_303":4293,"_174":4294},"e230920cbdae",[4290]," Allow me to put you on the spot, Sarah.",{"_37":4296,"_23":302,"_303":4297,"_174":2425},"a2f05166a3ef",[],{"_37":4299,"_23":302,"_303":4300,"_174":4302},"611c2ec755bf",[4301],"43b3dcd","We're gonna play a tabletop exercise here.",{"_37":4304,"_23":302,"_303":4305,"_174":2425},"497248083ad7",[],{"_37":4307,"_23":302,"_303":4308,"_174":4310},"5d7aa4cd4f72",[4309],"d7c6ba0","As we know, agents operate in this vicious loop.",{"_37":4312,"_23":302,"_303":4313,"_174":2425},"3ddeb792fb5f",[],{"_37":4315,"_23":302,"_303":4316,"_174":4318},"15863a4e80f3",[4317],"d1e8a6c","Sometimes that loop could lead to elation where you get something done and you're excited.",{"_37":4320,"_23":302,"_303":4321,"_174":2425},"f681de11802e",[],{"_37":4323,"_23":302,"_303":4324,"_174":4326},"69a0ed17d0d8",[4325],"931b618","Or it could be a tweet that wakes you up the next morning that catastrophe played out.",[4328,4330,4332,4334,4336],{"_37":4290,"_23":2441,"_2442":4329},"00:15:09.100",{"_37":4301,"_23":2441,"_2442":4331},"00:15:12.060",{"_37":4309,"_23":2441,"_2442":4333},"00:15:14.820",{"_37":4317,"_23":2441,"_2442":4335},"00:15:19.060",{"_37":4325,"_23":2441,"_2442":4337},"00:15:24.940",{"_37":4339,"_23":297,"_298":4340,"_306":4362,"_308":309},"c21e0166b9f5",[4341,4346,4349,4354,4357],{"_37":4342,"_23":302,"_303":4343,"_174":4345},"4889b4410a81",[4344],"5f6feb2","So let's assume that you've released a new feature in Notion that is behind the feature flag that Ben has provided you as a product.",{"_37":4347,"_23":302,"_303":4348,"_174":2425},"a00d0c5be8c1",[],{"_37":4350,"_23":302,"_303":4351,"_174":4353},"4622b7948e8b",[4352],"74f8f41","And this is being traced on Charity's ecosystem.",{"_37":4355,"_23":302,"_303":4356,"_174":2425},"9aea4a7bc638",[],{"_37":4358,"_23":302,"_303":4359,"_174":4361},"1c502b43218d",[4360],"a78050e","And this agent is operating in a vicious loop.",[4363,4365,4367],{"_37":4344,"_23":2441,"_2442":4364},"00:15:29.900",{"_37":4352,"_23":2441,"_2442":4366},"00:15:39.279",{"_37":4360,"_23":2441,"_2442":4368},"00:15:43.680",{"_37":4370,"_23":297,"_298":4371,"_306":4401,"_308":309},"65b9d13f5830",[4372,4377,4380,4385,4388,4393,4396],{"_37":4373,"_23":302,"_303":4374,"_174":4376},"213d5f9c0b0b",[4375],"4c4c532","So it is constrained.",{"_37":4378,"_23":302,"_303":4379,"_174":2425},"4dac2eff2c3c",[],{"_37":4381,"_23":302,"_303":4382,"_174":4384},"52f1fd66f41d",[4383],"66baa5b","It realizes that this feature is not available.",{"_37":4386,"_23":302,"_303":4387,"_174":2425},"5bf86c22bb02",[],{"_37":4389,"_23":302,"_303":4390,"_174":4392},"f0ca10b10aa9",[4391],"02e1319","It finds a way to privilege escalate by flipping the flag on Ben's product.",{"_37":4394,"_23":302,"_303":4395,"_174":2425},"205231834790",[],{"_37":4397,"_23":302,"_303":4398,"_174":4400},"dbdc37237275",[4399],"fd3be7d","And then that leads to all documents being deleted in the user's ecosystem in Notion.",[4402,4404,4406,4408],{"_37":4375,"_23":2441,"_2442":4403},"00:15:47.460",{"_37":4383,"_23":2441,"_2442":4405},"00:15:49.060",{"_37":4391,"_23":2441,"_2442":4407},"00:15:52.100",{"_37":4399,"_23":2441,"_2442":4409},"00:15:58.100",{"_37":4411,"_23":297,"_298":4412,"_306":4442,"_308":309},"5233fbc3e245",[4413,4418,4421,4426,4429,4434,4437],{"_37":4414,"_23":302,"_303":4415,"_174":4417},"9cb8c2b3ff9d",[4416],"de71f21","And then an engineer wakes up looking at the dashboards that Charity produces with span trees and traces.",{"_37":4419,"_23":302,"_303":4420,"_174":2425},"706c4ab8dfcf",[],{"_37":4422,"_23":302,"_303":4423,"_174":4425},"fc8846d1787f",[4424],"55c667c","And you gotta work backwards to figure out what you need to do.",{"_37":4427,"_23":302,"_303":4428,"_174":2425},"3a26077885ab",[],{"_37":4430,"_23":302,"_303":4431,"_174":4433},"81a65e5102db",[4432],"c6e97f7","So in this new world, how would you react?",{"_37":4435,"_23":302,"_303":4436,"_174":2425},"3297d731e17b",[],{"_37":4438,"_23":302,"_303":4439,"_174":4441},"fe60a9d3644a",[4440],"35f4f18","And then we'll take it to Charity and Ben as well.",[4443,4445,4447,4449],{"_37":4416,"_23":2441,"_2442":4444},"00:16:04.500",{"_37":4424,"_23":2441,"_2442":4446},"00:16:11.720",{"_37":4432,"_23":2441,"_2442":4448},"00:16:14.760",{"_37":4440,"_23":2441,"_2442":4450},"00:16:17.400",{"_37":4452,"_23":297,"_298":4453,"_306":4494,"_308":309},"64014977c87a",[4454,4458,4462,4465,4470,4473,4478,4481,4486,4489],{"_37":4455,"_23":302,"_303":4456,"_174":3144},"9222775622e0",[4457,1281],"e19d873",{"_37":4459,"_23":302,"_303":4460,"_174":4461},"a87b145ba880",[4457]," You don't want to talk about why that wouldn't happen?",{"_37":4463,"_23":302,"_303":4464,"_174":2425},"b7532d63975d",[],{"_37":4466,"_23":302,"_303":4467,"_174":4469},"bc3523668274",[4468],"9c57002","We're assuming that happens.",{"_37":4471,"_23":302,"_303":4472,"_174":2425},"66a3fe6a9f1c",[],{"_37":4474,"_23":302,"_303":4475,"_174":4477},"94a379968d34",[4476],"d4f9e2e","Pretty much.",{"_37":4479,"_23":302,"_303":4480,"_174":2425},"cf3099f48490",[],{"_37":4482,"_23":302,"_303":4483,"_174":4485},"7649f04c8cfc",[4484],"6d34218","Yeah, okay.",{"_37":4487,"_23":302,"_303":4488,"_174":2425},"f412212b36ff",[],{"_37":4490,"_23":302,"_303":4491,"_174":4493},"2c772b07844b",[4492],"810db25","Never role-played that one.",[4495,4497,4499,4501,4503],{"_37":4457,"_23":2441,"_2442":4496},"00:16:19.640",{"_37":4468,"_23":2441,"_2442":4498},"00:16:23.420",{"_37":4476,"_23":2441,"_2442":4500},"00:16:24.940",{"_37":4484,"_23":2441,"_2442":4502},"00:16:25.500",{"_37":4492,"_23":2441,"_2442":4504},"00:16:26.940",{"_37":4506,"_23":297,"_298":4507,"_306":4513,"_308":309},"b891a07fddc6",[4508],{"_37":4509,"_23":302,"_303":4510,"_174":4512},"522ceabfe2b9",[4511],"0586882","So the question is if everything gets deleted from Notion, how do I use AI to fix it?",[4514],{"_37":4511,"_23":2441,"_2442":4515},"00:16:29.660",{"_37":4517,"_23":297,"_298":4518,"_306":4527,"_308":309},"0b9f5d346ef6",[4519,4523],{"_37":4520,"_23":302,"_303":4521,"_174":3026},"90e587cd0162",[4522,1281],"43f18ef",{"_37":4524,"_23":302,"_303":4525,"_174":4526},"42ca331d44cd",[4522]," Exactly.",[4528],{"_37":4522,"_23":2441,"_2442":4529},"00:16:35.579",{"_37":4531,"_23":297,"_298":4532,"_306":4565,"_308":309},"f2606fff9292",[4533,4537,4541,4544,4549,4552,4557,4560],{"_37":4534,"_23":302,"_303":4535,"_174":3144},"83cbafb1fab0",[4536,1281],"826ff4f",{"_37":4538,"_23":302,"_303":4539,"_174":4540},"406a756a6ce2",[4536]," I mean, okay, so a couple of things.",{"_37":4542,"_23":302,"_303":4543,"_174":2425},"a860d30fe137",[],{"_37":4545,"_23":302,"_303":4546,"_174":4548},"0eb427f1ea77",[4547],"429e819","Number one, I actually think that this is my conviction.",{"_37":4550,"_23":302,"_303":4551,"_174":2425},"6c42a91344da",[],{"_37":4553,"_23":302,"_303":4554,"_174":4556},"8a3134ef4054",[4555],"044bab4","Obviously",{"_37":4558,"_23":302,"_303":4559,"_174":2425},"1b780060613c",[],{"_37":4561,"_23":302,"_303":4562,"_174":4564},"e4f66b67c785",[4563],"129bac8","super, super, super biased. Why Notion has an advantage with enterprise collaboration with agents is that version control is really effing hard.",[4566,4568,4570,4572],{"_37":4536,"_23":2441,"_2442":4567},"00:16:39.620",{"_37":4547,"_23":2441,"_2442":4569},"00:16:42.020",{"_37":4555,"_23":2441,"_2442":4571},"00:16:46.020",{"_37":4563,"_23":2441,"_2442":4573},"00:16:46.940",{"_37":4575,"_23":297,"_298":4576,"_306":4590,"_308":309},"5732a0b1b869",[4577,4582,4585],{"_37":4578,"_23":302,"_303":4579,"_174":4581},"ed6f732de290",[4580],"4ab5aea","It's very complicated.",{"_37":4583,"_23":302,"_303":4584,"_174":2425},"af1841fd2a2c",[],{"_37":4586,"_23":302,"_303":4587,"_174":4589},"6425b4495c06",[4588],"6b9424a","Attribution is very complicated, particularly when you have different primitive types.",[4591,4593],{"_37":4580,"_23":2441,"_2442":4592},"00:16:56.020",{"_37":4588,"_23":2441,"_2442":4594},"00:16:58.020",{"_37":4596,"_23":297,"_298":4597,"_306":4635,"_308":309},"d01e2ec2ae23",[4598,4603,4606,4611,4614,4619,4622,4627,4630],{"_37":4599,"_23":302,"_303":4600,"_174":4602},"63122a932fc0",[4601],"c3a5cb8","And so I'm assuming not all of Notion Postgres has been deleted.",{"_37":4604,"_23":302,"_303":4605,"_174":2425},"4e665963521f",[],{"_37":4607,"_23":302,"_303":4608,"_174":4610},"1179a82f96f0",[4609],"3d461b1","We would be in a lot of trouble if everything was lost.",{"_37":4612,"_23":302,"_303":4613,"_174":2425},"b5b120bb80c7",[],{"_37":4615,"_23":302,"_303":4616,"_174":4618},"8a1077c21498",[4617],"957dd50","I'm not the SRE here, but we have version control and capabilities, and actually undo was one of the most blocking things that we had to build in our product.",{"_37":4620,"_23":302,"_303":4621,"_174":2425},"e702b3382abb",[],{"_37":4623,"_23":302,"_303":4624,"_174":4626},"497f005b6d89",[4625],"3bb6b2a","Being able to undo what an agent did but not undo how the human edited what the agent did, or maybe if the human's editing the agent's edit, you undo that too.",{"_37":4628,"_23":302,"_303":4629,"_174":2425},"267ceec4e360",[],{"_37":4631,"_23":302,"_303":4632,"_174":4634},"d2826bbbc422",[4633],"8e37dc3","So we've had to invest a lot of technology into making that work.",[4636,4638,4640,4642,4644],{"_37":4601,"_23":2441,"_2442":4637},"00:17:02.340",{"_37":4609,"_23":2441,"_2442":4639},"00:17:06.339",{"_37":4617,"_23":2441,"_2442":4641},"00:17:10.560",{"_37":4625,"_23":2441,"_2442":4643},"00:17:20.679",{"_37":4633,"_23":2441,"_2442":4645},"00:17:28.840",{"_37":4647,"_23":297,"_298":4648,"_306":4662,"_308":693},"0b068d8ec0f4",[4649,4654,4657],{"_37":4650,"_23":302,"_303":4651,"_174":4653},"f5d239a230ed",[4652],"94351d9","Claude Code is only successful because presumably GitHub is working, which is that version control and attribution are actually the most important thing for autonomy,",{"_37":4655,"_23":302,"_303":4656,"_174":2425},"7e798db060c9",[],{"_37":4658,"_23":302,"_303":4659,"_174":4661},"218afc4cd5c6",[4660],"0f69a88","in my perspective.",[4663,4665],{"_37":4652,"_23":2441,"_2442":4664},"00:17:34.040",{"_37":4660,"_23":2441,"_2442":4666},"00:17:45.340",{"_37":4668,"_23":297,"_298":4669,"_306":4699,"_308":309},"e4dffbfd9453",[4670,4675,4678,4683,4686,4691,4694],{"_37":4671,"_23":302,"_303":4672,"_174":4674},"66ccaad62105",[4673],"2208a1d","I don't know if that answers your question.",{"_37":4676,"_23":302,"_303":4677,"_174":2425},"01b62e7facd6",[],{"_37":4679,"_23":302,"_303":4680,"_174":4682},"f911a6046519",[4681],"72578ce","I don't really know the answer to your question.",{"_37":4684,"_23":302,"_303":4685,"_174":2425},"998ae6647187",[],{"_37":4687,"_23":302,"_303":4688,"_174":4690},"6c0a94d970b5",[4689],"18e4af9","No, but we would be effed if that happened.",{"_37":4692,"_23":302,"_303":4693,"_174":2425},"625560d00d54",[],{"_37":4695,"_23":302,"_303":4696,"_174":4698},"ad9c50231f8a",[4697],"d58ec5b","It's a leading question.",[4700,4702,4704,4706],{"_37":4673,"_23":2441,"_2442":4701},"00:17:46.539",{"_37":4681,"_23":2441,"_2442":4703},"00:17:47.820",{"_37":4689,"_23":2441,"_2442":4705},"00:17:49.580",{"_37":4697,"_23":2441,"_2442":4707},"00:17:52.059",{"_37":4709,"_23":297,"_298":4710,"_306":4732,"_308":309},"eb89ae98a4b0",[4711,4716,4719,4724,4727],{"_37":4712,"_23":302,"_303":4713,"_174":4715},"52deea5611b8",[4714],"58b09a8","I think the more important thing is that when we use coding agents, we also have those coding agents, in our skills markdown files, require Honeycomb observability metrics to be built.",{"_37":4717,"_23":302,"_303":4718,"_174":2425},"e3c286aaaeb5",[],{"_37":4720,"_23":302,"_303":4721,"_174":4723},"19aa5cd16abe",[4722],"7183d4c","We have linting about egress of data and things like that.",{"_37":4725,"_23":302,"_303":4726,"_174":2425},"34c96f97216c",[],{"_37":4728,"_23":302,"_303":4729,"_174":4731},"05ac7ee97f81",[4730],"71d883a","So we have a lot of scaffolding around our coding agents, and with our usage of AI code reviews, so that that ideally would never ever happen.",[4733,4735,4737],{"_37":4714,"_23":2441,"_2442":4734},"00:17:56.060",{"_37":4722,"_23":2441,"_2442":4736},"00:18:05.820",{"_37":4730,"_23":2441,"_2442":4738},"00:18:09.340",{"_37":4740,"_23":297,"_298":4741,"_306":4750,"_308":309},"89ed97aa3f3b",[4742,4746],{"_37":4743,"_23":302,"_303":4744,"_174":2983},"1418cd51e3a2",[4745,1281],"d02b11c",{"_37":4747,"_23":302,"_303":4748,"_174":4749},"99397b88d81c",[4745]," As the SRE in the room.",[4751],{"_37":4745,"_23":2441,"_2442":4752},"00:18:23.220",{"_37":4754,"_23":297,"_298":4755,"_306":4769,"_308":693},"8301340a9046",[4756,4761,4764],{"_37":4757,"_23":302,"_303":4758,"_174":4760},"05421a2b28e9",[4759],"2552c3e","A rule of thumb that I was using long before AI that seems to have renewed relevance is:",{"_37":4762,"_23":302,"_303":4763,"_174":2425},"37d8c9339db8",[],{"_37":4765,"_23":302,"_303":4766,"_174":4768},"f0a3db27b61a",[4767],"c2630da","the closer you get to laying bits down on disk, the more paranoid you should be.",[4770,4772],{"_37":4759,"_23":2441,"_2442":4771},"00:18:26.740",{"_37":4767,"_23":2441,"_2442":4773},"00:18:34.340",{"_37":4775,"_23":297,"_298":4776,"_306":4814,"_308":309},"d9fdff39b0b8",[4777,4782,4785,4790,4793,4798,4801,4806,4809],{"_37":4778,"_23":302,"_303":4779,"_174":4781},"a6b7bb762fb3",[4780],"358a0ec","There are so many areas where vibe coding is great, and kind of like, can I do it?",{"_37":4783,"_23":302,"_303":4784,"_174":2425},"0b9363df5622",[],{"_37":4786,"_23":302,"_303":4787,"_174":4789},"2fc866247f5e",[4788],"6c03065","Cool, YOLO,",{"_37":4791,"_23":302,"_303":4792,"_174":2425},"eccfafceb6da",[],{"_37":4794,"_23":302,"_303":4795,"_174":4797},"986d53d8c34c",[4796],"97ded27","is great, it's the right thing to do.",{"_37":4799,"_23":302,"_303":4800,"_174":2425},"7db8f574525f",[],{"_37":4802,"_23":302,"_303":4803,"_174":4805},"bcdde3f76414",[4804],"5f3e2bc","But the weird and amazing and fun thing about computers is the farther up the stack you go, the more",{"_37":4807,"_23":302,"_303":4808,"_174":2425},"a3d821a14ce7",[],{"_37":4810,"_23":302,"_303":4811,"_174":4813},"c13c139d6b12",[4812],"c278a7c","your rigor only comes from the metaphors, the language, like you decide what validation means, right?",[4815,4817,4819,4821,4823],{"_37":4780,"_23":2441,"_2442":4816},"00:18:40.419",{"_37":4788,"_23":2441,"_2442":4818},"00:18:45.700",{"_37":4796,"_23":2441,"_2442":4820},"00:18:47.340",{"_37":4804,"_23":2441,"_2442":4822},"00:18:49.419",{"_37":4812,"_23":2441,"_2442":4824},"00:18:59.500",{"_37":4826,"_23":297,"_298":4827,"_306":4857,"_308":309},"666b555a544c",[4828,4833,4836,4841,4844,4849,4852],{"_37":4829,"_23":302,"_303":4830,"_174":4832},"9c5deaa933f0",[4831],"3b4f288","But at the edges, whether that is where bits meet atoms, whether that's persistent storage or",{"_37":4834,"_23":302,"_303":4835,"_174":2425},"a7aaa17db736",[],{"_37":4837,"_23":302,"_303":4838,"_174":4840},"254a43de9a12",[4839],"9b1c1b0","people interacting, change is governed by different phenomena.",{"_37":4842,"_23":302,"_303":4843,"_174":2425},"803c7149d451",[],{"_37":4845,"_23":302,"_303":4846,"_174":4848},"c95b079d45c6",[4847],"920d509","And so there are some places where you just, nobody wants to wake up in the morning and have",{"_37":4850,"_23":302,"_303":4851,"_174":2425},"033d75038bb4",[],{"_37":4853,"_23":302,"_303":4854,"_174":4856},"7158e53feb0c",[4855],"a89fb10","all of the pixels on their Slack apps subtly change every day.",[4858,4860,4862,4864],{"_37":4831,"_23":2441,"_2442":4859},"00:19:09.580",{"_37":4839,"_23":2441,"_2442":4861},"00:19:18.360",{"_37":4847,"_23":2441,"_2442":4863},"00:19:24.520",{"_37":4855,"_23":2441,"_2442":4865},"00:19:30.780",{"_37":4867,"_23":297,"_298":4868,"_306":4906,"_308":309},"010f17c42694",[4869,4874,4877,4882,4885,4890,4893,4898,4901],{"_37":4870,"_23":302,"_303":4871,"_174":4873},"9e592b23ed16",[4872],"d87c376","No one wants that.",{"_37":4875,"_23":302,"_303":4876,"_174":2425},"ba6e12eeb131",[],{"_37":4878,"_23":302,"_303":4879,"_174":4881},"d154725c536e",[4880],"b04566a","No one wants",{"_37":4883,"_23":302,"_303":4884,"_174":2425},"4d317e7d6440",[],{"_37":4886,"_23":302,"_303":4887,"_174":4889},"3364c61c6067",[4888],"b4734b7","persistence problems to just be, because they're famously impossible, difficult to roll back from, right?",{"_37":4891,"_23":302,"_303":4892,"_174":2425},"2f0a678d2567",[],{"_37":4894,"_23":302,"_303":4895,"_174":4897},"3352609c9a4a",[4896],"30fe8b8","So can you do it is different than should you do it.",{"_37":4899,"_23":302,"_303":4900,"_174":2425},"1ea73211cb95",[],{"_37":4902,"_23":302,"_303":4903,"_174":4905},"49a5db716b55",[4904],"76b91ff","And I think that boundary between bits and atoms is where the really hard problems come.",[4907,4909,4911,4913,4915],{"_37":4872,"_23":2441,"_2442":4908},"00:19:35.100",{"_37":4880,"_23":2441,"_2442":4910},"00:19:37.980",{"_37":4888,"_23":2441,"_2442":4912},"00:19:39.679",{"_37":4896,"_23":2441,"_2442":4914},"00:19:46.880",{"_37":4904,"_23":2441,"_2442":4916},"00:19:50.240",{"_37":4918,"_23":297,"_298":4919,"_306":4960,"_308":309},"146d05b04344",[4920,4924,4928,4931,4936,4939,4944,4947,4952,4955],{"_37":4921,"_23":302,"_303":4922,"_174":3743},"fa56c11d894a",[4923,1281],"2457576",{"_37":4925,"_23":302,"_303":4926,"_174":4927},"9d182dbb1096",[4923]," Chetan, your question was",{"_37":4929,"_23":302,"_303":4930,"_174":2425},"5a3992ba0a9b",[],{"_37":4932,"_23":302,"_303":4933,"_174":4935},"8e439d3cde85",[4934],"fdfd5dd","premised on this situation happened, how would you respond?",{"_37":4937,"_23":302,"_303":4938,"_174":2425},"826e7c053af0",[],{"_37":4940,"_23":302,"_303":4941,"_174":4943},"c26db0601cc0",[4942],"c3cc187","And I think that's the wrong question.",{"_37":4945,"_23":302,"_303":4946,"_174":2425},"af8d08888827",[],{"_37":4948,"_23":302,"_303":4949,"_174":4951},"33c5b6ad87cc",[4950],"c10dc8f","The better question is how can you prevent this in the first place from happening?",{"_37":4953,"_23":302,"_303":4954,"_174":2425},"874f65ab8420",[],{"_37":4956,"_23":302,"_303":4957,"_174":4959},"66dbce0ff08c",[4958],"6c17736","So I'll answer that question.",[4961,4963,4965,4967,4969],{"_37":4923,"_23":2441,"_2442":4962},"00:19:56.720",{"_37":4934,"_23":2441,"_2442":4964},"00:19:59.260",{"_37":4942,"_23":2441,"_2442":4966},"00:20:02.940",{"_37":4950,"_23":2441,"_2442":4968},"00:20:05.260",{"_37":4958,"_23":2441,"_2442":4970},"00:20:09.400",{"_37":4972,"_23":297,"_298":4973,"_306":5003,"_308":309},"baca8dd4120b",[4974,4979,4982,4987,4990,4995,4998],{"_37":4975,"_23":302,"_303":4976,"_174":4978},"dfd0c71b2a4c",[4977],"46f8a97","LaunchDarkly was built as an enterprise-grade platform for humans.",{"_37":4980,"_23":302,"_303":4981,"_174":2425},"e005604e6e30",[],{"_37":4983,"_23":302,"_303":4984,"_174":4986},"cfe0522f07ce",[4985],"1eab6c1","Those same enterprise-grade principles that work for humans thankfully also work for agents.",{"_37":4988,"_23":302,"_303":4989,"_174":2425},"9bc36236565c",[],{"_37":4991,"_23":302,"_303":4992,"_174":4994},"3ae76a1ea64f",[4993],"39c4484","Except now they're even more important for agents, because as you said, if agents get blocked,",{"_37":4996,"_23":302,"_303":4997,"_174":2425},"c2c14d470322",[],{"_37":4999,"_23":302,"_303":5000,"_174":5002},"fcf13d615855",[5001],"5e607a9","they will keep investigating and keep trying to find a way to get through.",[5004,5006,5008,5010],{"_37":4977,"_23":2441,"_2442":5005},"00:20:12.760",{"_37":4985,"_23":2441,"_2442":5007},"00:20:18.919",{"_37":4993,"_23":2441,"_2442":5009},"00:20:23.960",{"_37":5001,"_23":2441,"_2442":5011},"00:20:29.120",{"_37":5013,"_23":297,"_298":5014,"_306":5052,"_308":309},"8a2800b23d14",[5015,5020,5023,5028,5031,5036,5039,5044,5047],{"_37":5016,"_23":302,"_303":5017,"_174":5019},"49ab0acb1a60",[5018],"929a24c","So in particular, LaunchDarkly has always had fine-grained RBAC.",{"_37":5021,"_23":302,"_303":5022,"_174":2425},"63e11eb43ab9",[],{"_37":5024,"_23":302,"_303":5025,"_174":5027},"8f3e81f63fad",[5026],"0b74e5e","So you could",{"_37":5029,"_23":302,"_303":5030,"_174":2425},"784d3cda4fd1",[],{"_37":5032,"_23":302,"_303":5033,"_174":5035},"225501759a2c",[5034],"cc5b5e8","perhaps have permission to update a feature flag description, but not update the targeting.",{"_37":5037,"_23":302,"_303":5038,"_174":2425},"94f044a5ce67",[],{"_37":5040,"_23":302,"_303":5041,"_174":5043},"90946930948a",[5042],"30ef641","Or maybe you could have access to update the targeting on staging",{"_37":5045,"_23":302,"_303":5046,"_174":2425},"6efe478d7eaa",[],{"_37":5048,"_23":302,"_303":5049,"_174":5051},"3e2694ef319f",[5050],"08725d2","but not on production.",[5053,5055,5057,5059,5061],{"_37":5018,"_23":2441,"_2442":5054},"00:20:33.040",{"_37":5026,"_23":2441,"_2442":5056},"00:20:39.040",{"_37":5034,"_23":2441,"_2442":5058},"00:20:40.440",{"_37":5042,"_23":2441,"_2442":5060},"00:20:45.159",{"_37":5050,"_23":2441,"_2442":5062},"00:20:48.140",{"_37":5064,"_23":297,"_298":5065,"_306":5087,"_308":309},"fd7607f22c40",[5066,5071,5074,5079,5082],{"_37":5067,"_23":302,"_303":5068,"_174":5070},"3b9d8bd97171",[5069],"bd915fb","And so depending on your role, depending on the purpose of your API key, depending on whatever it is you're trying to do for your business case,",{"_37":5072,"_23":302,"_303":5073,"_174":2425},"658be090000c",[],{"_37":5075,"_23":302,"_303":5076,"_174":5078},"3fef3a890091",[5077],"0a46a18","I think you just need to be very, very cognizant and intentional behind how you scope your work.",{"_37":5080,"_23":302,"_303":5081,"_174":2425},"87bf1ed36fa7",[],{"_37":5083,"_23":302,"_303":5084,"_174":5086},"70feef5adbe3",[5085],"733f070","And with LaunchDarkly, we have those fine-grained permissions to allow that to happen.",[5088,5090,5092],{"_37":5069,"_23":2441,"_2442":5089},"00:20:49.260",{"_37":5077,"_23":2441,"_2442":5091},"00:20:58.820",{"_37":5085,"_23":2441,"_2442":5093},"00:21:05.060",{"_37":5095,"_23":297,"_298":5096,"_306":5118,"_308":309},"e54b64fed778",[5097,5102,5105,5110,5113],{"_37":5098,"_23":302,"_303":5099,"_174":5101},"6ee810069980",[5100],"be2d0dc","And then the second piece is auditability and being able to trace activity in your system.",{"_37":5103,"_23":302,"_303":5104,"_174":2425},"302cb5fe0315",[],{"_37":5106,"_23":302,"_303":5107,"_174":5109},"334d4de218d1",[5108],"7a49857","We have always had audit logging capabilities for any change in the system, and that change could be major, such as, hey, this entire flag",{"_37":5111,"_23":302,"_303":5112,"_174":2425},"dd4380332b04",[],{"_37":5114,"_23":302,"_303":5115,"_174":5117},"3fc3f323eea8",[5116],"9bf35a6","got turned on on production.",[5119,5121,5123],{"_37":5100,"_23":2441,"_2442":5120},"00:21:08.600",{"_37":5108,"_23":2441,"_2442":5122},"00:21:16.140",{"_37":5116,"_23":2441,"_2442":5124},"00:21:24.440",{"_37":5126,"_23":297,"_298":5127,"_306":5149,"_308":309},"edf8d8c14f54",[5128,5133,5136,5141,5144],{"_37":5129,"_23":302,"_303":5130,"_174":5132},"5e52b84326c3",[5131],"c6243c3","Or it could be just as simple as, you know, a new member was added to our account because a new engineer was onboarded yesterday.",{"_37":5134,"_23":302,"_303":5135,"_174":2425},"9bc58f294795",[],{"_37":5137,"_23":302,"_303":5138,"_174":5140},"7ff4f7e71ee6",[5139],"c904669","And through those two, there are other parts of enterprisability that are very important for agents, but I think those are the two most important",{"_37":5142,"_23":302,"_303":5143,"_174":2425},"c1a6dbabbd28",[],{"_37":5145,"_23":302,"_303":5146,"_174":5148},"36110a8b0680",[5147],"afccf50","I wanted to focus on.",[5150,5152,5154],{"_37":5131,"_23":2441,"_2442":5151},"00:21:25.960",{"_37":5139,"_23":2441,"_2442":5153},"00:21:33.780",{"_37":5147,"_23":2441,"_2442":5155},"00:21:41.679",{"_37":5157,"_23":297,"_298":5158,"_306":5175,"_308":693},"24a5a29fecec",[5159,5163,5167,5170],{"_37":5160,"_23":302,"_303":5161,"_174":2983},"3026a1afa1d0",[5162,1281],"165d30a",{"_37":5164,"_23":302,"_303":5165,"_174":5166},"b008db3b1a3b",[5162]," Yeah, the auditability and replayability is huge.",{"_37":5168,"_23":302,"_303":5169,"_174":2425},"161919b58044",[],{"_37":5171,"_23":302,"_303":5172,"_174":5174},"a48d45ee00e2",[5173],"f54efaf","The less control you have, the more you need the ability to actually understand what did happen.",[5176,5178],{"_37":5162,"_23":2441,"_2442":5177},"00:21:42.559",{"_37":5173,"_23":2441,"_2442":5179},"00:21:45.440",{"_37":5181,"_23":297,"_298":5182,"_306":5187,"_308":382},"6cf5b2f16bf9",[5183],{"_37":5184,"_23":302,"_303":5185,"_174":5186},"4e79f93e1fb2",[],"Define the Constraint or Control the Experience",[],{"_37":5189,"_23":297,"_298":5190,"_306":5223,"_308":309},"52f3b66727f9",[5191,5195,5199,5202,5207,5210,5215,5218],{"_37":5192,"_23":302,"_303":5193,"_174":3026},"97870545f779",[5194,1281],"6b3b61c",{"_37":5196,"_23":302,"_303":5197,"_174":5198},"0c2987d2a664",[5194]," And my lead-on question was pertaining to what I call as the constraint.",{"_37":5200,"_23":302,"_303":5201,"_174":2425},"5271c00cec23",[],{"_37":5203,"_23":302,"_303":5204,"_174":5206},"23d7ff17ee81",[5205],"49f0f80","Often",{"_37":5208,"_23":302,"_303":5209,"_174":2425},"1cf09c0fdcdb",[],{"_37":5211,"_23":302,"_303":5212,"_174":5214},"689159e2d494",[5213],"500545b","security is defined by constraint.",{"_37":5216,"_23":302,"_303":5217,"_174":2425},"a143d56e5f77",[],{"_37":5219,"_23":302,"_303":5220,"_174":5222},"f7590608fb6b",[5221],"2253aef","You say this is the action, the action has certain constraints, and you're permitted to perform this action, right?",[5224,5226,5228,5230],{"_37":5194,"_23":2441,"_2442":5225},"00:21:51.900",{"_37":5205,"_23":2441,"_2442":5227},"00:21:56.860",{"_37":5213,"_23":2441,"_2442":5229},"00:21:58.220",{"_37":5221,"_23":2441,"_2442":5231},"00:22:00.860",{"_37":5233,"_23":297,"_298":5234,"_306":5256,"_308":309},"c9263b8bfa79",[5235,5240,5243,5248,5251],{"_37":5236,"_23":302,"_303":5237,"_174":5239},"45d8d91d949b",[5238],"0f5f830","From a human standpoint, for example, flipping the flag on LaunchDarkly gives me some feedback if I attempt to flip the flag, saying I don't have permissions to do that.",{"_37":5241,"_23":302,"_303":5242,"_174":2425},"0f27e96cf5af",[],{"_37":5244,"_23":302,"_303":5245,"_174":5247},"e576d20ca5da",[5246],"c9a8743","And likewise when I attempt to delete the document, I'm presented with feedback.",{"_37":5249,"_23":302,"_303":5250,"_174":2425},"185489aa5f83",[],{"_37":5252,"_23":302,"_303":5253,"_174":5255},"9799f152d30e",[5254],"67d3f92","We as humans know how to react to feedback, which means we contact our admins, ask for privilege escalation.",[5257,5259,5261],{"_37":5238,"_23":2441,"_2442":5258},"00:22:08.720",{"_37":5246,"_23":2441,"_2442":5260},"00:22:20.160",{"_37":5254,"_23":2441,"_2442":5262},"00:22:25.600",{"_37":5264,"_23":297,"_298":5265,"_306":5279,"_308":309},"7008e57ef8bc",[5266,5271,5274],{"_37":5267,"_23":302,"_303":5268,"_174":5270},"2355be2d694c",[5269],"aabb4b1","But in this new world, you have this Ralph Wiggum loop, which our dear friend Geoffrey had coined the term on.",{"_37":5272,"_23":302,"_303":5273,"_174":2425},"c19a295ae75e",[],{"_37":5275,"_23":302,"_303":5276,"_174":5278},"57c7e5ffba87",[5277],"c88d8b8","It's the agent with goal-directedness continuously operating in this loop, finding ways to solve your problem in a helpful way, and then leads to catastrophe.",[5280,5282],{"_37":5269,"_23":2441,"_2442":5281},"00:22:31.540",{"_37":5277,"_23":2441,"_2442":5283},"00:22:40.080",{"_37":5285,"_23":297,"_298":5286,"_306":5324,"_308":309},"39937cb47c79",[5287,5292,5295,5300,5303,5308,5311,5316,5319],{"_37":5288,"_23":302,"_303":5289,"_174":5291},"8fe962f1fe3f",[5290],"96b85de","So",{"_37":5293,"_23":302,"_303":5294,"_174":2425},"250dcb2c1d36",[],{"_37":5296,"_23":302,"_303":5297,"_174":5299},"57c611c898a1",[5298],"7b0d46c","do you define constraints?",{"_37":5301,"_23":302,"_303":5302,"_174":2425},"75c67648fe22",[],{"_37":5304,"_23":302,"_303":5305,"_174":5307},"4ffc80d1702b",[5306],"e273b3a","Meaning do you create a sandbox?",{"_37":5309,"_23":302,"_303":5310,"_174":2425},"53e16a86c8b4",[],{"_37":5312,"_23":302,"_303":5313,"_174":5315},"53dc1669f4f8",[5314],"db7ad70","Do you apply RBAC?",{"_37":5317,"_23":302,"_303":5318,"_174":2425},"71860a29fc4b",[],{"_37":5320,"_23":302,"_303":5321,"_174":5323},"3798169aa8d5",[5322],"5459b7b","You go on the deep end in that sense?",[5325,5327,5329,5331,5333],{"_37":5290,"_23":2441,"_2442":5326},"00:22:51.120",{"_37":5298,"_23":2441,"_2442":5328},"00:22:52.419",{"_37":5306,"_23":2441,"_2442":5330},"00:22:54.500",{"_37":5314,"_23":2441,"_2442":5332},"00:22:56.500",{"_37":5322,"_23":2441,"_2442":5334},"00:22:58.100",{"_37":5336,"_23":297,"_298":5337,"_306":5367,"_308":309},"f21be989eb56",[5338,5343,5346,5351,5354,5359,5362],{"_37":5339,"_23":302,"_303":5340,"_174":5342},"bafd536e742c",[5341],"01cad1a","Or do you control the experience of the constraint?",{"_37":5344,"_23":302,"_303":5345,"_174":2425},"60e0b621ffaa",[],{"_37":5347,"_23":302,"_303":5348,"_174":5350},"3013f3a374b0",[5349],"4a1341c","Because if you provide a certain feedback or elicit error messages in a certain way that controls the agent from escalating, you might prevent this.",{"_37":5352,"_23":302,"_303":5353,"_174":2425},"fd56d684371f",[],{"_37":5355,"_23":302,"_303":5356,"_174":5358},"73b8a23e5450",[5357],"9fe2baf","So how are you thinking through this as you build experience in your respective products?",{"_37":5360,"_23":302,"_303":5361,"_174":2425},"95115062e447",[],{"_37":5363,"_23":302,"_303":5364,"_174":5366},"b1add2fc69d3",[5365],"4e12e8b","Define the constraint or control the experience of the constraint?",[5368,5370,5372,5374],{"_37":5341,"_23":2441,"_2442":5369},"00:23:00.900",{"_37":5349,"_23":2441,"_2442":5371},"00:23:05.280",{"_37":5357,"_23":2441,"_2442":5373},"00:23:14.560",{"_37":5365,"_23":2441,"_2442":5375},"00:23:19.980",{"_37":5377,"_23":297,"_298":5378,"_306":5403,"_308":309},"50bcbd5a346d",[5379,5383,5387,5390,5395,5398],{"_37":5380,"_23":302,"_303":5381,"_174":3743},"6d47657b590b",[5382,1281],"02563bc",{"_37":5384,"_23":302,"_303":5385,"_174":5386},"9f59e571e60e",[5382]," Yeah, I'll take this one first, being that I work for the company that helps release functionality.",{"_37":5388,"_23":302,"_303":5389,"_174":2425},"0fc87e8db71c",[],{"_37":5391,"_23":302,"_303":5392,"_174":5394},"44951f0066a0",[5393],"28efa6c","So at LaunchDarkly we dogfood our own product, drink our own champagne, whatever metaphor is the one you prefer.",{"_37":5396,"_23":302,"_303":5397,"_174":2425},"a30ebd79559b",[],{"_37":5399,"_23":302,"_303":5400,"_174":5402},"fa5111f0ff01",[5401],"4ff2510","And so the things we use are feature flagged.",[5404,5406,5408],{"_37":5382,"_23":2441,"_2442":5405},"00:23:23.580",{"_37":5393,"_23":2441,"_2442":5407},"00:23:30.200",{"_37":5401,"_23":2441,"_2442":5409},"00:23:37.720",{"_37":5411,"_23":297,"_298":5412,"_306":5434,"_308":309},"6d0e1b8a344a",[5413,5418,5421,5426,5429],{"_37":5414,"_23":302,"_303":5415,"_174":5417},"e695ea961894",[5416],"3a4c048","We of course",{"_37":5419,"_23":302,"_303":5420,"_174":2425},"4e4f9d38340b",[],{"_37":5422,"_23":302,"_303":5423,"_174":5425},"31f1ead22f6c",[5424],"d1b15ad","have a lot of pre-production gates as well to make sure that functionality is appropriate, but I'm not going to focus on that.",{"_37":5427,"_23":302,"_303":5428,"_174":2425},"3cc129c47660",[],{"_37":5430,"_23":302,"_303":5431,"_174":5433},"efe4da7b8b4e",[5432],"35babb7","I don't think those are as interesting.",[5435,5437,5439],{"_37":5416,"_23":2441,"_2442":5436},"00:23:41.960",{"_37":5424,"_23":2441,"_2442":5438},"00:23:43.800",{"_37":5432,"_23":2441,"_2442":5440},"00:23:49.320",{"_37":5442,"_23":297,"_298":5443,"_306":5449,"_308":309},"9a7f7575e99a",[5444],{"_37":5445,"_23":302,"_303":5446,"_174":5448},"8ff1a46c8713",[5447],"4a41107","Honeycomb and LaunchDarkly have jointly long had this perspective of testing in production, and that has never been more true than today in our agentic world.",[5450],{"_37":5447,"_23":2441,"_2442":5451},"00:23:52.500",{"_37":5453,"_23":297,"_298":5454,"_306":5492,"_308":309},"4874d08c99cd",[5455,5460,5463,5468,5471,5476,5479,5484,5487],{"_37":5456,"_23":302,"_303":5457,"_174":5459},"3356a45d2187",[5458],"b1ad4fe","At some point you have to release functionality and you have to retain control so that if something goes south, you can",{"_37":5461,"_23":302,"_303":5462,"_174":2425},"7774bea73163",[],{"_37":5464,"_23":302,"_303":5465,"_174":5467},"7d24f9c0b844",[5466],"32bbf81","siphon it off and kill it.",{"_37":5469,"_23":302,"_303":5470,"_174":2425},"0fc3c65bf900",[],{"_37":5472,"_23":302,"_303":5473,"_174":5475},"723b944ff423",[5474],"be10e4d","But you also have to observe so that you know when to kill it and you can make informed decisions.",{"_37":5477,"_23":302,"_303":5478,"_174":2425},"60076e143b1a",[],{"_37":5480,"_23":302,"_303":5481,"_174":5483},"3c3b67030f04",[5482],"c8d5749","Tools like Honeycomb, tools like LaunchDarkly, are",{"_37":5485,"_23":302,"_303":5486,"_174":2425},"af0beb6b4ae8",[],{"_37":5488,"_23":302,"_303":5489,"_174":5491},"2e1fcfca2307",[5490],"8e95486","great at doing that.",[5493,5495,5497,5499,5501],{"_37":5458,"_23":2441,"_2442":5494},"00:24:06.019",{"_37":5466,"_23":2441,"_2442":5496},"00:24:14.419",{"_37":5474,"_23":2441,"_2442":5498},"00:24:16.340",{"_37":5482,"_23":2441,"_2442":5500},"00:24:22.179",{"_37":5490,"_23":2441,"_2442":5502},"00:24:27.300",{"_37":5504,"_23":297,"_298":5505,"_306":5535,"_308":309},"bf22504d9c81",[5506,5511,5514,5519,5522,5527,5530],{"_37":5507,"_23":302,"_303":5508,"_174":5510},"ad3723b8a842",[5509],"76c593e","And so I mentioned guarded releases before.",{"_37":5512,"_23":302,"_303":5513,"_174":2425},"6cd2efefa582",[],{"_37":5515,"_23":302,"_303":5516,"_174":5518},"f6981680e4fb",[5517],"034537d","And I think that's the way that we release functionality in a safe manner, where agents may have access to something and then perhaps we realize actually granted",{"_37":5520,"_23":302,"_303":5521,"_174":2425},"8a976c7b6d33",[],{"_37":5523,"_23":302,"_303":5524,"_174":5526},"0991d7c34fc6",[5525],"4d3d9c4","too much access or inappropriate behavior for whatever reason.",{"_37":5528,"_23":302,"_303":5529,"_174":2425},"ce8eb9c689b4",[],{"_37":5531,"_23":302,"_303":5532,"_174":5534},"84dd1656f8e6",[5533],"fd1286c","And then we can always turn it back, iterate, and then re-release it.",[5536,5538,5540,5542],{"_37":5509,"_23":2441,"_2442":5537},"00:24:28.180",{"_37":5517,"_23":2441,"_2442":5539},"00:24:30.260",{"_37":5525,"_23":2441,"_2442":5541},"00:24:39.760",{"_37":5533,"_23":2441,"_2442":5543},"00:24:43.360",{"_37":5545,"_23":297,"_298":5546,"_306":5579,"_308":309},"e8378beee739",[5547,5551,5555,5558,5563,5566,5571,5574],{"_37":5548,"_23":302,"_303":5549,"_174":2983},"6fa13a93005b",[5550,1281],"173af10",{"_37":5552,"_23":302,"_303":5553,"_174":5554},"25da1f60cad5",[5550]," Yeah, I don't think it's either or.",{"_37":5556,"_23":302,"_303":5557,"_174":2425},"dfab9aa79228",[],{"_37":5559,"_23":302,"_303":5560,"_174":5562},"0caa41fed197",[5561],"f5b30a3","In fact I believe I have",{"_37":5564,"_23":302,"_303":5565,"_174":2425},"29a66f21d9f1",[],{"_37":5567,"_23":302,"_303":5568,"_174":5570},"3f59bff486c1",[5569],"aef3c32","like a superhero here.",{"_37":5572,"_23":302,"_303":5573,"_174":2425},"9865c2b27509",[],{"_37":5575,"_23":302,"_303":5576,"_174":5578},"a8aeabae9d08",[5577],"e3e4276","Test in prod or live a lie is what my shirt says.",[5580,5582,5584,5586],{"_37":5550,"_23":2441,"_2442":5581},"00:24:48.240",{"_37":5561,"_23":2441,"_2442":5583},"00:24:50.400",{"_37":5569,"_23":2441,"_2442":5585},"00:24:54.060",{"_37":5577,"_23":2441,"_2442":5587},"00:24:56.460",{"_37":5589,"_23":297,"_298":5590,"_306":5620,"_308":309},"7c17c5ff1bd4",[5591,5596,5599,5604,5607,5612,5615],{"_37":5592,"_23":302,"_303":5593,"_174":5595},"c8746cb47d6e",[5594],"a13e61d","I think I just finished writing the second edition of",{"_37":5597,"_23":302,"_303":5598,"_174":2425},"92114849f827",[],{"_37":5600,"_23":302,"_303":5601,"_174":5603},"9989820d74e8",[5602],"be98098","Observability Engineering, and I think this third chapter is on code crossing over.",{"_37":5605,"_23":302,"_303":5606,"_174":2425},"af422a0155ce",[],{"_37":5608,"_23":302,"_303":5609,"_174":5611},"a4176411f427",[5610],"f219de7","And a metaphor that O'Reilly would not let me put in the book, but I will serve it to you here on a platter, is:",{"_37":5613,"_23":302,"_303":5614,"_174":2425},"55f65f47de9c",[],{"_37":5616,"_23":302,"_303":5617,"_174":5619},"7682c8c7cd18",[5618],"3f3c9a7","the moment when code crosses over into production and the interval of time immediately after that, it's like the fourth trimester of human development.",[5621,5623,5625,5627],{"_37":5594,"_23":2441,"_2442":5622},"00:25:00.780",{"_37":5602,"_23":2441,"_2442":5624},"00:25:04.460",{"_37":5610,"_23":2441,"_2442":5626},"00:25:13.120",{"_37":5618,"_23":2441,"_2442":5628},"00:25:21.400",{"_37":5630,"_23":297,"_298":5631,"_306":5645,"_308":309},"54a414789bd0",[5632,5637,5640],{"_37":5633,"_23":302,"_303":5634,"_174":5636},"b33f1cf4115e",[5635],"92fba13","You know how they say our heads are too big, so babies are born an extra quarter early, they're helpless, we're not like giraffes, we can't just get up and toddle off, right?",{"_37":5638,"_23":302,"_303":5639,"_174":2425},"a84e109e87ba",[],{"_37":5641,"_23":302,"_303":5642,"_174":5644},"81f728da0e71",[5643],"c4a1951","Well, that is the most important state, you learn the most the fastest right after your code meets reality.",[5646,5648],{"_37":5635,"_23":2441,"_2442":5647},"00:25:30.100",{"_37":5643,"_23":2441,"_2442":5649},"00:25:39.620",{"_37":5651,"_23":297,"_298":5652,"_306":5698,"_308":309},"f4b2058dcbd1",[5653,5658,5661,5666,5669,5674,5677,5682,5685,5690,5693],{"_37":5654,"_23":302,"_303":5655,"_174":5657},"33a30cced4cf",[5656],"d3f2e1c","And I think we need to retrain developers to",{"_37":5659,"_23":302,"_303":5660,"_174":2425},"9010e2fb0c39",[],{"_37":5662,"_23":302,"_303":5663,"_174":5665},"8a3d9173878a",[5664],"7e33d16","focus on that.",{"_37":5667,"_23":302,"_303":5668,"_174":2425},"32e16c44b1f0",[],{"_37":5670,"_23":302,"_303":5671,"_174":5673},"99f546d1a56a",[5672],"5dc39bd","That is not development and then development ends and production begins.",{"_37":5675,"_23":302,"_303":5676,"_174":2425},"05faf8cf0da2",[],{"_37":5678,"_23":302,"_303":5679,"_174":5681},"6f41722f8660",[5680],"aa3494f","No, no, no, no, no.",{"_37":5683,"_23":302,"_303":5684,"_174":2425},"4359bdfa846e",[],{"_37":5686,"_23":302,"_303":5687,"_174":5689},"ae5f977b563d",[5688],"4f50486","Your code crossing over is your code being born.",{"_37":5691,"_23":302,"_303":5692,"_174":2425},"063b814c1ea0",[],{"_37":5694,"_23":302,"_303":5695,"_174":5697},"659fa7527190",[5696],"6914e9a","It needs you to pay closer attention in detail precisely as the most important part, right?",[5699,5701,5703,5705,5707,5709],{"_37":5656,"_23":2441,"_2442":5700},"00:25:48.660",{"_37":5664,"_23":2441,"_2442":5702},"00:25:52.700",{"_37":5672,"_23":2441,"_2442":5704},"00:25:53.900",{"_37":5680,"_23":2441,"_2442":5706},"00:25:57.580",{"_37":5688,"_23":2441,"_2442":5708},"00:25:58.780",{"_37":5696,"_23":2441,"_2442":5710},"00:26:02.440",{"_37":5712,"_23":297,"_298":5713,"_306":5743,"_308":309},"9ecbaee53003",[5714,5719,5722,5727,5730,5735,5738],{"_37":5715,"_23":302,"_303":5716,"_174":5718},"d0699b41d6f3",[5717],"fd24710","It's not what works in theory that we care about.",{"_37":5720,"_23":302,"_303":5721,"_174":2425},"fd88d542ae78",[],{"_37":5723,"_23":302,"_303":5724,"_174":5726},"ebeeedf3794c",[5725],"6b1ab54","It's what really works.",{"_37":5728,"_23":302,"_303":5729,"_174":2425},"f6b715b0b85d",[],{"_37":5731,"_23":302,"_303":5732,"_174":5734},"6517a07f4a26",[5733],"325b1dc","And if you don't, there's a flywheel of tools and practices that build on this, and ironically, none of them are agent-specific.",{"_37":5736,"_23":302,"_303":5737,"_174":2425},"d902abcbf403",[],{"_37":5739,"_23":302,"_303":5740,"_174":5742},"08c6cf5e4029",[5741],"423fd43","They're just more important now than they've ever been.",[5744,5746,5748,5750],{"_37":5717,"_23":2441,"_2442":5745},"00:26:10.519",{"_37":5725,"_23":2441,"_2442":5747},"00:26:12.919",{"_37":5733,"_23":2441,"_2442":5749},"00:26:14.740",{"_37":5741,"_23":2441,"_2442":5751},"00:26:22.100",{"_37":5753,"_23":297,"_298":5754,"_306":5800,"_308":309},"59e3c4d74f38",[5755,5760,5763,5768,5771,5776,5779,5784,5787,5792,5795],{"_37":5756,"_23":302,"_303":5757,"_174":5759},"12a158f8bce0",[5758],"a7c4ea3","You need high-cardinality data, you need rich context, you need feature flags, you need progressive deployments.",{"_37":5761,"_23":302,"_303":5762,"_174":2425},"863d62c0d43d",[],{"_37":5764,"_23":302,"_303":5765,"_174":5767},"bf02ff22dba7",[5766],"e7d3c4c","You need all of these things.",{"_37":5769,"_23":302,"_303":5770,"_174":2425},"a7530b4fa9c3",[],{"_37":5772,"_23":302,"_303":5773,"_174":5775},"1683a3bf565d",[5774],"a6f5b18","They're greater than the sum of their parts",{"_37":5777,"_23":302,"_303":5778,"_174":2425},"a596cf1dbe07",[],{"_37":5780,"_23":302,"_303":5781,"_174":5783},"ab12c3956604",[5782],"f5e2369","if you can see precisely, and you can combine them, and then everybody's like, oh, test in prod, that's so scary.",{"_37":5785,"_23":302,"_303":5786,"_174":2425},"9219e832f8a8",[],{"_37":5788,"_23":302,"_303":5789,"_174":5791},"f7657751578e",[5790],"a855ca9","It doesn't have to be.",{"_37":5793,"_23":302,"_303":5794,"_174":2425},"f151e5d022c9",[],{"_37":5796,"_23":302,"_303":5797,"_174":5799},"59b189c2fca4",[5798],"35cc0fe","It's scary if you don't have good tools.",[5801,5803,5805,5807,5809,5811],{"_37":5758,"_23":2441,"_2442":5802},"00:26:24.360",{"_37":5766,"_23":2441,"_2442":5804},"00:26:31.000",{"_37":5774,"_23":2441,"_2442":5806},"00:26:32.840",{"_37":5782,"_23":2441,"_2442":5808},"00:26:35.040",{"_37":5790,"_23":2441,"_2442":5810},"00:26:41.600",{"_37":5798,"_23":2441,"_2442":5812},"00:26:43.419",{"_37":5814,"_23":297,"_298":5815,"_306":5845,"_308":309},"d073da7ae518",[5816,5821,5824,5829,5832,5837,5840],{"_37":5817,"_23":302,"_303":5818,"_174":5820},"42e02f755817",[5819],"03dee63","You're just like, YOLO, ship it.",{"_37":5822,"_23":302,"_303":5823,"_174":2425},"881b842757af",[],{"_37":5825,"_23":302,"_303":5826,"_174":5828},"1000d6fab781",[5827],"c4c3372","Well, 99.5% latency looks fine.",{"_37":5830,"_23":302,"_303":5831,"_174":2425},"373874715379",[],{"_37":5833,"_23":302,"_303":5834,"_174":5836},"e5afe2ff71b2",[5835],"92feb63","Yeah, that's terrifying.",{"_37":5838,"_23":302,"_303":5839,"_174":2425},"dbe75a2488a4",[],{"_37":5841,"_23":302,"_303":5842,"_174":5844},"200a77079416",[5843],"e722bd0","Don't do that.",[5846,5848,5850,5852],{"_37":5819,"_23":2441,"_2442":5847},"00:26:45.660",{"_37":5827,"_23":2441,"_2442":5849},"00:26:48.220",{"_37":5835,"_23":2441,"_2442":5851},"00:26:51.660",{"_37":5843,"_23":2441,"_2442":5853},"00:26:53.020",{"_37":5855,"_23":297,"_298":5856,"_306":5886,"_308":309},"a949ad45b7e0",[5857,5862,5865,5870,5873,5878,5881],{"_37":5858,"_23":302,"_303":5859,"_174":5861},"4267f581fa48",[5860],"2ddac3e","But if you have 10 million requests per second with the right tools, you can tease out five requests a second",{"_37":5863,"_23":302,"_303":5864,"_174":2425},"f7aefd9edde3",[],{"_37":5866,"_23":302,"_303":5867,"_174":5869},"50a7895b1ec4",[5868],"0566e94","that are alarmingly high latency, right?",{"_37":5871,"_23":302,"_303":5872,"_174":2425},"2a2448675f11",[],{"_37":5874,"_23":302,"_303":5875,"_174":5877},"6e5d3a8389bc",[5876],"8c9ebba","You can trace it, you can break down by, sorry, I could just talk about this all day.",{"_37":5879,"_23":302,"_303":5880,"_174":2425},"00f003e42952",[],{"_37":5882,"_23":302,"_303":5883,"_174":5885},"045417c9ae99",[5884],"f293c08","It's my favorite topic.",[5887,5889,5891,5893],{"_37":5860,"_23":2441,"_2442":5888},"00:26:54.960",{"_37":5868,"_23":2441,"_2442":5890},"00:27:04.760",{"_37":5876,"_23":2441,"_2442":5892},"00:27:07.799",{"_37":5884,"_23":2441,"_2442":5894},"00:27:11.720",{"_37":5896,"_23":297,"_298":5897,"_306":5911,"_308":693},"3193122f47d5",[5898,5903,5906],{"_37":5899,"_23":302,"_303":5900,"_174":5902},"3d30109fb496",[5901],"39b807f","Code crossing over is code becoming real.",{"_37":5904,"_23":302,"_303":5905,"_174":2425},"613df05adba8",[],{"_37":5907,"_23":302,"_303":5908,"_174":5910},"da5acb083cdc",[5909],"b70eb80","There is nothing more important than that.",[5912,5914],{"_37":5901,"_23":2441,"_2442":5913},"00:27:13.640",{"_37":5909,"_23":2441,"_2442":5915},"00:27:17.480",{"_37":5917,"_23":297,"_298":5918,"_306":5951,"_308":309},"143f975faab4",[5919,5923,5927,5930,5935,5938,5943,5946],{"_37":5920,"_23":302,"_303":5921,"_174":3144},"678953f832ee",[5922,1281],"9856178",{"_37":5924,"_23":302,"_303":5925,"_174":5926},"1cd4342aaaec",[5922]," Yeah, I wouldn't dare disagree with that.",{"_37":5928,"_23":302,"_303":5929,"_174":2425},"d807e28422ed",[],{"_37":5931,"_23":302,"_303":5932,"_174":5934},"506f2a14a42a",[5933],"ad94e47","But from the other perspective,",{"_37":5936,"_23":302,"_303":5937,"_174":2425},"7056f8045d64",[],{"_37":5939,"_23":302,"_303":5940,"_174":5942},"6a8795ee3536",[5941],"9f8c132","all of that is true and we do all of that.",{"_37":5944,"_23":302,"_303":5945,"_174":2425},"b9b52990bf1b",[],{"_37":5947,"_23":302,"_303":5948,"_174":5950},"d94bd64e87b1",[5949],"4248e14","That's about being secure, but capital-S Security I think is slightly different.",[5952,5954,5956,5958],{"_37":5922,"_23":2441,"_2442":5953},"00:27:21.519",{"_37":5933,"_23":2441,"_2442":5955},"00:27:25.519",{"_37":5941,"_23":2441,"_2442":5957},"00:27:29.620",{"_37":5949,"_23":2441,"_2442":5959},"00:27:31.860",{"_37":5961,"_23":297,"_298":5962,"_306":5992,"_308":309},"f4f124754e69",[5963,5968,5971,5976,5979,5984,5987],{"_37":5964,"_23":302,"_303":5965,"_174":5967},"5bd8ce9d7733",[5966],"0a03532","That is not something that we experiment with in production.",{"_37":5969,"_23":302,"_303":5970,"_174":2425},"c3f127e74cf1",[],{"_37":5972,"_23":302,"_303":5973,"_174":5975},"378a25f455da",[5974],"4b1431d","And actually in the early days of our agent, about a year and a half ago, a year ago, we were not",{"_37":5977,"_23":302,"_303":5978,"_174":2425},"f867a94eba1e",[],{"_37":5980,"_23":302,"_303":5981,"_174":5983},"3433f380da21",[5982],"c366325","as aware of how involved that would have to be.",{"_37":5985,"_23":302,"_303":5986,"_174":2425},"c7a300fc5465",[],{"_37":5988,"_23":302,"_303":5989,"_174":5991},"a429587971b5",[5990],"0062425","And now we're hiring a whole team dedicated to it.",[5993,5995,5997,5999],{"_37":5966,"_23":2441,"_2442":5994},"00:27:37.060",{"_37":5974,"_23":2441,"_2442":5996},"00:27:41.760",{"_37":5982,"_23":2441,"_2442":5998},"00:27:52.200",{"_37":5990,"_23":2441,"_2442":6000},"00:27:55.880",{"_37":6002,"_23":297,"_298":6003,"_306":6033,"_308":309},"b40afc18cd46",[6004,6009,6012,6017,6020,6025,6028],{"_37":6005,"_23":302,"_303":6006,"_174":6008},"fb867a771bb0",[6007],"add369c","But one of the things that's really important about what we do is, we're talking about how engineers can observe and watch, but",{"_37":6010,"_23":302,"_303":6011,"_174":2425},"5de5cffb3f1c",[],{"_37":6013,"_23":302,"_303":6014,"_174":6016},"cdb34795da66",[6015],"adfeee9","IT administrators don't do that with the agents that they build for their companies.",{"_37":6018,"_23":302,"_303":6019,"_174":2425},"3c49e8d0af8f",[],{"_37":6021,"_23":302,"_303":6022,"_174":6024},"d1be9a5b0dea",[6023],"c02c162","They kind of just hope that you won't get one person social from the other, right?",{"_37":6026,"_23":302,"_303":6027,"_174":2425},"9fccb77ce56b",[],{"_37":6029,"_23":302,"_303":6030,"_174":6032},"da9a2c066bff",[6031],"b71ec33","If you're accessing Workday via MCP or whatever it is.",[6034,6036,6038,6040],{"_37":6007,"_23":2441,"_2442":6035},"00:28:00.120",{"_37":6015,"_23":2441,"_2442":6037},"00:28:08.040",{"_37":6023,"_23":2441,"_2442":6039},"00:28:12.120",{"_37":6031,"_23":2441,"_2442":6041},"00:28:16.440",{"_37":6043,"_23":297,"_298":6044,"_306":6082,"_308":309},"09834389ec0d",[6045,6050,6053,6058,6061,6066,6069,6074,6077],{"_37":6046,"_23":302,"_303":6047,"_174":6049},"2f454d5f5c8c",[6048],"5433b1f","So we have to actually kind of nerf our agent.",{"_37":6051,"_23":302,"_303":6052,"_174":2425},"74926f97589c",[],{"_37":6054,"_23":302,"_303":6055,"_174":6057},"30f4dfcdb7fe",[6056],"d8e09e2","You can't, like, full capability and autonomy,",{"_37":6059,"_23":302,"_303":6060,"_174":2425},"1bcbfb5cec13",[],{"_37":6062,"_23":302,"_303":6063,"_174":6065},"df85f945b864",[6064],"f5ab7fe","you have to at a certain point decide where you want to be on that spectrum, and you're always feeling some type of tension.",{"_37":6067,"_23":302,"_303":6068,"_174":2425},"53910d577b41",[],{"_37":6070,"_23":302,"_303":6071,"_174":6073},"f6c593eb2eae",[6072],"2d82a9a","And we basically default",{"_37":6075,"_23":302,"_303":6076,"_174":2425},"9c54b6214a47",[],{"_37":6078,"_23":302,"_303":6079,"_174":6081},"544ac3da09e8",[6080],"4fc62bd","to no autonomy, and administrators have to constantly be aware in product whenever they reveal that autonomy.",[6083,6085,6087,6089,6091],{"_37":6048,"_23":2441,"_2442":6084},"00:28:20.120",{"_37":6056,"_23":2441,"_2442":6086},"00:28:24.120",{"_37":6064,"_23":2441,"_2442":6088},"00:28:31.280",{"_37":6072,"_23":2441,"_2442":6090},"00:28:39.040",{"_37":6080,"_23":2441,"_2442":6092},"00:28:42.960",{"_37":6094,"_23":297,"_298":6095,"_306":6117,"_308":309},"185e01f2c88f",[6096,6101,6104,6109,6112],{"_37":6097,"_23":302,"_303":6098,"_174":6100},"377c231c4363",[6099],"63425f3","So for instance, if I have an agent that has access to a personal Slack channel or private Slack channel,",{"_37":6102,"_23":302,"_303":6103,"_174":2425},"aed67ee0e89d",[],{"_37":6105,"_23":302,"_303":6106,"_174":6108},"eab099bde9fd",[6107],"42f5cc7","and that agent lives in a different Slack channel.",{"_37":6110,"_23":302,"_303":6111,"_174":2425},"6f8949bfed61",[],{"_37":6113,"_23":302,"_303":6114,"_174":6116},"1ea439491b20",[6115],"bacab43","If people want to talk to that agent, do they access?",[6118,6120,6122],{"_37":6099,"_23":2441,"_2442":6119},"00:28:51.040",{"_37":6107,"_23":2441,"_2442":6121},"00:28:56.240",{"_37":6115,"_23":2441,"_2442":6123},"00:28:59.600",{"_37":6125,"_23":297,"_298":6126,"_306":6156,"_308":309},"205b52300a16",[6127,6132,6135,6140,6143,6148,6151],{"_37":6128,"_23":302,"_303":6129,"_174":6131},"939c8507db77",[6130],"fc3f7f7","This actually happened the very, very beginning of Notion custom agents.",{"_37":6133,"_23":302,"_303":6134,"_174":2425},"3de28d5f6956",[],{"_37":6136,"_23":302,"_303":6137,"_174":6139},"bd8cd682a70b",[6138],"2212a8a","Our CEO's chief of staff accidentally added an all-hands Q&A bot that had access to the private C-suite Slack channel.",{"_37":6141,"_23":302,"_303":6142,"_174":2425},"dd3bd5bc3222",[],{"_37":6144,"_23":302,"_303":6145,"_174":6147},"1b4150cf8fee",[6146],"7811a99","So those are the types of things that we learned really early.",{"_37":6149,"_23":302,"_303":6150,"_174":2425},"a2b14eea080b",[],{"_37":6152,"_23":302,"_303":6153,"_174":6155},"8a64a5e34d83",[6154],"42bd102","You shouldn't be able to configure, and it is impossible to do that.",[6157,6159,6161,6163],{"_37":6130,"_23":2441,"_2442":6158},"00:29:02.080",{"_37":6138,"_23":2441,"_2442":6160},"00:29:05.160",{"_37":6146,"_23":2441,"_2442":6162},"00:29:12.600",{"_37":6154,"_23":2441,"_2442":6164},"00:29:15.560",{"_37":6166,"_23":297,"_298":6167,"_306":6197,"_308":309},"b0ee965f9404",[6168,6173,6176,6181,6184,6189,6192],{"_37":6169,"_23":302,"_303":6170,"_174":6172},"275831497df2",[6171],"0b79862","And people complain that their agents feel nerfed and dumb.",{"_37":6174,"_23":302,"_303":6175,"_174":2425},"c24edbd3fa82",[],{"_37":6177,"_23":302,"_303":6178,"_174":6180},"53ee14eaab9d",[6179],"0716408","And our job is to not take that away and then use LLMs to make it harder to do that.",{"_37":6182,"_23":302,"_303":6183,"_174":2425},"378e18fa1a80",[],{"_37":6185,"_23":302,"_303":6186,"_174":6188},"aed7a6d0fbcc",[6187],"ec362ea","Our job is to keep it there and add the right toggles and customer education.",{"_37":6190,"_23":302,"_303":6191,"_174":2425},"b2ce81de3b35",[],{"_37":6193,"_23":302,"_303":6194,"_174":6196},"aadcec307746",[6195],"15e0fa6","And then at a certain point, I think that in about five years we'll all feel a lot more comfortable reasoning through those trade-offs.",[6198,6200,6202,6204],{"_37":6171,"_23":2441,"_2442":6199},"00:29:18.460",{"_37":6179,"_23":2441,"_2442":6201},"00:29:21.820",{"_37":6187,"_23":2441,"_2442":6203},"00:29:26.460",{"_37":6195,"_23":2441,"_2442":6205},"00:29:30.919",{"_37":6207,"_23":297,"_298":6208,"_306":6213,"_308":382},"b854ed336dbb",[6209],{"_37":6210,"_23":302,"_303":6211,"_174":6212},"6ccb03474899",[],"Discerning Attackers from the Vicious Loop",[],{"_37":6215,"_23":297,"_298":6216,"_306":6249,"_308":309},"7c1d3f923b4c",[6217,6221,6225,6228,6233,6236,6241,6244],{"_37":6218,"_23":302,"_303":6219,"_174":3026},"e4c1f385ca82",[6220,1281],"33705a3",{"_37":6222,"_23":302,"_303":6223,"_174":6224},"ea21c232cb14",[6220]," One specific question pertaining to product, to all three of you folks, which is agents viciously are goal-directed to get the job done.",{"_37":6226,"_23":302,"_303":6227,"_174":2425},"25c9cb115582",[],{"_37":6229,"_23":302,"_303":6230,"_174":6232},"8c8968b27a5e",[6231],"4f5719a","And often when we are measuring the value of our product, we want to sense how users are using our product.",{"_37":6234,"_23":302,"_303":6235,"_174":2425},"931628940683",[],{"_37":6237,"_23":302,"_303":6238,"_174":6240},"d906ae5dd076",[6239],"24fa19a","But when these agents are running in the loop,",{"_37":6242,"_23":302,"_303":6243,"_174":2425},"b68946681694",[],{"_37":6245,"_23":302,"_303":6246,"_174":6248},"9b960aa8a8bb",[6247],"34dafde","you don't necessarily have a pattern to discern anomaly from standard behavior.",[6250,6252,6254,6256],{"_37":6220,"_23":2441,"_2442":6251},"00:29:39.320",{"_37":6231,"_23":2441,"_2442":6253},"00:29:50.940",{"_37":6239,"_23":2441,"_2442":6255},"00:29:56.779",{"_37":6247,"_23":2441,"_2442":6257},"00:29:59.720",{"_37":6259,"_23":297,"_298":6260,"_306":6282,"_308":309},"e82b3d9fc8fc",[6261,6266,6269,6274,6277],{"_37":6262,"_23":302,"_303":6263,"_174":6265},"c4b531ae4323",[6264],"8a25e6f","Which means when someone expresses intent, go to Notion using an agent, just list my",{"_37":6267,"_23":302,"_303":6268,"_174":2425},"0eed90926f72",[],{"_37":6270,"_23":302,"_303":6271,"_174":6273},"4fd29eb5a3c7",[6272],"5cb5ec1","files in a folder.",{"_37":6275,"_23":302,"_303":6276,"_174":2425},"88b436bea020",[],{"_37":6278,"_23":302,"_303":6279,"_174":6281},"85738491a2b5",[6280],"f0b1d1c","The agent lands up going through a situational awareness where it scans through your entire workspace based on the permission and then figures out how to align with the intent.",[6283,6285,6287],{"_37":6264,"_23":2441,"_2442":6284},"00:30:05.640",{"_37":6272,"_23":2441,"_2442":6286},"00:30:14.320",{"_37":6280,"_23":2441,"_2442":6288},"00:30:16.480",{"_37":6290,"_23":297,"_298":6291,"_306":6305,"_308":309},"e7511566dd53",[6292,6297,6300],{"_37":6293,"_23":302,"_303":6294,"_174":6296},"7a02067c761b",[6295],"2668966","So each of our product, we are measuring user engagement, value outcomes, goal alignment, and anomalies, which are non-standard behaviors, which could be essentially triggered by attackers as well.",{"_37":6298,"_23":302,"_303":6299,"_174":2425},"8d9526dee485",[],{"_37":6301,"_23":302,"_303":6302,"_174":6304},"e6184457f287",[6303],"1ee6759","So as you think through monitoring in your product, how are you discerning an attacker from an agent going through its normal vicious loop?",[6306,6308],{"_37":6295,"_23":2441,"_2442":6307},"00:30:28.419",{"_37":6303,"_23":2441,"_2442":6309},"00:30:41.380",{"_37":6311,"_23":297,"_298":6312,"_306":6353,"_308":309},"ce5e7af659d8",[6313,6317,6321,6324,6329,6332,6337,6340,6345,6348],{"_37":6314,"_23":302,"_303":6315,"_174":2983},"7699d8f8e95a",[6316,1281],"5becfdb",{"_37":6318,"_23":302,"_303":6319,"_174":6320},"70081a1f52f6",[6316]," Well, I'm not a security person.",{"_37":6322,"_23":302,"_303":6323,"_174":2425},"fcafbb4cdc53",[],{"_37":6325,"_23":302,"_303":6326,"_174":6328},"a9289df89116",[6327],"ac28542","This is really not my area of expertise, but this is one of the reasons that I think that the way that you observe,",{"_37":6330,"_23":302,"_303":6331,"_174":2425},"a94c0907d005",[],{"_37":6333,"_23":302,"_303":6334,"_174":6336},"7209a67909d3",[6335],"69f6646","if all you have are aggregates, an attacker is often only one outlier.",{"_37":6338,"_23":302,"_303":6339,"_174":2425},"cf64d7e10b91",[],{"_37":6341,"_23":302,"_303":6342,"_174":6344},"d23aff1a045c",[6343],"21dca27","And if all you can see is a dashboard, you can't find, and this is something where AI could really help, like:",{"_37":6346,"_23":302,"_303":6347,"_174":2425},"24bf3a3813ef",[],{"_37":6349,"_23":302,"_303":6350,"_174":6352},"32c1fdbd3aa7",[6351],"f9a1398","show me the max every day is, from a security standpoint, or the min, right?",[6354,6356,6358,6360,6362],{"_37":6316,"_23":2441,"_2442":6355},"00:30:54.059",{"_37":6327,"_23":2441,"_2442":6357},"00:30:56.299",{"_37":6335,"_23":2441,"_2442":6359},"00:31:03.480",{"_37":6343,"_23":2441,"_2442":6361},"00:31:10.440",{"_37":6351,"_23":2441,"_2442":6363},"00:31:17.440",{"_37":6365,"_23":297,"_298":6366,"_306":6372,"_308":693},"c6e49a675f17",[6367],{"_37":6368,"_23":302,"_303":6369,"_174":6371},"fc18cf9b5373",[6370],"2a169ee","They're often outliers, but if you aren't starting with that raw granular source of truth, you literally have no ability to see them.",[6373],{"_37":6370,"_23":2441,"_2442":6374},"00:31:21.760",{"_37":6376,"_23":297,"_298":6377,"_306":6394,"_308":309},"edafbfe96171",[6378,6382,6386,6389],{"_37":6379,"_23":302,"_303":6380,"_174":3144},"a15edbff4b21",[6381,1281],"0042443",{"_37":6383,"_23":302,"_303":6384,"_174":6385},"ecd08c88df9a",[6381]," Yeah, we actually use agents, we have a whole process where we use agents to continuously try and find vulnerabilities of",{"_37":6387,"_23":302,"_303":6388,"_174":2425},"1b939fd8840b",[],{"_37":6390,"_23":302,"_303":6391,"_174":6393},"6affba26c108",[6392],"3b78368","deprecated endpoints or things inside Notion that might expose other information.",[6395,6397],{"_37":6381,"_23":2441,"_2442":6396},"00:31:32.360",{"_37":6392,"_23":2441,"_2442":6398},"00:31:42.740",{"_37":6400,"_23":297,"_298":6401,"_306":6431,"_308":309},"b534051d54c5",[6402,6407,6410,6415,6418,6423,6426],{"_37":6403,"_23":302,"_303":6404,"_174":6406},"c8622a8455af",[6405],"d9c75b1","But we certainly, I mean, Notion",{"_37":6408,"_23":302,"_303":6409,"_174":2425},"d666c1fe2d60",[],{"_37":6411,"_23":302,"_303":6412,"_174":6414},"6d4e263cc604",[6413],"49e4e77","up until usage-based pricing launch on Tuesday was the only place you could get unlimited Opus for twenty dollars a month.",{"_37":6416,"_23":302,"_303":6417,"_174":2425},"9a14dde54553",[],{"_37":6419,"_23":302,"_303":6420,"_174":6422},"8f73402cea53",[6421],"1591466","So we certainly had a lot of maybe foreign attempts at distillation or",{"_37":6424,"_23":302,"_303":6425,"_174":2425},"82d67e7c400d",[],{"_37":6427,"_23":302,"_303":6428,"_174":6430},"4d1b49e8891f",[6429],"bf3a946","really, really, really heavy usage in languages of countries that are not allowed to use Notion, and we certainly had to create our abuse team, which I can't speak too much to, but",[6432,6434,6436,6438],{"_37":6405,"_23":2441,"_2442":6433},"00:31:48.500",{"_37":6413,"_23":2441,"_2442":6435},"00:31:51.840",{"_37":6421,"_23":2441,"_2442":6437},"00:31:58.720",{"_37":6429,"_23":2441,"_2442":6439},"00:32:04.040",{"_37":6441,"_23":297,"_298":6442,"_306":6456,"_308":309},"f951924cfb46",[6443,6448,6451],{"_37":6444,"_23":302,"_303":6445,"_174":6447},"ae3a7e8ad868",[6446],"9f6a1e7","with MCP usage and Notion usage being 30% of the hit on our system, then it's like, accordingly, our abuse and fraud team focus on that",{"_37":6449,"_23":302,"_303":6450,"_174":2425},"29a6a74ae678",[],{"_37":6452,"_23":302,"_303":6453,"_174":6455},"a2a440837f85",[6454],"d97bf37","and abuse of our MCP just as much as users hosting inappropriate content on Notion as individuals, which is their former role, right?",[6457,6459],{"_37":6446,"_23":2441,"_2442":6458},"00:32:16.800",{"_37":6454,"_23":2441,"_2442":6460},"00:32:27.580",{"_37":6462,"_23":297,"_298":6463,"_306":6496,"_308":309},"ded656a0a79a",[6464,6468,6472,6475,6480,6483,6488,6491],{"_37":6465,"_23":302,"_303":6466,"_174":3743},"e30de1bf0bcc",[6467,1281],"e8cd479",{"_37":6469,"_23":302,"_303":6470,"_174":6471},"fc1db9490632",[6467]," Yeah, I really like Charity's answer here.",{"_37":6473,"_23":302,"_303":6474,"_174":2425},"fb6f993a6af4",[],{"_37":6476,"_23":302,"_303":6477,"_174":6479},"68f682a1d95a",[6478],"fa0b9e1","And I'm not gonna be on stage to disagree with Charity about observability ever.",{"_37":6481,"_23":302,"_303":6482,"_174":2425},"c922dbb646f3",[],{"_37":6484,"_23":302,"_303":6485,"_174":6487},"6f312afe428c",[6486],"dd09f42","So I think it's first of all all about the data, and then secondly, different",{"_37":6489,"_23":302,"_303":6490,"_174":2425},"5fb002267b89",[],{"_37":6492,"_23":302,"_303":6493,"_174":6495},"fe199e951c36",[6494],"278fd91","sets of data will have different levels of privacy or sensitivity.",[6497,6499,6501,6503],{"_37":6467,"_23":2441,"_2442":6498},"00:32:37.780",{"_37":6478,"_23":2441,"_2442":6500},"00:32:40.260",{"_37":6486,"_23":2441,"_2442":6502},"00:32:45.140",{"_37":6494,"_23":2441,"_2442":6504},"00:32:51.019",{"_37":6506,"_23":297,"_298":6507,"_306":6545,"_308":309},"715971d3fe4c",[6508,6513,6516,6521,6524,6529,6532,6537,6540],{"_37":6509,"_23":302,"_303":6510,"_174":6512},"d2809ef04369",[6511],"8e351a4","And so I think that there is some nuance that's needed as well.",{"_37":6514,"_23":302,"_303":6515,"_174":2425},"957a7c0a1977",[],{"_37":6517,"_23":302,"_303":6518,"_174":6520},"dbba56ce96a2",[6519],"eae8e14","And depending on the kind of data or the kind of attack vectors, you can then, yeah, you can pattern match and",{"_37":6522,"_23":302,"_303":6523,"_174":2425},"b3ed2145f51a",[],{"_37":6525,"_23":302,"_303":6526,"_174":6528},"e2bef19436d6",[6527],"3e71ed8","work accordingly.",{"_37":6530,"_23":302,"_303":6531,"_174":2425},"d48e2aec0b7b",[],{"_37":6533,"_23":302,"_303":6534,"_174":6536},"5d3f46ea096f",[6535],"a1343ca","I think this is a very hard problem.",{"_37":6538,"_23":302,"_303":6539,"_174":2425},"7dc8de2ae5c1",[],{"_37":6541,"_23":302,"_303":6542,"_174":6544},"37a727492874",[6543],"5a9628d","I don't have a very good answer, because I am also not a security person and it's something that we're actively",[6546,6548,6550,6552,6554],{"_37":6511,"_23":2441,"_2442":6547},"00:32:55.580",{"_37":6519,"_23":2441,"_2442":6549},"00:33:01.720",{"_37":6527,"_23":2441,"_2442":6551},"00:33:10.720",{"_37":6535,"_23":2441,"_2442":6553},"00:33:11.440",{"_37":6543,"_23":2441,"_2442":6555},"00:33:12.640",{"_37":6557,"_23":297,"_298":6558,"_306":6583,"_308":309},"7fdeb31a4ace",[6559,6563,6567,6570,6575,6578],{"_37":6560,"_23":302,"_303":6561,"_174":3144},"fbbf2258c23c",[6562,1281],"a526bce",{"_37":6564,"_23":302,"_303":6565,"_174":6566},"9ea1e5cb1dc9",[6562]," There's a lot of really cool companies that do that.",{"_37":6568,"_23":302,"_303":6569,"_174":2425},"57e7692fd7b8",[],{"_37":6571,"_23":302,"_303":6572,"_174":6574},"cb023559d04a",[6573],"daca394","I think OpenAI just acquired one, but we use like a number of vendors that do",{"_37":6576,"_23":302,"_303":6577,"_174":2425},"2e34582f0538",[],{"_37":6579,"_23":302,"_303":6580,"_174":6582},"d8d96594db1e",[6581],"4664d28","trace analysis and also red teaming, where you basically set up a reinforcement learning environment in your system for your agents and then they automatically red team your system.",[6584,6586,6588],{"_37":6562,"_23":2441,"_2442":6585},"00:33:20.300",{"_37":6573,"_23":2441,"_2442":6587},"00:33:21.740",{"_37":6581,"_23":2441,"_2442":6589},"00:33:26.640",{"_37":6591,"_23":297,"_298":6592,"_306":6606,"_308":309},"73864aee5bbe",[6593,6598,6601],{"_37":6594,"_23":302,"_303":6595,"_174":6597},"ad2839835abc",[6596],"2b11ec7","I think also Simon's giving a talk later, he also",{"_37":6599,"_23":302,"_303":6600,"_174":2425},"fc4b157c7c44",[],{"_37":6602,"_23":302,"_303":6603,"_174":6605},"640fbe3bf287",[6604],"a0f5abf","has done a lot of research and has a lot of thoughts on like the lethal trifecta in this space.",[6607,6609],{"_37":6596,"_23":2441,"_2442":6608},"00:33:36.400",{"_37":6604,"_23":2441,"_2442":6610},"00:33:38.980",{"_37":6612,"_23":297,"_298":6613,"_306":6630,"_308":309},"6d11980b78ba",[6614,6618,6622,6625],{"_37":6615,"_23":302,"_303":6616,"_174":2983},"11f380ff8bab",[6617,1281],"61ad6eb",{"_37":6619,"_23":302,"_303":6620,"_174":6621},"4d1b3f78336f",[6617]," Yeah, plus one, Simon's stuff is so great.",{"_37":6623,"_23":302,"_303":6624,"_174":2425},"9a4b810edae7",[],{"_37":6626,"_23":302,"_303":6627,"_174":6629},"e3845ffb9ef8",[6628],"40139a2","Also I just made a sticker that says tracing is caring.",[6631,6633],{"_37":6617,"_23":2441,"_2442":6632},"00:33:43.220",{"_37":6628,"_23":2441,"_2442":6634},"00:33:46.180",{"_37":6636,"_23":297,"_298":6637,"_306":6642,"_308":382},"cc7e3a5d9984",[6638],{"_37":6639,"_23":302,"_303":6640,"_174":6641},"8c3daeab2489",[],"Traces, Sampling, and What Comes Next",[],{"_37":6644,"_23":297,"_298":6645,"_306":6670,"_308":309},"199d173f35d5",[6646,6650,6654,6657,6662,6665],{"_37":6647,"_23":302,"_303":6648,"_174":3026},"e51cbf4ef8fd",[6649,1281],"ecef8a7",{"_37":6651,"_23":302,"_303":6652,"_174":6653},"98e3d135d06b",[6649]," Charity, I have a follow-up on tracing, right?",{"_37":6655,"_23":302,"_303":6656,"_174":2425},"2cd3af94982f",[],{"_37":6658,"_23":302,"_303":6659,"_174":6661},"16ff6a5a4cae",[6660],"46cfa7b","The notion of tracing has primitives in it, primitives like spans and traces.",{"_37":6663,"_23":302,"_303":6664,"_174":2425},"6b254cf3a8cb",[],{"_37":6666,"_23":302,"_303":6667,"_174":6669},"6e01a6461db6",[6668],"ef481b2","There's certain work trees, etc.",[6671,6673,6675],{"_37":6649,"_23":2441,"_2442":6672},"00:33:50.860",{"_37":6660,"_23":2441,"_2442":6674},"00:33:55.820",{"_37":6668,"_23":2441,"_2442":6676},"00:34:00.380",{"_37":6678,"_23":297,"_298":6679,"_306":6709,"_308":309},"6a99bd603aa8",[6680,6685,6688,6693,6696,6701,6704],{"_37":6681,"_23":302,"_303":6682,"_174":6684},"2adb53b572c0",[6683],"0978f9a","Now",{"_37":6686,"_23":302,"_303":6687,"_174":2425},"1aae2450c019",[],{"_37":6689,"_23":302,"_303":6690,"_174":6692},"4bada9ab2ed0",[6691],"184bcb8","those fundamentals are somewhat changing to a certain extent, or getting cargo-culted, or perhaps getting reframed.",{"_37":6694,"_23":302,"_303":6695,"_174":2425},"78cd3d13597a",[],{"_37":6697,"_23":302,"_303":6698,"_174":6700},"137848ca807f",[6699],"7cf5fc0","What is your opinion on that?",{"_37":6702,"_23":302,"_303":6703,"_174":2425},"ac2e2983c411",[],{"_37":6705,"_23":302,"_303":6706,"_174":6708},"fd9a23860045",[6707],"9f0e95b","How do you see this new world evolve?",[6710,6712,6714,6716],{"_37":6683,"_23":2441,"_2442":6711},"00:34:01.980",{"_37":6691,"_23":2441,"_2442":6713},"00:34:02.440",{"_37":6699,"_23":2441,"_2442":6715},"00:34:10.119",{"_37":6707,"_23":2441,"_2442":6717},"00:34:11.399",{"_37":6719,"_23":297,"_298":6720,"_306":6729,"_308":309},"a17ee093a9b2",[6721,6725],{"_37":6722,"_23":302,"_303":6723,"_174":2983},"92d780426cb6",[6724,1281],"01c34bb",{"_37":6726,"_23":302,"_303":6727,"_174":6728},"6a62a42a48bb",[6724]," You know, data is data, and I really hate that we get so wrapped around the axle about, well, a trace is just a log with special fields.",[6730],{"_37":6724,"_23":2441,"_2442":6731},"00:34:13.720",{"_37":6733,"_23":297,"_298":6734,"_306":6743,"_308":309},"78ec301f64c5",[6735,6739],{"_37":6736,"_23":302,"_303":6737,"_174":3144},"e813102fa511",[6738,1281],"f0b7c6f",{"_37":6740,"_23":302,"_303":6741,"_174":6742},"05615cc78e1d",[6738]," No, it's a log with user data.",[6744],{"_37":6738,"_23":2441,"_2442":6745},"00:34:23.399",{"_37":6747,"_23":297,"_298":6748,"_306":6773,"_308":309},"8212232026ba",[6749,6753,6757,6760,6765,6768],{"_37":6750,"_23":302,"_303":6751,"_174":2983},"16026997624d",[6752,1281],"90f7cb5",{"_37":6754,"_23":302,"_303":6755,"_174":6756},"1516012f48e0",[6752]," Oh, that's an interesting take.",{"_37":6758,"_23":302,"_303":6759,"_174":2425},"2eda48ac56f2",[],{"_37":6761,"_23":302,"_303":6762,"_174":6764},"3ee7abbf9ffb",[6763],"1268a16","So look, logs have user data in them.",{"_37":6766,"_23":302,"_303":6767,"_174":2425},"bbf0fb568993",[],{"_37":6769,"_23":302,"_303":6770,"_174":6772},"e921d00d7f10",[6771],"aa69f8f","No?",[6774,6776,6778],{"_37":6752,"_23":2441,"_2442":6775},"00:34:26.200",{"_37":6763,"_23":2441,"_2442":6777},"00:34:28.440",{"_37":6771,"_23":2441,"_2442":6779},"00:34:31.399",{"_37":6781,"_23":297,"_298":6782,"_306":6807,"_308":309},"36e96e4a3cbe",[6783,6787,6791,6794,6799,6802],{"_37":6784,"_23":302,"_303":6785,"_174":3144},"e26d6e7f384d",[6786,1281],"a6ad759",{"_37":6788,"_23":302,"_303":6789,"_174":6790},"e681585b2718",[6786]," At least at Notion, trace observability is so much harder because we don't auto-collect it from our users.",{"_37":6792,"_23":302,"_303":6793,"_174":2425},"d5637e9f7c63",[],{"_37":6795,"_23":302,"_303":6796,"_174":6798},"f030b512490f",[6797],"39a4390","It includes like all of their search history of their workspace, and our enterprise users, we don't",{"_37":6800,"_23":302,"_303":6801,"_174":2425},"466034977a7d",[],{"_37":6803,"_23":302,"_303":6804,"_174":6806},"c547026e9d11",[6805],"cdfcbc1","collect that.",[6808,6810,6812],{"_37":6786,"_23":2441,"_2442":6809},"00:34:32.580",{"_37":6797,"_23":2441,"_2442":6811},"00:34:40.980",{"_37":6805,"_23":2441,"_2442":6813},"00:34:45.040",{"_37":6815,"_23":297,"_298":6816,"_306":6865,"_308":309},"8f4378192694",[6817,6821,6825,6828,6833,6836,6841,6844,6849,6852,6857,6860],{"_37":6818,"_23":302,"_303":6819,"_174":2983},"267c4b9c5e85",[6820,1281],"56b5845",{"_37":6822,"_23":302,"_303":6823,"_174":6824},"15d782a835a0",[6820]," All right, I have so many questions for you, and maybe we'll talk about them later.",{"_37":6826,"_23":302,"_303":6827,"_174":2425},"2c33b27faf2e",[],{"_37":6829,"_23":302,"_303":6830,"_174":6832},"5a46fb89239f",[6831],"cde687e","For a long time I used to tell everyone,",{"_37":6834,"_23":302,"_303":6835,"_174":2425},"3df513a2e652",[],{"_37":6837,"_23":302,"_303":6838,"_174":6840},"1cbe0cf27d83",[6839],"78de92d","just collect rich structured logs like the Stripe canonical log style, right?",{"_37":6842,"_23":302,"_303":6843,"_174":2425},"1cc154f18722",[],{"_37":6845,"_23":302,"_303":6846,"_174":6848},"2f8889de0bf8",[6847],"c806cef","Just make them as wide as possible, because you want as much context as possible,",{"_37":6850,"_23":302,"_303":6851,"_174":2425},"8613b093cc29",[],{"_37":6853,"_23":302,"_303":6854,"_174":6856},"1fa4dae9abb4",[6855],"825474d","because the context wraps them together and says these things all hold true for the same request, for the same user.",{"_37":6858,"_23":302,"_303":6859,"_174":2425},"d02cc1466834",[],{"_37":6861,"_23":302,"_303":6862,"_174":6864},"3fc0e2243c1c",[6863],"3d6cb2e","It's magical.",[6866,6868,6870,6872,6874,6876],{"_37":6820,"_23":2441,"_2442":6867},"00:34:45.760",{"_37":6831,"_23":2441,"_2442":6869},"00:34:49.920",{"_37":6839,"_23":2441,"_2442":6871},"00:34:53.780",{"_37":6847,"_23":2441,"_2442":6873},"00:34:58.100",{"_37":6855,"_23":2441,"_2442":6875},"00:35:02.660",{"_37":6863,"_23":2441,"_2442":6877},"00:35:08.500",{"_37":6879,"_23":297,"_298":6880,"_306":6910,"_308":309},"7e9c4274ca64",[6881,6886,6889,6894,6897,6902,6905],{"_37":6882,"_23":302,"_303":6883,"_174":6885},"7c5cf0ede8a0",[6884],"3aec5d5","And if you explode, if you tear it up into many different signal types,",{"_37":6887,"_23":302,"_303":6888,"_174":2425},"a2b4f49bbeaf",[],{"_37":6890,"_23":302,"_303":6891,"_174":6893},"288093d27666",[6892],"081167f","you can never put Humpty back together again.",{"_37":6895,"_23":302,"_303":6896,"_174":2425},"5e4d0e23e8ed",[],{"_37":6898,"_23":302,"_303":6899,"_174":6901},"4e7387040f6d",[6900],"94bc31d","And that is the most valuable part of your data.",{"_37":6903,"_23":302,"_303":6904,"_174":2425},"4d1e4d750a9f",[],{"_37":6906,"_23":302,"_303":6907,"_174":6909},"5d99f14a279d",[6908],"fc38855","I know this is a dead horse, but I'm just gonna beat it a little bit more.",[6911,6913,6915,6917],{"_37":6884,"_23":2441,"_2442":6912},"00:35:09.460",{"_37":6892,"_23":2441,"_2442":6914},"00:35:14.559",{"_37":6900,"_23":2441,"_2442":6916},"00:35:17.119",{"_37":6908,"_23":2441,"_2442":6918},"00:35:19.359",{"_37":6920,"_23":297,"_298":6921,"_306":6927,"_308":693},"4a42826c2800",[6922],{"_37":6923,"_23":302,"_303":6924,"_174":6926},"0799c7caae85",[6925],"9294d31","The value of additional context goes up, not linearly, not even exponentially, combinatorially.",[6928],{"_37":6925,"_23":2441,"_2442":6929},"00:35:23.620",{"_37":6931,"_23":297,"_298":6932,"_306":6962,"_308":309},"7b6fd48a50de",[6933,6938,6941,6946,6949,6954,6957],{"_37":6934,"_23":302,"_303":6935,"_174":6937},"6cd3c8fa763e",[6936],"ca7a562","If you have a",{"_37":6939,"_23":302,"_303":6940,"_174":2425},"417c4a794b68",[],{"_37":6942,"_23":302,"_303":6943,"_174":6945},"f40b73b31831",[6944],"9b22195","structured log or a trace that is 29 dimensions wide and you add a 30th, you have made it more valuable than all of the previous 29 were.",{"_37":6947,"_23":302,"_303":6948,"_174":2425},"5229431240c1",[],{"_37":6950,"_23":302,"_303":6951,"_174":6953},"b7dff1e446b2",[6952],"07350ea","It is so valuable.",{"_37":6955,"_23":302,"_303":6956,"_174":2425},"5cfd6a22d2e6",[],{"_37":6958,"_23":302,"_303":6959,"_174":6961},"221bef4b76a2",[6960],"e03c997","All right, I think I've made my point.",[6963,6965,6967,6969],{"_37":6936,"_23":2441,"_2442":6964},"00:35:30.340",{"_37":6944,"_23":2441,"_2442":6966},"00:35:31.540",{"_37":6952,"_23":2441,"_2442":6968},"00:35:40.820",{"_37":6960,"_23":2441,"_2442":6970},"00:35:42.260",{"_37":6972,"_23":297,"_298":6973,"_306":6994,"_308":309},"c9f80e7b72cd",[6974,6978,6981,6986,6989],{"_37":6975,"_23":302,"_303":6976,"_174":5291},"d9497dde131e",[6977],"0d716a1",{"_37":6979,"_23":302,"_303":6980,"_174":2425},"a95c2dbe6cb8",[],{"_37":6982,"_23":302,"_303":6983,"_174":6985},"2127446163b0",[6984],"4474132","it's hard to separate in people's minds what is changing in the way that vendors are defining their products from the data itself.",{"_37":6987,"_23":302,"_303":6988,"_174":2425},"12ccad012f9e",[],{"_37":6990,"_23":302,"_303":6991,"_174":6993},"2bc999a80529",[6992],"ac2b9cf","And I would say it is not that complicated.",[6995,6997,6999],{"_37":6977,"_23":2441,"_2442":6996},"00:35:44.900",{"_37":6984,"_23":2441,"_2442":6998},"00:35:46.099",{"_37":6992,"_23":2441,"_2442":7000},"00:35:55.460",{"_37":7002,"_23":297,"_298":7003,"_306":7025,"_308":309},"f1284873117a",[7004,7009,7012,7017,7020],{"_37":7005,"_23":302,"_303":7006,"_174":7008},"722fc624e8d3",[7007],"d5160a3","Traces are important, but they've historically been treated kind of like a second-class citizen, because if you're an observability company and you've got 30 or 40 products,",{"_37":7010,"_23":302,"_303":7011,"_174":2425},"0da6e5792b61",[],{"_37":7013,"_23":302,"_303":7014,"_174":7016},"becacf8a006d",[7015],"406aa8e","I don't want to say that you're motivated to keep selling more and more products.",{"_37":7018,"_23":302,"_303":7019,"_174":2425},"a07d02cb2c89",[],{"_37":7021,"_23":302,"_303":7022,"_174":7024},"7d96b2e56651",[7023],"5164ca8","You don't, because I'm sure that that's not the core motivation, but",[7026,7028,7030],{"_37":7007,"_23":2441,"_2442":7027},"00:35:58.700",{"_37":7015,"_23":2441,"_2442":7029},"00:36:10.779",{"_37":7023,"_23":2441,"_2442":7031},"00:36:14.299",{"_37":7033,"_23":297,"_298":7034,"_306":7059,"_308":309},"384affbd3f96",[7035,7039,7043,7046,7051,7054],{"_37":7036,"_23":302,"_303":7037,"_174":3144},"1ba40c767044",[7038,1281],"e454cde",{"_37":7040,"_23":302,"_303":7041,"_174":7042},"369852e08b24",[7038]," No, I mean today LLM traces are treated as after-the-fact, ad hoc, or CI offline-based evaluation mechanisms.",{"_37":7044,"_23":302,"_303":7045,"_174":2425},"5aeaee040353",[],{"_37":7047,"_23":302,"_303":7048,"_174":7050},"cfb9502f0b60",[7049],"cc05fb0","The market needs to converge.",{"_37":7052,"_23":302,"_303":7053,"_174":2425},"a112a41034aa",[],{"_37":7055,"_23":302,"_303":7056,"_174":7058},"682aaed53ab1",[7057],"e2a7b2e","Stop splintering.",[7060,7062,7064],{"_37":7038,"_23":2441,"_2442":7061},"00:36:19.839",{"_37":7049,"_23":2441,"_2442":7063},"00:36:29.260",{"_37":7057,"_23":2441,"_2442":7065},"00:36:31.020",{"_37":7067,"_23":297,"_298":7068,"_306":7090,"_308":309},"3ca8ec081db3",[7069,7074,7077,7082,7085],{"_37":7070,"_23":302,"_303":7071,"_174":7073},"72fc88230cd4",[7072],"b77b34d","But it's a lot of data.",{"_37":7075,"_23":302,"_303":7076,"_174":2425},"3619117f7cb4",[],{"_37":7078,"_23":302,"_303":7079,"_174":7081},"0aeb606de020",[7080],"27fdde5","It's expensive.",{"_37":7083,"_23":302,"_303":7084,"_174":2425},"5170ffaf25fc",[],{"_37":7086,"_23":302,"_303":7087,"_174":7089},"921dda731ea2",[7088],"06529c7","We can talk about that privately too, but it's an expensive amount of data.",[7091,7093,7095],{"_37":7072,"_23":2441,"_2442":7092},"00:36:32.060",{"_37":7080,"_23":2441,"_2442":7094},"00:36:35.180",{"_37":7088,"_23":2441,"_2442":7096},"00:36:37.180",{"_37":7098,"_23":297,"_298":7099,"_306":7132,"_308":309},"8512de5601eb",[7100,7104,7108,7111,7116,7119,7124,7127],{"_37":7101,"_23":302,"_303":7102,"_174":2983},"693fd744b463",[7103,1281],"559c61f",{"_37":7105,"_23":302,"_303":7106,"_174":7107},"83268e1c403e",[7103]," But do you know what's more expensive?",{"_37":7109,"_23":302,"_303":7110,"_174":2425},"1a36766fed5b",[],{"_37":7112,"_23":302,"_303":7113,"_174":7115},"8baebe51b519",[7114],"e62e588","Storing the traces and storing the logs and the metrics and the exceptions and the errors and the profiling data.",{"_37":7117,"_23":302,"_303":7118,"_174":2425},"72c2fd98f9bf",[],{"_37":7120,"_23":302,"_303":7121,"_174":7123},"ed55d93e4ed7",[7122],"005892e","Store it once,",{"_37":7125,"_23":302,"_303":7126,"_174":2425},"127a21cd61d7",[],{"_37":7128,"_23":302,"_303":7129,"_174":7131},"38f2ca048120",[7130],"4aab9f7","preserve relationships, and then I feel like a whole generation of engineers were scarred by Splunk marketing.",[7133,7135,7137,7139],{"_37":7103,"_23":2441,"_2442":7134},"00:36:41.180",{"_37":7114,"_23":2441,"_2442":7136},"00:36:43.120",{"_37":7122,"_23":2441,"_2442":7138},"00:36:50.400",{"_37":7130,"_23":2441,"_2442":7140},"00:36:51.740",{"_37":7142,"_23":297,"_298":7143,"_306":7165,"_308":309},"6e0a393a878f",[7144,7149,7152,7157,7160],{"_37":7145,"_23":302,"_303":7146,"_174":7148},"bea5e1de2379",[7147],"40ed2f3","Every log line is sacred.",{"_37":7150,"_23":302,"_303":7151,"_174":2425},"af01d2b62b17",[],{"_37":7153,"_23":302,"_303":7154,"_174":7156},"3eed49b28d1b",[7155],"c1a373f","You cannot",{"_37":7158,"_23":302,"_303":7159,"_174":2425},"aeb131a398c2",[],{"_37":7161,"_23":302,"_303":7162,"_174":7164},"971639e3d31a",[7163],"e74e17e","afford to lose a single, come on.",[7166,7168,7170],{"_37":7147,"_23":2441,"_2442":7167},"00:36:58.940",{"_37":7155,"_23":2441,"_2442":7169},"00:37:01.420",{"_37":7163,"_23":2441,"_2442":7171},"00:37:03.180",{"_37":7173,"_23":297,"_298":7174,"_306":7204,"_308":309},"e28bcfe22a88",[7175,7180,7183,7188,7191,7196,7199],{"_37":7176,"_23":302,"_303":7177,"_174":7179},"202cc6261e36",[7178],"07731e6","Health checks to your root domain?",{"_37":7181,"_23":302,"_303":7182,"_174":2425},"26820dd1b2e3",[],{"_37":7184,"_23":302,"_303":7185,"_174":7187},"30f812bae9ff",[7186],"195fe19","200s under a second.",{"_37":7189,"_23":302,"_303":7190,"_174":2425},"1f4da297b730",[],{"_37":7192,"_23":302,"_303":7193,"_174":7195},"388391038a07",[7194],"96cbb0b","I think we can afford to sample those.",{"_37":7197,"_23":302,"_303":7198,"_174":2425},"a0a8685e89c4",[],{"_37":7200,"_23":302,"_303":7201,"_174":7203},"f3cd12543692",[7202],"710b2e6","So I just feel like we're not very well educated and literate when it comes to sampling.",[7205,7207,7209,7211],{"_37":7178,"_23":2441,"_2442":7206},"00:37:05.740",{"_37":7186,"_23":2441,"_2442":7208},"00:37:07.660",{"_37":7194,"_23":2441,"_2442":7210},"00:37:09.420",{"_37":7202,"_23":2441,"_2442":7212},"00:37:11.180",{"_37":7214,"_23":297,"_298":7215,"_306":7224,"_308":693},"0987dcb30815",[7216,7220],{"_37":7217,"_23":302,"_303":7218,"_174":3743},"52e3c38f91e9",[7219,1281],"9fb628c",{"_37":7221,"_23":302,"_303":7222,"_174":7223},"f4bb1bd60ef0",[7219]," I will also say, sure, tracing might be expensive, but what's even more expensive is all of your lost business revenue because something bad happened and you can't figure out what happened.",[7225],{"_37":7219,"_23":2441,"_2442":7226},"00:37:16.039",{"_37":7228,"_23":297,"_298":7229,"_306":7246,"_308":309},"a7f52cc77968",[7230,7234,7238,7241],{"_37":7231,"_23":302,"_303":7232,"_174":3026},"8ee764099cb2",[7233,1281],"403102f",{"_37":7235,"_23":302,"_303":7236,"_174":7237},"837cf460efa4",[7233]," Well, thank you very much.",{"_37":7239,"_23":302,"_303":7240,"_174":2425},"9c4f6c44de44",[],{"_37":7242,"_23":302,"_303":7243,"_174":7245},"0673aaecb2d9",[7244],"ff93a92","This was very intriguing, and thank you for being patient and answering everything as per point.",[7247,7249],{"_37":7233,"_23":2441,"_2442":7248},"00:37:26.800",{"_37":7244,"_23":2441,"_2442":7250},"00:37:28.160",{"_23":7252,"_114":7253},"mux.video",{"_11":7254,"_13":7255,"_14":7256,"_23":7257,"_24":7258,"_7259":7260,"_7":7261,"_7321":7288,"_2337":7297,"_7322":7318,"_119":7323},"2026-07-22T20:18:55Z","af213bc6-23a5-4759-8afd-89f1a4c66353","g4rJ04viFnWjgGFaBUM3vb","mux.videoAsset","2026-07-22T20:45:43Z","assetId","01Cb747sNtnVMdb2tOUhNhwAwK7Pilm8O",{"_7262":7263,"_7264":7265,"_326":7266,"_7267":7268,"_20":7260,"_7269":7270,"_7271":7272,"_7273":7274,"_7275":7276,"_7277":7278,"_7279":7272,"_7280":7281,"_7284":7255,"_7285":7286,"_7291":7292,"_7296":7274,"_2337":7297,"_7298":7299,"_7317":7318,"_7319":7320},"aspect_ratio","16:9","created_at","1784752156",2262.470833,"encoding_tier","smart","ingest_type","on_demand_direct_upload","master_access","none","max_resolution_tier","1080p","max_stored_frame_rate",23.976,"max_stored_resolution","UHD","mp4_support","non_standard_input_reasons",{"_7282":7283},"video_resolution","3840x2160","passthrough","playback_ids",[7287],{"_20":7288,"_7289":7290},"A8P9KbEuYw5W1yEXuLo9FLk02kP224bEb","policy","public","progress",{"_7291":7293,"_7294":7295},100,"state","completed","resolution_tier","ready","tracks",[7300,7310],{"_326":7301,"_20":7302,"_7303":7304,"_7305":7306,"_7307":171,"_7308":7309},2262.426833,"UC801CQh3NldlObZrANY008UVNx5lb21pG","max_channel_layout","stereo","max_channels",2,"primary","type","audio",{"_326":7301,"_20":7311,"_7312":7276,"_7313":7314,"_7315":7316,"_7308":282},"KfoaHHXlGie02b5hrlXUOZdye8YyhphKKoff02SJlwEL8","max_frame_rate","max_height",1152,"max_width",2048,"upload_id","XjR2myuT9vv005oThown6VOcoCwIZ8nkp","video_quality","plus","playbackId","uploadId","https://stream.mux.com/A8P9KbEuYw5W1yEXuLo9FLk02kP224bEb","preview","pageNumber","actionData","errors"]

Agent-Led Trust and Safety

Speakers