[{"_1":2,"_5451":-5,"_5452":-5},"loaderData",{"_3":4,"_234":235},"root",{"_5":6,"_7":8,"_216":217},"ua","Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","data",{"_9":10,"_123":124,"_130":131},"footer",{"_11":12,"_17":18,"_84":85},"emailSubscription",{"_13":14,"_15":16},"description","Unlock success from technical pioneers in software infrastructure. We send the best strategies, tools, and industry updates directly to your inbox.","targetURL","https://share.hsforms.com/1_KYaR4lAQm-C8_9H7r4MIwqdoyu","footerColumns",[19,44,66],{"_20":21,"_22":23,"_24":25},"_key","30a87ef6e7c1","heading","About","items",[26,34,39],{"_20":27,"_28":29,"_30":31,"_32":33},"3dfef1283ab5","label","Portfolio","link","/portfolio","title","Navigate to portfolio page",{"_20":35,"_28":36,"_30":37,"_32":38},"42666d0b3987","Team","/team","Navigate to team page",{"_20":40,"_28":41,"_30":42,"_32":43},"398a9b50e214","News","/press","Navigate to press page",{"_20":45,"_22":46,"_24":47},"a584a6acf07b","Community",[48,52,57,62],{"_20":49,"_28":50,"_30":51,"_32":50},"4358699f4ca4","Events","/events",{"_20":53,"_28":54,"_30":55,"_32":56},"e2d5641ef7de","DevGuild","/devguild","Navigate to devguild page",{"_20":58,"_28":59,"_30":60,"_32":61},"feb038ce46a1","Jobs","/jobs","Portfolio Company Jobs",{"_20":63,"_28":64,"_30":65,"_32":64},"e22bfa6b2330","Inclusion","/community/diversity-and-inclusion",{"_20":67,"_22":68,"_24":69},"ea618aa34610","Library",[70,75,80],{"_20":71,"_28":72,"_30":73,"_32":74},"c82a64edc4dd","Blog","/library","Navigate to browse on library page",{"_20":76,"_28":77,"_30":78,"_32":79},"e1673433290e","Podcast Network","/library/podcasts","Navigate to podcasts page",{"_20":81,"_28":82,"_30":83,"_32":82},"96a97ceba3a2","DevToolsDigest","/devtoolsdigest","socialMedia",[86,99,105,111,117],{"_20":87,"_88":89,"_97":98},"ee025c96397c","icon",{"_90":91,"_92":93},"_type","mainImage","asset",{"_94":95,"_90":96},"_ref","image-9d618da0948299428b29e5d499af3fa5369a24d2-20x20-svg","reference","url","https://twitter.com/heavybit",{"_20":100,"_88":101,"_97":104},"5348b028f0d41c3dac62b3dd2b5531ee",{"_90":91,"_92":102},{"_94":103,"_90":96},"image-01bd92eb06873bdc30b1cb5f9c78b1ef5a9559d4-512x512-png","https://www.linkedin.com/company/heavybit",{"_20":106,"_88":107,"_97":110},"fae1cc0df5c6",{"_90":91,"_92":108},{"_94":109,"_90":96},"image-6c548d2e815d1fb3b3077c9b8be1ec251e1bc418-512x512-png","https://www.youtube.com/c/heavybit",{"_20":112,"_88":113,"_97":116},"a100f6448036",{"_90":91,"_92":114},{"_94":115,"_90":96},"image-b38b7ee9ccd05f61ca43d5b29fe7058f822f87eb-512x512-webp","https://www.instagram.com/heavybitindustries/",{"_20":118,"_88":119,"_97":122},"639729864c2d",{"_90":91,"_92":120},{"_94":121,"_90":96},"image-48ac9473e30341c4bee9f6634b77b6f8f1dd6b2f-849x980-webp","https://www.tiktok.com/@heavybitindustries","globalConfig",{"_125":126},"OGimage",{"_90":127,"_92":128},"image",{"_94":129,"_90":96},"image-32fdfae4fd13b8f9a3c5bf6b13a635068cb0ecbb-1332x529-png","header",{"_132":133,"_144":145},"announcementBar",{"_134":135,"_136":-5,"_137":138,"_139":140,"_141":142,"_97":143},"buttonText","RSVP","end","show",true,"start","2026-08-03T18:37:28.492Z","text","🚨 Almost sold out [Sep 24] Long-Horizon Stack conference feat Hex, Baseten, and more","https://luma.com/long-horizon-stack?utm_source=heavybit_site&utm_medium=banner","headerNavLinks",[146,163,173,186,212],{"_20":147,"_148":149,"_28":23,"_30":159,"_160":161,"_32":162},"a8d641babed2","dropdown",[150,152,157],{"_20":151,"_28":36,"_30":37,"_32":38},"964627410473",{"_20":153,"_28":154,"_30":155,"_32":156},"3d279775d73b","FAQ","/faq","Navigate to FAQ page",{"_20":158,"_28":41,"_30":42,"_32":43},"84e68f9f5c33","/about","mobileOnly",false,"Navigate to about page",{"_20":164,"_148":165,"_28":29,"_30":31,"_160":161,"_32":33},"90b68b861c06",[166,171],{"_20":167,"_28":168,"_30":169,"_32":170},"d481fcbbd3af","Spotlights","/portfolio/spotlights","Navigate to spotlights page",{"_20":172,"_28":59,"_30":60,"_32":61},"e50fa43ad2e7",{"_20":174,"_148":175,"_28":46,"_30":184,"_160":161,"_32":185},"716b76469486",[176,178,182],{"_20":177,"_28":50,"_30":51,"_32":50},"3a177efc4649",{"_20":179,"_28":180,"_30":181,"_32":180},"1fb7c9c465d0","Advisors","/community/advisors",{"_20":183,"_28":54,"_30":55,"_32":56},"23077fca0e2f","/community","Navigate to community page",{"_20":187,"_148":188,"_28":68,"_30":73,"_160":161,"_32":211},"4bec0dd665d4",[189,194,199,201,205,209],{"_20":190,"_28":191,"_30":192,"_32":193},"63e48d609548","Articles","/library?filter=articles","Read educational articles in Heavybit's Library",{"_20":195,"_28":196,"_30":197,"_32":198},"6526c69f5260","Videos","/library?filter=videos","Watch educational videos in Heavybit's Library",{"_20":200,"_28":77,"_30":78,"_32":79},"1f3fb936f3b9",{"_20":202,"_28":203,"_30":204,"_32":203},"ac06751f0372","Collections","/library/collections",{"_20":206,"_28":207,"_30":208,"_32":207},"2ead3bd14715","Reports","/reports",{"_20":210,"_28":82,"_30":83,"_32":82},"66773779036c","Navigate to library page",{"_20":213,"_148":-5,"_28":214,"_30":42,"_160":138,"_32":215},"3702498a502c","Contact","Navigate to contact page","ENV",{"_218":219,"_220":221,"_222":223,"_224":225,"_226":227,"_228":229,"_230":231,"_232":219,"_233":219},"SANITY_DATASET","production","ALGOLIA_INDEX","heavybit-library","ALGOLIA_APPLICATION_ID","0QCKBC1FMK","ALGOLIA_SITEWIDE_INDEX","heavybit-sitewide","ALGOLIA_JOBS_INDEX","heavybit-jobs","MUX_ENV_KEY","dRI98bL8ze03gYDL6-Ogle2Js","ALGOLIA_SEARCH_API_KEY","7733fdd199ec7af7ee2923ec39c3f036","MODE","VITE_CONTEXT","routes/library.video.$slug",{"_7":236,"_5449":161,"_5450":-5,"_1084":2928},{"_237":238},"routeData",{"_239":240,"_241":242,"_243":244,"_90":245,"_246":247,"_248":138,"_249":-5,"_250":251,"_252":253,"_285":286,"_287":288,"_297":298,"_281":299,"_2897":-5,"_2898":2899,"_2906":2907,"_1082":2916,"_2917":2918,"_2925":-5,"_1084":2926,"_2929":-5,"_2930":2931,"_3035":2916,"_3014":3036,"_32":3057,"_3058":3059,"_245":5382},"_createdAt","2026-09-23T19:32:16Z","_id","2bfdda73-ffe5-429b-aa9e-0b12835c788d","_rev","mp6XYBI2Ev1Nw6ci78fVkh","video","_updatedAt","2026-09-23T23:00:40Z","afterEvent","agenda","collections",[],"cta",{"_239":254,"_241":255,"_243":256,"_90":252,"_246":257,"_258":259,"_260":138,"_261":262,"_13":263,"_279":-5,"_28":280,"_281":161,"_30":16,"_282":283,"_284":-5,"_32":283},"2023-04-21T21:45:14Z","7e3a37a4-a56d-440a-85a2-de52431c5fde","3wgj8X5vear6qceF9bUPYZ","2024-11-12T22:18:04Z","ctaType","email","decoration","decorationPosition","top_left",[264],{"_20":265,"_90":266,"_267":268,"_275":276,"_277":278},"481da9627862","block","children",[269],{"_20":270,"_90":271,"_272":273,"_141":274},"4eeeef09e3560","span","marks",[],"You don’t have to build on your own. We help you stay ahead with the hottest resources, latest product updates, and top job opportunities from the community. Don’t miss out—subscribe now.","markDefs",[],"style","normal","file","Subscribe Now","library","name","Subscribe to Heavybit Updates","override","endDate","2022-11-17T18:25:00.000Z","excerpt",[289],{"_20":290,"_90":266,"_267":291,"_275":296,"_277":278},"ex1mandi",[292],{"_20":293,"_90":271,"_272":294,"_141":295},"ex1mandi0",[],"PagerDuty's Mandi Walls discusses the future of human involvement in observability, dependency management, and incident response. Learn how dev teams can minimize human error by up-leveling their tools and best practices with better technology and better training.",[],"filmedOn","2022-11-17",{"_24":300,"_1123":1124},[301,1090,1107],{"_302":303,"_1077":1078,"_127":1079,"_1080":1081,"_1082":1083,"_1084":1085,"_1086":-5,"_32":1087,"_1088":1089},"content",[304,313,355,370,411,428,440,452,461,469,473,521,529,542,554,566,575,583,591,600,608,616,624,628,643,655,667,679,687,695,703,711,719,727,735,739,754,766,778,786,794,802,810,818,826,834,842,850,854,891,903,915,927,935,943,951,959,967,975,983,991,1010,1018,1037,1045,1056,1066],{"_20":305,"_90":266,"_267":306,"_275":311,"_277":312},"eb11bc467dae",[307],{"_20":308,"_90":271,"_272":309,"_141":310},"eb3a12ad81230",[],"How Does Generative AI Work With Incident Response?",[],"h2",{"_20":314,"_90":266,"_267":315,"_275":347,"_277":278},"eefe6d5307c6",[316,320,325,329,334,338,343],{"_20":317,"_90":271,"_272":318,"_141":319},"3369fc8096020",[],"Software continues to eat the world, as more dev teams depend on third-party microservices as their daily infrastructure. Which means that outages are more common and costly than ever, costing upwards of ",{"_20":321,"_90":271,"_272":322,"_141":324},"3369fc8096021",[323],"390b5989751f","$100K",{"_20":326,"_90":271,"_272":327,"_141":328},"3369fc8096022",[]," per incident, and that successful ",{"_20":330,"_90":271,"_272":331,"_141":333},"3369fc8096023",[332],"ecb6c9b7ae9b","incident response workflows",{"_20":335,"_90":271,"_272":336,"_141":337},"3369fc8096024",[]," are more important than ever as well. What about the wondrous wave of artificial intelligence products from Microsoft, GitHub, and OpenAI? Reports suggest generative AI tools boost ",{"_20":339,"_90":271,"_272":340,"_141":342},"3369fc8096025",[341],"f7e0a6dbbe49","developer productivity",{"_20":344,"_90":271,"_272":345,"_141":346},"3369fc8096026",[],", reducing bottlenecks by streamlining the process of coding with code snippets, search, and summaries. Could generative AI be a breakthrough for IT operations in managing incidents that helps stem this rising tide?",[348,351,353],{"_20":323,"_90":30,"_349":350},"href","https://llcbuddy.com/data/incident-management-statistics/",{"_20":332,"_90":30,"_349":352},"https://www.heavybit.com/devguild/incident-response",{"_20":341,"_90":30,"_349":354},"https://arxiv.org/abs/2302.06590",{"_20":356,"_90":266,"_267":357,"_275":367,"_277":278},"a723646c9047",[358,363],{"_20":359,"_90":271,"_272":360,"_141":362},"daeb999233620",[361],"bbc8f76bd312","62%",{"_20":364,"_90":271,"_272":365,"_141":366},"daeb999233621",[]," of the general populace is “concerned” about modern AI, and 86% “believe AI could accidentally cause a catastrophic event.” So where, if at all, does GenAI fit into incident reponse and day-to-day site reliability engineering? In this article, we consulted with a panel of site incident management veterans with more than 40 years of collective experience. As one of our experts put it, GenAI is good at “confidently delivering text that is pleasant to read, but not always complete, or correct.” As another suggested, GenAI is “not good at making decisions for you...or [emulating other people’s] expertise.”",[368],{"_20":361,"_90":30,"_349":369},"https://www.axios.com/2023/08/09/ai-voters-trust-government-regulation",{"_20":371,"_90":266,"_267":372,"_275":404,"_277":278},"6fbe7f5e506a",[373,377,382,386,391,395,400],{"_20":374,"_90":271,"_272":375,"_141":376},"a1521bc4c9840",[],"Below, our panel explores known GenAI vulnerabilities in ",{"_20":378,"_90":271,"_272":379,"_141":381},"a1521bc4c9841",[380],"ce8e92ee7c73","security",{"_20":383,"_90":271,"_272":384,"_141":385},"a1521bc4c9842",[]," and ",{"_20":387,"_90":271,"_272":388,"_141":390},"a1521bc4c9843",[389],"759c619acc17","privacy",{"_20":392,"_90":271,"_272":393,"_141":394},"a1521bc4c9844",[],", not to mention its well-documented ",{"_20":396,"_90":271,"_272":397,"_141":399},"a1521bc4c9845",[398],"de9b54f26917","hallucinations",{"_20":401,"_90":271,"_272":402,"_141":403},"a1521bc4c9846",[],", and the need for ad hoc collaboration and consequential decisions in IM. Is there an eventual future for AI-powered incident commanders, or will teams always need that proverbial human in the loop? Our panel discusses:",[405,407,409],{"_20":380,"_90":30,"_349":406},"https://www.wired.com/story/generative-ai-prompt-injection-hacking/",{"_20":389,"_90":30,"_349":408},"https://iapp.org/news/a/data-protection-issues-for-employers-to-consider-when-using-generative-ai/",{"_20":398,"_90":30,"_349":410},"https://www.wired.com/story/fast-forward-chatbot-hallucinations-are-poisoning-web-search/",{"_20":412,"_90":266,"_267":413,"_423":424,"_425":426,"_275":427,"_277":278},"4633d3d98974",[414,419],{"_20":415,"_90":271,"_272":416,"_141":418},"262b6d9995b10",[417],"strong","The Strengths and Weaknesses of GenAI for IR and SRE:",{"_20":420,"_90":271,"_272":421,"_141":422},"846d30174cca",[]," Which capabilities of GenAI are a strong fit for day-to-day incident management.","level",1,"listItem","bullet",[],{"_20":429,"_90":266,"_267":430,"_423":424,"_425":426,"_275":439,"_277":278},"baf937bd97b7",[431,435],{"_20":432,"_90":271,"_272":433,"_141":434},"964e75e826ac0",[417],"How GenAI Will Affect the DevOps and SRE Professions:",{"_20":436,"_90":271,"_272":437,"_141":438},"899855b21399",[]," How GenAI will impact professionals who work on both product and the operational side of product.",[],{"_20":441,"_90":266,"_267":442,"_423":424,"_425":426,"_275":451,"_277":278},"840519f7d8bb",[443,447],{"_20":444,"_90":271,"_272":445,"_141":446},"3a1dc2a430f90",[417],"How GenAI Will Ultimately Affect Dev:",{"_20":448,"_90":271,"_272":449,"_141":450},"ec1f1456ce11",[]," Our panel also weighed in with their thoughts on how GenAI will impact the general business of software development.",[],{"_20":453,"_90":266,"_267":454,"_275":460,"_277":278},"a3306c00fe1c",[455],{"_20":456,"_90":271,"_272":457,"_141":459},"c73bcfeb284d0",[458,417],"em","Disclaimer: While the panelists interviewed here hail from companies such as Jeli, Amazon, and PagerDuty, the views expressed below are those of the individual panelists and do not reflect the views of their employers.",[],{"_20":462,"_90":266,"_267":463,"_275":468,"_277":312},"217fdd6806f9",[464],{"_20":465,"_90":271,"_272":466,"_141":467},"674d8be3068b0",[],"How to Utilize GenAI Within Incident Management Platforms",[],{"_20":470,"_90":127,"_92":471},"4e587cfaeee2",{"_94":472,"_90":96},"image-eeaae91fe0006fb9d98aced3b6ef71737f890e98-200x200-jpg",{"_20":474,"_90":266,"_267":475,"_275":512,"_277":278},"b8d85d2aec9a",[476,481,485,490,494,499,503,508],{"_20":477,"_90":271,"_272":478,"_141":480},"8ecc46e9ad880",[479],"f6f47f4e8b83","Nora Jones",{"_20":482,"_90":271,"_272":483,"_141":484},"8ecc46e9ad881",[]," is an incident response veteran who led IR teams at Slack and Netflix before founding the developer-first IR startup ",{"_20":486,"_90":271,"_272":487,"_141":489},"8ecc46e9ad882",[488],"dddcfa115e70","Jeli",{"_20":491,"_90":271,"_272":492,"_141":493},"8ecc46e9ad883",[],". She’s also a co-founder of the IR community ",{"_20":495,"_90":271,"_272":496,"_141":498},"8ecc46e9ad884",[497],"6d13b6e1f768","LFI",{"_20":500,"_90":271,"_272":501,"_141":502},"8ecc46e9ad885",[],". Her company has implemented ",{"_20":504,"_90":271,"_272":505,"_141":507},"8ecc46e9ad886",[506],"22351c07f276","GenAI directly into its platform",{"_20":509,"_90":271,"_272":510,"_141":511},"8ecc46e9ad887",[],", utilizing natural language to rapidly spin up shareable incident reports to quickly get team members up to speed as well as to draft overarching narratives based on different touch points over an incident’s life (including detection, diagnosis, and repair moments directly from chat logs, which the platform has already been fully annotating). Jones believes that GenAI’s ability to accelerate incident logging is a valuable tool and makes GenAI worth considering as another member of the team–but not as the overarching decision maker:",[513,515,517,519],{"_20":479,"_90":30,"_349":514},"https://www.linkedin.com/in/norajones1/",{"_20":488,"_90":30,"_349":516},"https://www.jeli.io",{"_20":497,"_90":30,"_349":518},"https://www.learningfromincidents.io/about",{"_20":506,"_90":30,"_349":520},"https://techcrunch.com/2023/08/10/jeli-is-bringing-generative-ai-to-incident-report-analysis/",{"_20":522,"_90":266,"_267":523,"_423":424,"_425":426,"_275":528,"_277":278},"f10746e0fce9",[524],{"_20":525,"_90":271,"_272":526,"_141":527},"ff5fc5ab5b310",[417],"GenAI Has [At Least] Two Key Strengths for IR:",[],{"_20":530,"_90":266,"_267":531,"_423":540,"_425":426,"_275":541,"_277":278},"ddb2984d384d",[532,536],{"_20":533,"_90":271,"_272":534,"_141":535},"43c847de77270",[458],"Spinning Up Summaries to Catch Up Teams:",{"_20":537,"_90":271,"_272":538,"_141":539},"0487ba908ddb",[]," As incidents are happening, GenAI’s ability to quickly spin up content can be useful to provide instantaneous summaries of incidents to relevant team members to keep everyone in the loop as things happen (rather than pulling people sideways by requiring them to drop everything and hunt down the details).",2,[],{"_20":543,"_90":266,"_267":544,"_423":540,"_425":426,"_275":553,"_277":278},"dcc615e16ae7",[545,549],{"_20":546,"_90":271,"_272":547,"_141":548},"6ae8b15d3bc80",[458],"Incident Analysis:",{"_20":550,"_90":271,"_272":551,"_141":552},"1ab9f47e95f3",[]," Post-incident, GenAI can help teams uncover and compile context around incidents to create richer, more-valuable post-mortems by collecting insights and notes across the incident lifecycle.",[],{"_20":555,"_90":266,"_267":556,"_423":424,"_425":426,"_275":565,"_277":278},"ca7b27531b17",[557,561],{"_20":558,"_90":271,"_272":559,"_141":560},"54895b2ede500",[417],"Why GenAI May Not Be Taking the Incident Commander Chair Anytime Soon:",{"_20":562,"_90":271,"_272":563,"_141":564},"254cde6562b4",[]," Incident response continues to be a field full of unknowns and exceptions–not exactly a good fit for tools that are built largely to pattern-match based on past data. Fully AI-run incident remediation is unlikely to be “a thing” anytime soon.",[],{"_20":567,"_90":266,"_267":568,"_275":573,"_277":574},"1c027b6bf0c8",[569],{"_20":570,"_90":271,"_272":571,"_141":572},"0f8b61910aa40",[],"Discussion: Where GenAI Makes Sense for IR with Nora Jones",[],"h3",{"_20":576,"_90":266,"_267":577,"_275":582,"_277":278},"a7e405120178",[578],{"_20":579,"_90":271,"_272":580,"_141":581},"efeda41062f10",[],"In addition to offering the above observations, Jones opines that modern SRE can optimize their GenAI usage by recognizing its assorted strengths and weaknesses. Specifically, GenAI was not built on all-knowing, benevolent algorithms developed solely to decide how to manage important decisions, such as issue resolution steps. GenAI algorithms, at least for now, are optimized to generate and summarize content.",[],{"_20":584,"_90":266,"_267":585,"_275":590,"_277":278},"e8b281476205",[586],{"_20":587,"_90":271,"_272":588,"_141":589},"b60fdb20a2ed0",[],"“I don't think you trust GenAI to be an ‘expert’ in anything. It's not good at making decisions for you. It's not good at [emulating other people’s] expertise. It is good at summarizing pieces of information. But just because it's easy to use and easy to ‘sprinkle’ AI on anything you're doing doesn't mean you should. I would really encourage folks that are starting to play around with it to understand actually how it works.”",[],{"_20":592,"_90":266,"_267":593,"_275":598,"_277":599},"a0b6c6257711",[594],{"_20":595,"_90":271,"_272":596,"_141":597},"86bdc7b348510",[],"I think what we really want to do is use AI to get people more curious about what's happening in their incidents.” -Nora Jones, Founder / Jeli",[],"blockquote",{"_20":601,"_90":266,"_267":602,"_275":607,"_277":278},"8aaae63f334a",[603],{"_20":604,"_90":271,"_272":605,"_141":606},"47b3e090cbcd0",[],"“Ultimately, I think what we really want to do is use AI to get people more curious about what's happening in their incidents. I’ve always believed that if you learn how an incident actually happens, you'll be better off in the future. You can be more proactive about your incidents, resolving some of them more quickly, getting the right people in the room more quickly,” Jones explains. “Where AI seems really interesting for incident management is when we can use it to bubble up some of those interesting learnings, which then gets people investigating the incident...and gets people a little bit more curious about how it unfolded in the first place.”",[],{"_20":609,"_90":266,"_267":610,"_275":615,"_277":278},"e5b372d584b2",[611],{"_20":612,"_90":271,"_272":613,"_141":614},"715b2d8c8e2d0",[],"Jones suggests that artificial intelligence provides opportunities to help both professional SREs and developers of all stripes. “I think GenAI will bring big changes in the field of incident management in terms of how incidents get communicated to stakeholders that are impacted by those incidents. But for developers in general, I think there’s an opportunity for them to use AI to accelerate their processes and help them get curious about other areas.” In the future, Jones suggests the possibility of AIs trained on large amounts of previous incident data being helpful in taking a more-proactive approach. “I don't think generative AI is going to fix the incidents for you, but I think eventually, it might help point you to previous incidents that look like the one that you're solving right now. But as far as I know, it can't get people to talk to each other. And I don’t see it being a magic box for auto-remediation anytime soon.”",[],{"_20":617,"_90":266,"_267":618,"_275":623,"_277":312},"3a3e9cbcc658",[619],{"_20":620,"_90":271,"_272":621,"_141":622},"6f353338e7870",[],"Where GenAI Impacts DevOps and the Future of Software Dev",[],{"_20":625,"_90":127,"_92":626},"19bdb282d922",{"_94":627,"_90":96},"image-c5983aa5c027d4ed93b4206b4c981083c3fcfdc4-200x200-jpg",{"_20":629,"_90":266,"_267":630,"_275":640,"_277":278},"46e856c55e1e",[631,636],{"_20":632,"_90":271,"_272":633,"_141":635},"1bdeba82b0320",[634],"19704df303f7","Jeremy Edberg",{"_20":637,"_90":271,"_272":638,"_141":639},"1bdeba82b0321",[]," is a longtime DevOps expert who currently helps lead Amazon’s Alexa Operational Excellence Team, but has done tours of duty at leading tech companies including eBay, Reddit, and Netflix–where he was a founding member of Netflix’s SRE team.",[641],{"_20":634,"_90":30,"_349":642},"https://www.linkedin.com/in/jedberg/",{"_20":644,"_90":266,"_267":645,"_423":424,"_425":426,"_275":654,"_277":278},"1dbb3acff7c9",[646,650],{"_20":647,"_90":271,"_272":648,"_141":649},"c36702b414dd0",[417],"Large Language Models May Be the Next Major Evolutionary Step in Human-Computer Interfaces:",{"_20":651,"_90":271,"_272":652,"_141":653},"e714dd32ab8f",[]," Whether GenAI achieves the Nirvana-like goal of artificial general intelligence (AGI), prompt-based LLM chatbots may well represent the next step in the way humans interact with technology, as they effectively help computers take a big step toward being able to understand human language.",[],{"_20":656,"_90":266,"_267":657,"_423":424,"_425":426,"_275":666,"_277":278},"db8e7bfd00e0",[658,662],{"_20":659,"_90":271,"_272":660,"_141":661},"cdb7c55bf0fa0",[417],"We’re Not Yet at a Point Where LLMs Can Credibly Recommend Remediation Steps:",{"_20":663,"_90":271,"_272":664,"_141":665},"cbd72b4a4f1c",[]," Right now, human users will trust their monitoring and alerting systems after those systems have proven themselves reliable, even going as far as allowing them to take automatic actions. But we are not there yet with LLMs. Right now, the best we can hope for is LLMs trained on previous incidents and patterns producing one (or a few) possible remediation steps and having a human select the best course of action. Over time, if the LLMs prove to produce the correct course of action in almost every case, they will be trusted to work autonomously.",[],{"_20":668,"_90":266,"_267":669,"_423":424,"_425":426,"_275":678,"_277":278},"ece9d2eadad3",[670,674],{"_20":671,"_90":271,"_272":672,"_141":673},"6dce022412990",[417],"Potential Career Evolution for DevOps:",{"_20":675,"_90":271,"_272":676,"_141":677},"7f5d7561f671",[]," Language Model Operations in AIOps?: DevOps, being generally tasked with the maintenance and caretaking of infrastructure, may also inherit the care and feeding of language models. As LLMs come to represent more-significant components in infrastructure, organizations will need people who understand distributed computing, machine learning inference, managing GPUs and CPUs next to each other, storage, and other maintenance considerations. Could there be a point where entire careers are focused on monitoring AI models, updating them, and making sure the models are getting the right inputs and appropriately learning new things?",[],{"_20":680,"_90":266,"_267":681,"_275":686,"_277":574},"cd2a80857355",[682],{"_20":683,"_90":271,"_272":684,"_141":685},"cfe3e34a31410",[],"Discussion: What the Future Looks Like for Devs Using GenAI with Jeremy Edberg",[],{"_20":688,"_90":266,"_267":689,"_275":694,"_277":278},"8e3e2b6a453d",[690],{"_20":691,"_90":271,"_272":692,"_141":693},"4df59a9902ef0",[],"“Right now, GenAI is something of an advisory tool. We're not to the point where we trust it enough to take the actions based on what it says,” Edberg explains. “In some ways, you could compare some of GenAI’s use cases to those of what monitoring used to be–or how things are when you're first starting out because you don't know that your monitoring and alerting are correct.”",[],{"_20":696,"_90":266,"_267":697,"_275":702,"_277":278},"e00983e08adb",[698],{"_20":699,"_90":271,"_272":700,"_141":701},"529ff5e69a680",[],"“As an advisory tool, GenAI can tell you, ‘Hey, something is probably wrong here, and you should look into it,’ but there still needs to be a human in the loop there. Eventually, we'll get to the point where we can take the human out of the loop for the easy stuff...maybe. The thing is, better monitoring and alerting have already made changes to the way we operate. And LLMs will definitely make changes to the way we operate, but there'll be new challenges instead. Overall, I don’t think GenAI will eliminate DevOps jobs. It will, hopefully, make DevOps practices–and practitioners–much more efficient. So maybe in that regard, it would actually generate some net-new jobs.”",[],{"_20":704,"_90":266,"_267":705,"_275":710,"_277":599},"9c33c24fa7d1",[706],{"_20":707,"_90":271,"_272":708,"_141":709},"8acf5425396c",[],"In the future, if you are good at logic and want to learn how to reason about computer systems, [software engineering will still be] a great place to be.” -Jeremy Edberg, Principal Engineer / Amazon",[],{"_20":712,"_90":266,"_267":713,"_275":718,"_277":278},"9716642e50c2",[714],{"_20":715,"_90":271,"_272":716,"_141":717},"d4685a3f43b90",[],"What effect will GenAI have on day-to-day dev workflows, or on software engineers as a profession? “If I were addressing a class of junior developers, I’d tell them, ‘GenAI is going to be a tool that will drastically speed up your development process, but it will not replace you.’ Not yet, anyway,” says Edberg. “Could it lower the barrier to entry for getting a job as an engineer? I could definitely see a situation where people–who hadn't considered this type of career before, maybe because they weren't interested in learning the details of coding syntax, for example, but are still good at general reasoning—might choose engineering now instead of business, law, or some other path.\"",[],{"_20":720,"_90":266,"_267":721,"_275":726,"_277":278},"e87e791a1399",[722],{"_20":723,"_90":271,"_272":724,"_141":725},"a21774cafd06",[],"\"Somebody who has these reasoning, logic, and analytical skills might be more interested now because the ‘hard parts’ are taken care of, the syntax, the math, that kind of stuff. In the future, I think if you are good at reasoning, good at logic, and want to learn how to reason about computer systems, it's still a great place to be. If jobs do end up going away, they will be the ‘I've learned enough to know how to write code, and I'm going to spend most of my days writing basic, boilerplate' stuff,’ because the LLMs will take care of that.”",[],{"_20":728,"_90":266,"_267":729,"_275":734,"_277":312},"0b261b3fa906",[730],{"_20":731,"_90":271,"_272":732,"_141":733},"ac4e0d817c130",[],"Deferring Low-Level Tasks to AI so Humans Can Focus on Strategy",[],{"_20":736,"_90":127,"_92":737},"02d5b29278d2",{"_94":738,"_90":96},"image-c87a2249f5a3a3696fc35defa73409803afeb2a6-200x200-jpg",{"_20":740,"_90":266,"_267":741,"_275":751,"_277":278},"4f595a74a5d3",[742,747],{"_20":743,"_90":271,"_272":744,"_141":746},"c66b562052680",[745],"dd7cc970b104","Mandi Walls",{"_20":748,"_90":271,"_272":749,"_141":750},"c66b562052681",[]," is a long-tenured developer advocate who has been advocating for AI and automated solutions to help make SRE and DevOps teams more productive for some time. She’s currently building communities of highly engaged developers at PagerDuty and has also served tours of duty at Chef and AOL.",[752],{"_20":745,"_90":30,"_349":753},"https://www.linkedin.com/in/mandiwalls/",{"_20":755,"_90":266,"_267":756,"_423":424,"_425":426,"_275":765,"_277":278},"dbdf2d9904ed",[757,761],{"_20":758,"_90":271,"_272":759,"_141":760},"3508a9164e9a0",[417],"We’re a Ways Off From Fully AI-Powered L1 Responders:",{"_20":762,"_90":271,"_272":763,"_141":764},"8e4cd4bec5fa",[]," The sheer amount of training datasets from real, recorded incidents across a single organization required to stand up completely AI-powered Level One incident responders just doesn’t exist yet. The closest path to something similar to this in the future might be from large orgs running similar services on a very similar platform with similar runtimes, which would presumably generate incidents of a similar character with similar symptoms.",[],{"_20":767,"_90":266,"_267":768,"_423":424,"_425":426,"_275":777,"_277":278},"f793b3f245cc",[769,773],{"_20":770,"_90":271,"_272":771,"_141":772},"b608858530530",[417],"The Most Immediate AI Opportunity in SRE May Be for Low-Level Remediation Tasks:",{"_20":774,"_90":271,"_272":775,"_141":776},"3b15706ef23a",[]," Generative AI might be able to make the most immediate impact if it were trained to manage the low-level hiccups and false alarms that do not require extensive triage, and which experienced SREs resolve in minutes, anyway. There’s also strategic value in developing AI tools that can manage most low-level remediation tasks–because such tools would free up veteran SRE teams to focus more of their time and undivided attention on higher-priority incidents and post-mortems.",[],{"_20":779,"_90":266,"_267":780,"_275":785,"_277":574},"9e82dab8ea81",[781],{"_20":782,"_90":271,"_272":783,"_141":784},"3721f0e7df3b0",[],"Discussion: The Division of Labor Between Human SRE and AI with Mandi Walls",[],{"_20":787,"_90":266,"_267":788,"_275":793,"_277":278},"85f907947ac3",[789],{"_20":790,"_90":271,"_272":791,"_141":792},"f6c1d20b7fed0",[],"Walls suggests that the immediate value of generative AI in SRE might come from spinning up documentation and after-action reports, but also in a variety of other areas. “Our incident response process includes Zoom calls, recordings, transcripts, and Slack channels, along with charts and graphs and many other kinds of data and artifacts...it’s a slog. So there’s value in letting AI generate all the components and artifacts we need.”",[],{"_20":795,"_90":266,"_267":796,"_275":801,"_277":278},"56c3d402e563",[797],{"_20":798,"_90":271,"_272":799,"_141":800},"bf8a22f3dea90",[],"Regarding how GenAI and its associated tools, such as code generators, could affect the profession of development as a whole, Walls sees opportunities in many areas for GenAI to be valuable. “Stuff like coding assistants are super interesting. Some of it is really clever and is already doing a really good job for folks doing some of that work. But as someone who uses a lot of products, I'm hoping for improved documentation—and API documentation in particular–that developers don't have to write themselves. It’d be good to see tools improve enough to automatically generate all that stuff and make it more useful.”",[],{"_20":803,"_90":266,"_267":804,"_275":809,"_277":278},"bd27f196045a",[805],{"_20":806,"_90":271,"_272":807,"_141":808},"b292935276510",[],"Walls suggests that testing may be another area of opportunity for GenAI to improve development pipelines. “Another use case would be generating tests. I think there's a lot of knowledge already in that space, especially over the last 10 years as that whole practice has become more automated, and maybe it will become even more so. So maybe the work of test engineers will move more towards creating better harnesses and doing performance monitoring on the testing process rather than anything like writing a tool. Also, developers have artifact repositories. There's all this stuff that has to run together really closely. And keeping that all in line plus maintaining changes that come in from the vendors would definitely be helped by additional tooling that's a little bit smarter than what we have right now.”",[],{"_20":811,"_90":266,"_267":812,"_275":817,"_277":599},"5ccc0fa9d705",[813],{"_20":814,"_90":271,"_272":815,"_141":816},"cbcb5a31dc4c",[],"For positions like SRE that are usually more directly integrated with an engineering practice, they'll see more benefits from coding tools, which could start to learn as much about infrastructure tooling as they do about regular languages and runtime-application code.” -Mandi Walls, Developer Advocate / PagerDuty",[],{"_20":819,"_90":266,"_267":820,"_275":825,"_277":278},"9817bac2933f",[821],{"_20":822,"_90":271,"_272":823,"_141":824},"d41f4287d5f90",[],"“I’m thinking about something that’s even a level up from Dependabot–which right now, will send you an email that says, ‘Hey, here's this thing that needs to be updated.’ It would be really useful to see this kind of use case broadening out to alert you that your vendor is doing an upgrade. An alert that could tell you, ‘Here's what we recommend for your specific use case.’ For example, if your cloud provider is turning off instances of your level, here's where you need to migrate...and then starting to do that work for you without having to really intervene.”",[],{"_20":827,"_90":266,"_267":828,"_275":833,"_277":278},"dbe2f688d98a",[829],{"_20":830,"_90":271,"_272":831,"_141":832},"fcaeed8a24910",[],"On how GenAI may affect the business of DevOps workflows, Walls is less eager to make predictions due to variance across orgs. “DevOps jobs are different in every organization. So it's possible that GenAI tools, such as code generators, could make a difference because, in some places, a lot of those folks are writing more code. However, in other places, they're just working more in advisory positions. And then, some orgs take more of a build-and-release approach. So it's hard to stay. At the macro level, if there's going to be a deep change in what it means to work in DevOps due to generative AI, well...I'm not sure there's enough of a consensus of what a DevOps engineer should be doing to be able to say that.”",[],{"_20":835,"_90":266,"_267":836,"_275":841,"_277":278},"10afd7e078b1",[837],{"_20":838,"_90":271,"_272":839,"_141":840},"598db06fac460",[],"On how GenAI will affect the business of SRE, Walls is significantly more bullish. “I think for those positions like SRE that are usually more directly integrated with an engineering practice, I think they'll see more benefits from coding tools and things like that...which potentially could start to learn as much about infrastructure tooling as they do about regular languages and runtime–application code versus infrastructure code. I’d like to see GenAI help SRE teams push forward along their golden path because so much of their infrastructure is hopefully managed as code.” And in the same way that developer technology such as containers expanded into open source with Kubernetes, there may be opportunities to see open source contribute to generative coding assistants for SREs. “I think these teams will also benefit from those same code generation tools–but they may be in Terraform or Pulumi, rather than Python/Elixir/Go/Rust.”",[],{"_20":843,"_90":266,"_267":844,"_275":849,"_277":312},"46adbb984ffc",[845],{"_20":846,"_90":271,"_272":847,"_141":848},"cde89f849f3e0",[],"Why Humans in the Loop May Always Be Needed in SRE",[],{"_20":851,"_90":127,"_92":852},"ff61903f5fab",{"_94":853,"_90":96},"image-28240d630e8fb1c07da810841e34f41fd91de686-200x200-jpg",{"_20":855,"_90":266,"_267":856,"_275":884,"_277":278},"daaaabb3cab5",[857,862,866,871,875,880],{"_20":858,"_90":271,"_272":859,"_141":861},"ccb0bd7bf47a0",[860],"52f00b83da13","Brent Chapman",{"_20":863,"_90":271,"_272":864,"_141":865},"ccb0bd7bf47a1",[]," is a pioneer in what is now known as modern SRE. Throughout his career in technology, he has always also worked as a volunteer in public safety and emergency services, starting as a search-and-rescue pilot and incident commander for air search and rescue. He applied the principles he learned in emergency services to his tenure at Google, where developed the company’s internal ",{"_20":867,"_90":271,"_272":868,"_141":870},"ccb0bd7bf47a2",[869],"616e5552cb59","Incident Management at Google (IMAG)",{"_20":872,"_90":271,"_272":873,"_141":874},"ccb0bd7bf47a3",[]," practice, and later, brought similar foundational practices to Slack. He currently runs the incident management consultancy ",{"_20":876,"_90":271,"_272":877,"_141":879},"ccb0bd7bf47a4",[878],"e66bb0e4a105","Great Circle Associates",{"_20":881,"_90":271,"_272":882,"_141":883},"ccb0bd7bf47a5",[],".",[885,887,889],{"_20":860,"_90":30,"_349":886},"https://www.linkedin.com/in/brentchapman/",{"_20":869,"_90":30,"_349":888},"https://sre.google/workbook/incident-response/",{"_20":878,"_90":30,"_349":890},"https://greatcircle.com/",{"_20":892,"_90":266,"_267":893,"_423":424,"_425":426,"_275":902,"_277":278},"56435edba8f3",[894,898],{"_20":895,"_90":271,"_272":896,"_141":897},"aeb7c676fb5d0",[417],"Large Language Models Have the “Natural Language” Part Down, But May Still Lack in Other Areas:",{"_20":899,"_90":271,"_272":900,"_141":901},"c43475645aab",[]," While things may certainly change in the future, LLM chatbots seem best at confidently delivering text that is pleasant to read, but not always complete, or correct, and certainly not above human verification. Today’s LLMs are sometimes wrong, but never uncertain.",[],{"_20":904,"_90":266,"_267":905,"_423":424,"_425":426,"_275":914,"_277":278},"b9dfd0319569",[906,910],{"_20":907,"_90":271,"_272":908,"_141":909},"40cd476dbc8f0",[417],"GenAI’s Greatest Value to SRE Might Be for After Reports:",{"_20":911,"_90":271,"_272":912,"_141":913},"cbfbd1134d86",[]," Post-incident phases call for a great many write-ups to document the conditions leading up to outages, the circumstances and effects of the outages, and the actions taken to resolve the outages. GenAI can certainly produce something readable and user-friendly for general audiences, though expert engineers may prefer to keep all the gory details.",[],{"_20":916,"_90":266,"_267":917,"_423":424,"_425":426,"_275":926,"_277":278},"ee0377037b44",[918,922],{"_20":919,"_90":271,"_272":920,"_141":921},"0fd3f4081e130",[417],"Maybe There’s a Future for AI-Powered Pattern-Matching and Timeframe Planning in IM:",{"_20":923,"_90":271,"_272":924,"_141":925},"3c72aebd49fd",[]," Chapman recalls his years working with highly experienced engineers who were so well-versed in their systems that they seemed to have a sixth sense when it came to browsing a series of graphs and detecting seemingly imperceptible irregularities when investigating root causes. Could AI tools eventually become “smart” enough to detect such inconsistencies? Maybe. They might be even more useful in helping size incidents and projected response time windows that correspond with severity.",[],{"_20":928,"_90":266,"_267":929,"_275":934,"_277":574},"ba5d422e08c3",[930],{"_20":931,"_90":271,"_272":932,"_141":933},"7239f87585710",[],"Discussion: The Brushes May Change, but Engineers Will Still Create Art with Brent Chapman",[],{"_20":936,"_90":266,"_267":937,"_275":942,"_277":278},"6931f57ba4e9",[938],{"_20":939,"_90":271,"_272":940,"_141":941},"c28dd3e235ad0",[],"As Chapman reflects on the fundamental practice of incident management, he finds few intersections with GenAI’s biggest strengths and sees many of the processes as still being fundamentally human. “In incident management, the challenge is always that we know something has gone wrong, but we don't always know what has gone wrong, or who needs to do what to fix it. The process is about figuring out the details, executing that response, getting things back to a stable situation, and then getting fully recovered to a normal state of operation. All very challenging activities that involve working under time pressure that people normally don't have, working across teams that don't routinely work together and don't know each other's capabilities and concerns and considerations and so forth.”",[],{"_20":944,"_90":266,"_267":945,"_275":950,"_277":278},"1de62f39e47e",[946],{"_20":947,"_90":271,"_272":948,"_141":949},"96071cbd4a730",[],"“In our day-to-day work, we establish project teams. We spend a lot of time ‘storming and norming’ to build that whole framework of ‘how do we learn about each other and work together effectively,’ and have debates and arguments and joint planning activities that let technology companies do the amazing things they do,” Chapman offers. But in the same way that Agile methodology teams emphasize flexibility and a pragmatic approach to delivery, incident teams also need to be realistic.”",[],{"_20":952,"_90":266,"_267":953,"_275":958,"_277":278},"cf9de04ffe52",[954],{"_20":955,"_90":271,"_272":956,"_141":957},"1b52a8d2b36b",[],"“Those things all take time and energy to establish. And you don't have that time and energy available during an emergency.” Downtime doesn’t just provide opportunities for collaboration–it demands collaboration. “You need to find a way to work together quickly and effectively enough for the emergency, even if it's not necessarily a great way to work together in the long run. It's very top-down, it's very authoritative, it's very hierarchical, it's very old-fashioned. But it works better in an emergency. Also, not everybody who's going to help you will be available at the same time, and certainly not at the start of the incident. You have to start responding with who's available at the time and incorporate more people over time. You need to have ways of effectively putting people to work and then putting more people in the process without disrupting the work that's already in progress, so people can come up to speed without disrupting those activities and then plug themselves in, offload, or take on some new tasks related to the emergency. And somebody has to manage and coordinate all of this and manage the communications.”",[],{"_20":960,"_90":266,"_267":961,"_275":966,"_277":599},"e48e9861485c",[962],{"_20":963,"_90":271,"_272":964,"_141":965},"dd6aa2fb3f2c",[],"[Working with GenAI is] a lot like dealing with a very junior programmer. You still have to check it. You still have to write the code. You still have to write the test.” -Brent Chapman, Principal / Great Circle Associates",[],{"_20":968,"_90":266,"_267":969,"_275":974,"_277":278},"df0683fdcb84",[970],{"_20":971,"_90":271,"_272":972,"_141":973},"5f270c56a7810",[],"Chapman suggests that part of the excitement, and confusion, around AI and its benefits may come from an excessive widening or narrowing of definitions. The SRE veteran discusses automated systems that he worked on at Google for real-time ‘traffic’ (server cluster load balancing) management. “I worked on a system that tracked incident response patterns whenever there were problems with a given cluster, such that the first thing we’d do is drain incoming traffic away from that cluster, and send the traffic somewhere else that's still healthy. Realizing that this was the first thing we almost always did, we decided to automate that. But we needed some guardrails. For instance, we didn't want to drain traffic away from the last cluster in any continent, or from a cluster when there were only two clusters left. So I built this system in Python which, when it was alerted to unhealthy clusters, it would ‘think’ about draining it, run through the list of checks for that service, and decide whether or not to drain the service in that cluster. All taking place while people were still responding to their pagers. But depending on your definitions, this might be more of a style of ‘mechanical turk’-style automation rather than ‘AI,’ I suppose.”",[],{"_20":976,"_90":266,"_267":977,"_275":982,"_277":278},"ff31cafe599e",[978],{"_20":979,"_90":271,"_272":980,"_141":981},"3e1dfbd9b0c90",[],"What will the future hold for software engineers? For operations, Chapman sees potential in highly-trained AI systems with the ability to take certain actions, such as taking steps to provision new systems, or adjusting configurations, autonomously. “For instance, if you tie a generative AI system to your AWS console–your control system for your cloud computing system–and you can start having a conversation with it about, let’s say, bringing online another 20% of capacity in London. And the system replies that there's not enough spare capacity in London, but it can give you 15% in London and 5% across Europe, for example. You can imagine starting to have these sorts of operative discussions with it that are going to result in things happening...under approval with your supervision, and so forth. Right now, a lot of our monitoring control systems present human operators with problems, and it's up to the operator to solve them. I think the next step is going to be to have generative AI propose solutions. Now, is it going to propose better solutions than your average new hire six months out of college? Maybe. That's going to be an interesting question.”",[],{"_20":984,"_90":266,"_267":985,"_275":990,"_277":278},"205a7fc5c7d3",[986],{"_20":987,"_90":271,"_272":988,"_141":989},"714991d7f8ac0",[],"On the topic of AI’s impact on developers as a whole, Chapman still feels strongly that systems will ultimately still need humans in the loop. “I think there's going to continue to be a need for more developers. However, there's going to be a new skill set that many of them have: ‘prompting,’ basically developing prompts, asking the right questions, feeding the right data to get a useful result out of the generative AI systems they're working with. It seems comparable to art. ‘Creating art’ is going to change from knowing how to mix paints and pick a brush and applying certain physical techniques to knowing how to describe what you're looking for to the AI...so that it can generate something that looks like what you want. I'm already hearing a bunch of my programmer colleagues talking about how they’ve had good luck using ChatGPT for tasks such as providing the framework of a Python application that does such and such. It’ll write the first hundred lines of code and create the first 10 files, and basically sets up your project for you. And then you can go from there. But it's a lot like dealing with a very junior programmer who doesn't always understand your intent and sometimes just goes off into the weeds. You still have to check it. You still have to write the code. You still have to write the test. I’ve also heard some people report success generating unit tests based on inputted code (which seems kind of backward, since you’re supposed to write your tests first and write the code to make the test pass), but is that better than no unit tests at all? Yeah, probably.”",[],{"_20":992,"_90":266,"_267":993,"_275":1007,"_277":278},"f03b7fd250a7",[994,998,1003],{"_20":995,"_90":271,"_272":996,"_141":997},"7a11aebb2d740",[],"“One of my favorite science fiction authors is ",{"_20":999,"_90":271,"_272":1000,"_141":1002},"7a11aebb2d741",[1001],"e7cbdbf374e7","Vernor Vinge",{"_20":1004,"_90":271,"_272":1005,"_141":1006},"7a11aebb2d742",[],", who was a professor of computer science at San Diego State University. He wrote a story with a character who, for various reasons having to do with relativistic time dilation and traveling near the speed of light, ends up returning home a thousand years later, despite only aging about 10 years. He believes his skills are going to be completely out of date in this new world. And it turns out, the guy's a programmer and there is a place for his skills as basically someone who understands the system 14 layers underneath what the AIs are doing in that present day. He has an ability, by understanding those underpinnings, to bypass a lot of layers and go straight to the ‘low level,’ a bit like a programmer today who still understands assembly language. Obviously, this is kind of a simplified example, but I think there are still going to be plenty of roles for humans in technology. Someone still has to verify–to ask the questions: ‘Is this right? Is this useful? Is this complete?’ I don't see that role moving away from human judgment.”",[1008],{"_20":1001,"_90":30,"_349":1009},"https://en.wikipedia.org/wiki/Vernor_Vinge",{"_20":1011,"_90":266,"_267":1012,"_275":1017,"_277":312},"8c3185928299",[1013],{"_20":1014,"_90":271,"_272":1015,"_141":1016},"3ec33e0d286e0",[],"Conclusion",[],{"_20":1019,"_90":266,"_267":1020,"_275":1034,"_277":278},"472ba1e8fbe6",[1021,1025,1030],{"_20":1022,"_90":271,"_272":1023,"_141":1024},"97a64d402e6d0",[],"GenAI is already seeing direct applications in day-to-day SRE work. For more information and discussion on how AI may affect the future of incident management, DevOps, and the business of software development as a whole, join the ",{"_20":1026,"_90":271,"_272":1027,"_141":1029},"97a64d402e6d1",[1028],"cc230a529f2d","DevGuild: AI Summit",{"_20":1031,"_90":271,"_272":1032,"_141":1033},"97a64d402e6d2",[]," event.",[1035],{"_20":1028,"_90":30,"_349":1036},"https://www.heavybit.com/devguild",{"_20":1038,"_90":266,"_267":1039,"_275":1044,"_277":312},"701eb23619a8",[1040],{"_20":1041,"_90":271,"_272":1042,"_141":1043},"ba3da29add990",[],"More Resources:",[],{"_20":1046,"_90":266,"_267":1047,"_423":424,"_425":426,"_275":1053,"_277":278},"df2f50be4a6f",[1048],{"_20":1049,"_90":271,"_272":1050,"_141":1052},"4d151d677cd00",[1051],"94c99a270c8e","Article - Three Key Best Practices for Modern Incident Response",[1054],{"_20":1051,"_90":30,"_349":1055},"https://www.heavybit.com/library/article/incident-response-best-practices",{"_20":1057,"_90":266,"_267":1058,"_423":424,"_425":426,"_275":1064,"_277":278},"784420f1c8e2",[1059],{"_20":1060,"_90":271,"_272":1061,"_141":1063},"17656e1581d00",[1062],"97fc93c1e8e1","On-Demand Video Series - DevGuild: Incident Response",[1065],{"_20":1062,"_90":30,"_349":352},{"_20":1067,"_90":266,"_267":1068,"_423":424,"_425":426,"_275":1074,"_277":278},"eb0c700a9de0",[1069],{"_20":1070,"_90":271,"_272":1071,"_141":1073},"ec5a9ee2c8360",[1072],"5bc485daa9db","Podcast - Getting There with Nora Jones, Niall Murphy and Laura De Vesine",[1075],{"_20":1072,"_90":30,"_349":1076},"https://www.heavybit.com/library/podcasts/getting-there/ep-7-the-march-2023-datadog-outage-with-laura-de-vesine","duration",31,"https://cdn.sanity.io/images/50q6fr1p/production/b67963a5a553d32ba4c307eb7425f753bcdaba6a-800x300.jpg","objectID","d285c495-1ad9-437f-9d3e-d39fd8e73869","publishDate","2023-10-12T22:08:00.000Z","slug","generative-ai-incident-response-devops","templateType","Incident Response and DevOps in the Age of Generative AI","typeName","blogArticle",{"_302":1091,"_1077":1100,"_127":1101,"_1080":1102,"_1082":1103,"_1084":1104,"_32":1105,"_1088":1106},[1092],{"_20":1093,"_90":266,"_267":1094,"_275":1099,"_277":278},"39c5027bc9ac",[1095],{"_20":1096,"_90":271,"_272":1097,"_141":1098},"e3258956b2e70",[],"In episode 7 of Getting There, Nora and Niall speak with Laura de Vesine of Datadog. Laura shares a unique perspective on the March 2023 Datadog outage, how the incident was handled internally, the resulting damage of the outage, and the many lessons learned.",[],"1:00:03","https://cdn.sanity.io/images/50q6fr1p/production/8e90f75f2586036c7de9693d7842080132f8a78f-3000x3000.jpg","fb82a5d4-825e-4779-9c94-28d44e946867","2023-06-12T07:10:00.000Z","getting-there/ep-7-the-march-2023-datadog-outage-with-laura-de-vesine","Getting There Ep. #7, The March 2023 Datadog Outage with Laura de Vesine","podcastEpisode",{"_302":1108,"_1077":1117,"_127":1118,"_1080":1119,"_1082":1120,"_1084":1121,"_32":1122,"_1088":1106},[1109],{"_20":1110,"_90":266,"_267":1111,"_275":1116,"_277":278},"00f7e8215407",[1112],{"_20":1113,"_90":271,"_272":1114,"_141":1115},"506c3d3df89b0",[],"In episode 59 of o11ycast, Jess and Martin speak with Laura Maguire of Jeli and Nick Travaglini of Honeycomb. They unpack Learning From Incidents (LFI), resilience engineering, process tracing, safety science, key takeaways from the LFI Conference, and the human side of observability.",[],"48:49","https://cdn.sanity.io/images/50q6fr1p/production/3e9f4710b6abe02488e6cc34e8ff0d0fc29a41a7-3000x3000.png","c887a73e-84cb-4e25-b244-84f8e4c0fe3c","2023-04-13T07:10:00.000Z","o11ycast/ep-59-learning-from-incidents-with-laura-maguire-of-jeli","O11ycast Ep. #59, Learning From Incidents with Laura Maguire of Jeli","recent",[1125,1736,2168],{"_302":1126,"_1077":1729,"_127":1730,"_1080":1731,"_1082":1732,"_1084":1733,"_1086":1734,"_32":1735,"_1088":1089},[1127,1135,1165,1194,1198,1213,1221,1240,1259,1267,1275,1283,1313,1321,1329,1337,1345,1364,1383,1391,1399,1426,1444,1460,1468,1476,1484,1492,1500,1508,1538,1546,1554,1562,1570,1578,1586,1594,1635,1665,1673,1681,1697,1705,1721],{"_20":1128,"_90":266,"_267":1129,"_275":1134,"_277":312},"f09e9612ef15",[1130],{"_20":1131,"_90":271,"_272":1132,"_141":1133},"85de00f41251",[],"Why Code-as-Text Doesn’t Work for Genuine Understanding",[],{"_20":1136,"_90":266,"_267":1137,"_275":1160,"_277":278},"4b5755f9b8bc",[1138,1142,1147,1151,1156],{"_20":1139,"_90":271,"_272":1140,"_141":1141},"adc4d52ad4cd",[],"Major engineering orgs claim that as much as ",{"_20":1143,"_90":271,"_272":1144,"_141":1146},"d1aebf2378c0",[1145],"afe823ab102f","30%",{"_20":1148,"_90":271,"_272":1149,"_141":1150},"47cf0e8a331c",[]," to ",{"_20":1152,"_90":271,"_272":1153,"_141":1155},"a7022507758b",[1154],"8042bdcd04fd","75%",{"_20":1157,"_90":271,"_272":1158,"_141":1159},"70dd8d1c5fc1",[]," of their new code is AI-generated, which does nothing to help the ongoing challenges of first-time onboarding for codebases and the updating of existing codebases for newly-hired engineers and growing teams (as well as for agentic coding assistants).",[1161,1163],{"_20":1145,"_90":30,"_349":1162},"https://techcrunch.com/2025/04/29/microsoft-ceo-says-up-to-30-of-the-companys-code-was-written-by-ai/",{"_20":1154,"_90":30,"_349":1164},"https://blog.google/innovation-and-ai/infrastructure-and-cloud/google-cloud/cloud-next-2026-sundar-pichai/",{"_20":1166,"_90":266,"_267":1167,"_275":1189,"_277":278},"9b432548c002",[1168,1172,1177,1181,1186],{"_20":1169,"_90":271,"_272":1170,"_141":1171},"9c50f1246f14",[],"As generating code becomes free, understanding what any of it does becomes much more costly, and updating it becomes significantly more complex. Developer ",{"_20":1173,"_90":271,"_272":1174,"_141":1176},"7aa3b54f64f6",[1175],"a50030cc5b04","Vitali Avagyan",{"_20":1178,"_90":271,"_272":1179,"_141":1180},"a0f94686e2c4",[]," asserts that the key to understanding code, particularly code across different languages, is to stop thinking of it as lines of plain text, and to treat it as a relational graph, which is the basis of his open-source project ",{"_20":1182,"_90":271,"_272":1183,"_141":1185},"c66f9ef7a776",[1184],"b6ce27c00ce8","code-graph-rag",{"_20":1187,"_90":271,"_272":1188,"_141":883},"c24ce7265e1d",[],[1190,1192],{"_20":1175,"_90":30,"_349":1191},"https://www.linkedin.com/in/vitali-avagyan-a1566234/",{"_20":1184,"_90":30,"_349":1193},"https://github.com/vitali87/code-graph-rag",{"_20":1195,"_90":127,"_92":1196},"c9d37b6922a5",{"_94":1197,"_90":96},"image-bc24d7b1e62740bb7448a1a80a9dfd59940c68d6-3840x1100-jpg",{"_20":1199,"_90":266,"_267":1200,"_275":1210,"_277":278},"b80310699127",[1201,1205],{"_20":1202,"_90":271,"_272":1203,"_141":1204},"a0931bc06a15",[458],"Vitali Avagyan discusses using GraphRAG to parse repos with Memgraph. Image courtesy ",{"_20":1206,"_90":271,"_272":1207,"_141":1209},"5ef047d4ccf9",[1208,458],"5b72c578e5d1","Memgraph",[1211],{"_20":1208,"_90":30,"_349":1212},"https://www.youtube.com/watch?v=O2jUTq6nCEY",{"_20":1214,"_90":266,"_267":1215,"_275":1220,"_277":312},"2fc0e81b9ef7",[1216],{"_20":1217,"_90":271,"_272":1218,"_141":1219},"4cb4e6406d24",[],"Viewing Code as a Retrievable Graph",[],{"_20":1222,"_90":266,"_267":1223,"_275":1237,"_277":278},"c357da4c0347",[1224,1228,1233],{"_20":1225,"_90":271,"_272":1226,"_141":1227},"b42b4a2c8abb",[],"Avagyan notes that when you hand a codebase to one of today’s coding assistants, it’ll often treat it as a text artifact. “Then, it traverses the code base with a ",{"_20":1229,"_90":271,"_272":1230,"_141":1232},"683e39e682aa",[1231],"67f4fbd2fe9a","grep",{"_20":1234,"_90":271,"_272":1235,"_141":1236},"7de49dd818b7",[]," or possibly some kind of agentic search. As you might imagine, that can become inefficient, especially considering some of the questions you might typically have about a codebase.”",[1238],{"_20":1231,"_90":30,"_349":1239},"https://en.wikipedia.org/wiki/Grep",{"_20":1241,"_90":266,"_267":1242,"_275":1256,"_277":278},"6bfeff21d7b3",[1243,1247,1252],{"_20":1244,"_90":271,"_272":1245,"_141":1246},"2814a2a422f6",[],"The creator gives the example of asking an agent to find the longest nested function call in your codebase. Today’s agents will search through the imports and invocations and do some counting, a task that LLMs aren’t great at. “But if you already have a graph that represents all core relationships between methods and functions, you or an AI agent can just write a ",{"_20":1248,"_90":271,"_272":1249,"_141":1251},"fd7434afa3fb",[1250],"40d5a9d1fab0","Cypher",{"_20":1253,"_90":271,"_272":1254,"_141":1255},"850209620fae",[]," query that will calculate it for you in milliseconds.” The creator suggests that treating codebases as graphs in this way can conserve both tokens and time.",[1257],{"_20":1250,"_90":30,"_349":1258},"https://en.wikipedia.org/wiki/Cypher_(query_language)",{"_20":1260,"_90":266,"_267":1261,"_275":1266,"_277":278},"8edb1e48b5a2",[1262],{"_20":1263,"_90":271,"_272":1264,"_141":1265},"b5cb7e6813d4",[],"The creator clarifies, “that doesn't mean you should use [text] or [graphs] exclusively. They can be used in conjunction with each other. You can have grep, other types of search, and graph search together. Then, given that the AI has both contexts, it will decide which way to go.”",[],{"_20":1268,"_90":266,"_267":1269,"_275":1274,"_277":278},"55289533b05a",[1270],{"_20":1271,"_90":271,"_272":1272,"_141":1273},"494b6d9eb54a",[],"The creator suggests that the project specifically addresses the dichotomy between graph search and traditional text search/grep. “The first part of the project is how you go from code to graph. The second is an assistant that has a context of a graph, but it also has all the tools that typical coding agents or harnesses have. I also use it alongside my other coding harnesses. For some problems, it can find solutions faster and more accurately.”",[],{"_20":1276,"_90":266,"_267":1277,"_275":1282,"_277":312},"cc2e0c67c8ad",[1278],{"_20":1279,"_90":271,"_272":1280,"_141":1281},"a7a96359e940",[],"Accounting for Variation Across Languages, Frameworks",[],{"_20":1284,"_90":266,"_267":1285,"_275":1308,"_277":278},"755c01bddb6f",[1286,1290,1295,1299,1304],{"_20":1287,"_90":271,"_272":1288,"_141":1289},"8cfad1aaed2e",[],"The creator clarifies that precisely mapping codebases to graphs requires a correct build for each framework and programming language. “Even with an underlying technology like ",{"_20":1291,"_90":271,"_272":1292,"_141":1294},"c4f79c67c17a",[1293],"d7c41e85c164","Tree-sitter",{"_20":1296,"_90":271,"_272":1297,"_141":1298},"2d414b50c68b",[],", which parses the ",{"_20":1300,"_90":271,"_272":1301,"_141":1303},"ecb02d6d29a7",[1302],"249727404c78","abstract syntax tree",{"_20":1305,"_90":271,"_272":1306,"_141":1307},"916d54525867",[]," and gives you all the syntactic relationships, it's not still enough because there’s so much variety between languages and frameworks.”",[1309,1311],{"_20":1293,"_90":30,"_349":1310},"https://tree-sitter.github.io/tree-sitter/",{"_20":1302,"_90":30,"_349":1312},"https://en.wikipedia.org/wiki/Abstract_syntax_tree",{"_20":1314,"_90":266,"_267":1315,"_275":1320,"_277":278},"4b92136c2a79",[1316],{"_20":1317,"_90":271,"_272":1318,"_141":1319},"916fbfa96bd2",[],"Avagyan’s approach was to unify the variety within a single graph ontology that worked across multiple languages, from which he designed the project to let users map relationships within a monorepo, even those that contain multiple languages (the project currently supports more than a dozen). With such precise relationships properly mapped, users can use Cypher or another query language (or some sort of tool to map natural language to a query language) to query any part of a codebase as a graph, and compare against semantic search.",[],{"_20":1322,"_90":266,"_267":1323,"_275":1328,"_277":278},"a36ee5a77fd2",[1324],{"_20":1325,"_90":271,"_272":1326,"_141":1327},"b5e47a104baa",[],"“Obviously, this has lots of benefits. If you ask a question about a specific function or a service with semantic search, it looks at the vector similarity and brings you some functions, some of which may have relevant names, but may not be relevant to your actual query. But using graphs, if you can translate your intent into Cypher (with the help of AI), you can basically pinpoint which function you are looking for.” The creator clarifies that the project also supports semantic search as an add-on to support things like code comments, and that in practice, graph parsing and semantic search tend to be complementary.",[],{"_20":1330,"_90":266,"_267":1331,"_275":1336,"_277":312},"6154d54ed4e6",[1332],{"_20":1333,"_90":271,"_272":1334,"_141":1335},"1641bec46e7a",[],"Making Code Graphs Deterministic with AST",[],{"_20":1338,"_90":266,"_267":1339,"_275":1344,"_277":278},"0815be5d9aee",[1340],{"_20":1341,"_90":271,"_272":1342,"_141":1343},"6ea22a083826",[],"As he sought to ensure the project yielded reliable graph representations, Avagyan found that asking LLMs to one-shot graph representations didn’t quite work. “[The LLMs] hallucinated some relationship edges and nodes. So, the project uses Tree-sitter parsers for different languages, which give you the basic relationships you can extract from AST as something of a unified way of using any type of language.”",[],{"_20":1346,"_90":266,"_267":1347,"_275":1361,"_277":278},"4ed1d30f3dc2",[1348,1352,1357],{"_20":1349,"_90":271,"_272":1350,"_141":1351},"9a4ee9a258b2",[],"“Obviously, there are so many considerations when you go into specifics, like finding call relationships. Each language has its own specific intricacies, like how ",{"_20":1353,"_90":271,"_272":1354,"_141":1356},"e56a902ff095",[1355],"1b1e54e0707e","polymorphism",{"_20":1358,"_90":271,"_272":1359,"_141":1360},"9b109a84fb06",[]," works.” The creator notes that tracking all the nuances isn’t necessarily covered by existing deterministic open-source tools either. What is even more difficult is the runtime behavior of the code. That's why recent efforts have been placed to trace call relationships apparent at runtime and merge them into the same graph with statically-derived ones.”",[1362],{"_20":1355,"_90":30,"_349":1363},"https://en.wikipedia.org/wiki/Polymorphism_(computer_science)",{"_20":1365,"_90":266,"_267":1366,"_275":1380,"_277":278},"9b53b53a2ef7",[1367,1371,1376],{"_20":1368,"_90":271,"_272":1369,"_141":1370},"84140f1e533e",[],"“And from there, we run thousands of test cases to ensure there are no regressions every time a new functionality is added. Every new functionality or bugfix is accompanied by a handful of new tests.” The creator suggests the project continuously runs more than 7,000 unit/integration tests using ",{"_20":1372,"_90":271,"_272":1373,"_141":1375},"a90a0f3ba6d2",[1374],"cb676615c060","red/green test-driven development",{"_20":1377,"_90":271,"_272":1378,"_141":1379},"a3e38bb4ef42",[],". Over time, the combination of hand-built code and agent-powered testing and development has proven to be effective. “At the beginning, I spent so much time on architecting [the project], setting the right relationships, that sort of thing, to ensure the foundation was ‘right.’ But when the foundation is ‘right,’ AI is great at doing the rest.”",[1381],{"_20":1374,"_90":30,"_349":1382},"https://simonwillison.net/guides/agentic-engineering-patterns/red-green-tdd/",{"_20":1384,"_90":266,"_267":1385,"_275":1390,"_277":312},"e71e561bda30",[1386],{"_20":1387,"_90":271,"_272":1388,"_141":1389},"8bfe3199c316",[],"Bottlenecks and Resource Management",[],{"_20":1392,"_90":266,"_267":1393,"_275":1398,"_277":278},"2e27e83e6ea0",[1394],{"_20":1395,"_90":271,"_272":1396,"_141":1397},"b4e8f24bb41b",[],"The creator suggests that the project is continuing to improve on traditional challenges in graph-based projects, which have historically not always been particularly responsive. “I would say storage has never been a problem. With this project, I spent significant time optimizing the build process of going from codebase to graph. (I actually optimised the build time by more than 5x with a separate repo I built that optimizes user-defined codebase segments against specific metrics like speed within accepted memory constraints. The agent then \"autosearches\" the best code among competing candidates while running the build against the ground-truthed evaluation cases which you or your agents have collected or constructed yourself.)”",[],{"_20":1400,"_90":266,"_267":1401,"_275":1423,"_277":278},"d8236069c0ba",[1402,1406,1411,1415,1419],{"_20":1403,"_90":271,"_272":1404,"_141":1405},"37e4c8419854",[],"“I would say that a practical use of code-graph-rag is ",{"_20":1407,"_90":271,"_272":1408,"_141":1410},"0f95656bd1a7",[1409],"ec14de8b6db9","dead code detection",{"_20":1412,"_90":271,"_272":1413,"_141":1414},"28ed3f1a3a15",[],". If you have vibe coded your repo, or it’s otherwise in a state that you don't know exactly what's happening because there are so many abstractions or code duplications, or that code isn’t reachable. For example, there is a ",{"_20":1416,"_90":271,"_272":1417,"_141":1418},"cba21817906a",[458],"code graph",{"_20":1420,"_90":271,"_272":1421,"_141":1422},"2dd3e29f6f5e",[]," command that will check the reachability of all your code. If some are not reachable, they’re likely candidates [for being dead code and potentially getting deleted].”",[1424],{"_20":1409,"_90":30,"_349":1425},"https://en.wikipedia.org/wiki/Dead-code_elimination",{"_20":1427,"_90":266,"_267":1428,"_275":1441,"_277":278},"623b0301e0bf",[1429,1433,1437],{"_20":1430,"_90":271,"_272":1431,"_141":1432},"c9fdbc40e4d6",[],"“Otherwise, I don't think storage is a problem. The project uses ",{"_20":1434,"_90":271,"_272":1435,"_141":1209},"34da333815d3",[1436],"c1ed526f5769",{"_20":1438,"_90":271,"_272":1439,"_141":1440},"da89921f17d9",[]," as a GraphDB and it's really fast, it runs in memory and you can deploy using Docker containers, and that would be enough to deploy locally, and the project has the building blocks for bigger deployments to the cloud. So far, it seems to scale well (people have even graphed the Linux kernel) and I haven’t seen any real degradation in speed when it comes to querying graphs.”",[1442],{"_20":1436,"_90":30,"_349":1443},"https://memgraph.com/",{"_20":1445,"_90":266,"_267":1446,"_275":1459,"_277":278},"824172ebddac",[1447,1451,1455],{"_20":1448,"_90":271,"_272":1449,"_141":1450},"7cdd05898655",[],"Avagyan notes that some users come to the project looking for performance and cost gains as well. “Some people claim that the project is as much as 65% more cost-efficient than traditional agentic coding and search, and I can’t verify those numbers myself. I ",{"_20":1452,"_90":271,"_272":1453,"_141":1454},"36ec44554375",[458],"can",{"_20":1456,"_90":271,"_272":1457,"_141":1458},"e051511f75df",[]," say that I’ve observed the project to be very fast, and it seems more fault-tolerant in a way.”",[],{"_20":1461,"_90":266,"_267":1462,"_275":1467,"_277":278},"2cfb4c4a9a81",[1463],{"_20":1464,"_90":271,"_272":1465,"_141":1466},"b8c18965ccb1",[],"“Let’s say the project fails in one Cypher query? Then it can just repeat it, and with that query, you can gain lots of context up front, very fast. And that failure is less costly compared to if AI were to go and perform its own search, realize it was searching in the wrong place, then go to another place...you can imagine how things would accumulate really fast.”",[],{"_20":1469,"_90":266,"_267":1470,"_275":1475,"_277":278},"6a4ddfd50139",[1471],{"_20":1472,"_90":271,"_272":1473,"_141":1474},"847581b898f9",[],"“To be honest, the goal of the project was never cost reduction, but I think it’s a byproduct, since you get the accuracy of a graph and the accuracy of an MCP and agent to invoke the correct tools to query the graph.",[],{"_20":1477,"_90":266,"_267":1478,"_275":1483,"_277":312},"06fc03e6d208",[1479],{"_20":1480,"_90":271,"_272":1481,"_141":1482},"0743e3b412d6",[],"Real-World Use Cases for Graphing a Codebase",[],{"_20":1485,"_90":266,"_267":1486,"_275":1491,"_277":278},"34664692f09c",[1487],{"_20":1488,"_90":271,"_272":1489,"_141":1490},"2b1eaf954306",[],"The creator calls out onboarding new engineers as a key use case for the project. “People tell me that using the project has significantly reduced onboarding times because it can gather information about different aspects of a monorepo codebase and give that directly to new junior engineers.”",[],{"_20":1493,"_90":266,"_267":1494,"_275":1499,"_277":278},"a9bdfd671d83",[1495],{"_20":1496,"_90":271,"_272":1497,"_141":1498},"b0569066d73c",[],"“Since we recently added FLOWS_TO, READS_FROM and WRITES_TO relationships in a graph, it’s now easy to analyze the relationships between microservices in your codebase as well. (I’ve actually been approached by security companies about using the project for security detection use cases across large codebases, which is a natural fit as tracing how data moves through a system is exactly what those relationships capture.)”",[],{"_20":1501,"_90":266,"_267":1502,"_275":1507,"_277":278},"e96ed7f2d62e",[1503],{"_20":1504,"_90":271,"_272":1505,"_141":1506},"e4b2c56f631d",[],"So you can see why companies are adopting the project as part of their onboarding tools. You could use it as an MCP server in Claude Code or any other harness or just use Cypher queries to search your codebase before even starting any agentic exploration. This approach is quick and gives you a really good grasp of a codebase, without getting sidetracked to other areas that may not be as important.”",[],{"_20":1509,"_90":266,"_267":1510,"_275":1533,"_277":278},"b003dd295bbf",[1511,1515,1520,1524,1529],{"_20":1512,"_90":271,"_272":1513,"_141":1514},"2a8e4e8ba899",[],"Avagyan notes growing interest among teams managing heavyweight ERP systems with some flavor of Python, as well as ongoing requests around app development. “",{"_20":1516,"_90":271,"_272":1517,"_141":1519},"fc85ec3d18b6",[1518],"fc884ca87ef9","Dart",{"_20":1521,"_90":271,"_272":1522,"_141":1523},"cee0616ee44a",[]," is already supported and includes ",{"_20":1525,"_90":271,"_272":1526,"_141":1528},"ab660c7a1ffd",[1527],"eb5b74dd23fd","Flutter",{"_20":1530,"_90":271,"_272":1531,"_141":1532},"b630193673be",[]," widget structure.”",[1534,1536],{"_20":1518,"_90":30,"_349":1535},"https://dart.dev/overview",{"_20":1527,"_90":30,"_349":1537},"https://flutter.dev/",{"_20":1539,"_90":266,"_267":1540,"_275":1545,"_277":312},"5a3ea8acd83b",[1541],{"_20":1542,"_90":271,"_272":1543,"_141":1544},"1cd77c6dd0f6",[],"Why Aren’t Enterprises Already Graphing Their Codebases?",[],{"_20":1547,"_90":266,"_267":1548,"_275":1553,"_277":278},"7cbaec07cf9b",[1549],{"_20":1550,"_90":271,"_272":1551,"_141":1552},"52c6bc34d2b2",[],"The creator suggests that graphing codebases is a useful approach for any organization, but the timing might not be quite right for large-scale enterprises that are slow to embrace change. “I believe many companies, including major AI vendors like Cursor or Anthropic, have been experimenting with graph-based approaches, if not for codebase mapping, then for use cases like memory management.”",[],{"_20":1555,"_90":266,"_267":1556,"_275":1561,"_277":278},"a89686ddcbd7",[1557],{"_20":1558,"_90":271,"_272":1559,"_141":1560},"eb00873aac4a",[],"Avagyan notes that those companies may be hitting a stumbling block. “I think one problem companies may be having is accuracy.”",[],{"_20":1563,"_90":266,"_267":1564,"_275":1569,"_277":278},"f2f91f55c9c2",[1565],{"_20":1566,"_90":271,"_272":1567,"_141":1568},"5556895009c6",[],"“You can get really far with one language, maybe with a toy example using a tool like Tree-sitter. But jobs that require you to unify graphs across different languages, and build usable and accurate relationships over top of them, and making sure it can work at scale, that’s different. For example, a monorepo is a living creature that changes every minute: People commit and delete code and so on. So your graph needs to keep up with that. It needs to recalculate [those relationships] with every change, every drop, every addition. The project has a graph updater designed specifically to make this real-time syncing possible.”",[],{"_20":1571,"_90":266,"_267":1572,"_275":1577,"_277":278},"526da0bc67f6",[1573],{"_20":1574,"_90":271,"_272":1575,"_141":1576},"b5c434461aac",[],"The creator suggests that while he spent the better part of a year building a system that could dynamically map and remap graph representations of codebases without significant latency, not every org is on the same page. “Even doing something like this with AI agents will take some time, and will cost internal resources. And people don’t always have patience for these things and may want to see a one-shot solution instead. But I think the approach will get more momentum and I think many companies will adopt it eventually.”",[],{"_20":1579,"_90":266,"_267":1580,"_275":1585,"_277":312},"05e8e1c9fe0a",[1581],{"_20":1582,"_90":271,"_272":1583,"_141":1584},"76872200c049",[],"Learnings from Building",[],{"_20":1587,"_90":266,"_267":1588,"_275":1593,"_277":278},"8c4981547047",[1589],{"_20":1590,"_90":271,"_272":1591,"_141":1592},"58335d9df518",[],"When asked what his most important learnings from the project are, Avagyan offers a few suggestions. “One, which I think is most important, is to just build something for yourself that you would like to have. That's pretty much how this project was born. Because I was dealing with [the challenges of managing a] monorepo, and at first, I just wanted to visualize it and understand its complexity. I looked for solutions to see if there was a good product for me. I didn't find one, so I decided I would build it myself, and after releasing it publicly, it turned to be useful for other people.”",[],{"_20":1595,"_90":266,"_267":1596,"_275":1628,"_277":278},"81d7413fc3b2",[1597,1601,1606,1610,1615,1619,1624],{"_20":1598,"_90":271,"_272":1599,"_141":1600},"321ac6bc0e40",[],"“Second, I would recommend building things from the perspective of thinking about what you do every day and how you can accelerate what you do and be more performant with your tools.” The creator notes another open-source project of his, ",{"_20":1602,"_90":271,"_272":1603,"_141":1605},"9f8ed6fdab01",[1604],"f040745fb71d","Croft",{"_20":1607,"_90":271,"_272":1608,"_141":1609},"630e70cbbe50",[],", as an example. “I’m a VS Code user, but I enjoy terminal as well, so I found myself going back and forth between VS Code, ",{"_20":1611,"_90":271,"_272":1612,"_141":1614},"94d7ac6fa32a",[1613],"3f971d287b17","Neovim",{"_20":1616,"_90":271,"_272":1617,"_141":1618},"e76bcfb5a8f3",[],", ",{"_20":1620,"_90":271,"_272":1621,"_141":1623},"93a91dcf4857",[1622],"2f238ed440f7","Zed",{"_20":1625,"_90":271,"_272":1626,"_141":1627},"1bb94c846e83",[]," and other alternatives. I found myself thinking there should be a better way, so I built a VS Code clone in the terminal with Rust to make it super-fast. I’ve been using it every day myself and I would recommend others try it out.”",[1629,1631,1633],{"_20":1604,"_90":30,"_349":1630},"https://github.com/vitali87/croft",{"_20":1613,"_90":30,"_349":1632},"https://neovim.io/",{"_20":1622,"_90":30,"_349":1634},"https://zed.dev/",{"_20":1636,"_90":266,"_267":1637,"_275":1660,"_277":278},"6869a525750a",[1638,1642,1647,1651,1656],{"_20":1639,"_90":271,"_272":1640,"_141":1641},"30637c109bb6",[],"“Finally, I’d recommend finding a distribution channel for your work. Even the greatest product on earth won’t necessarily be noticed if you just put it on GitHub and leave it there. You need to promote it. I was fortunate enough to have worked with the founder of ",{"_20":1643,"_90":271,"_272":1644,"_141":1646},"6424a1b96d65",[1645],"48182a105088","Daily Dose of Data Science",{"_20":1648,"_90":271,"_272":1649,"_141":1650},"3954a9df1f48",[]," when I was working on my own startup back in 2023. I got coverage for my product in this popular newsletter, which might have been the trigger that helped it ‘go viral’ on GitHub the first time. That said, I have also tried many other channels like Hacker News, LinkedIn and X posts, giving talks (including this session ",{"_20":1652,"_90":271,"_272":1653,"_141":1655},"6f0d56991c70",[1654],"41760ee29615","with Memgraph",{"_20":1657,"_90":271,"_272":1658,"_141":1659},"dd3d4a61f74f",[],"), but eventually, the project started trending again on GitHub organically. I guess it might have to do with the ‘right solution at the right time’ phenomenon.”",[1661,1663],{"_20":1645,"_90":30,"_349":1662},"https://blog.dailydoseofds.com/",{"_20":1654,"_90":30,"_349":1664},"https://www.youtube.com/watch?v=O2jUTq6nCEY&t=2s",{"_20":1666,"_90":266,"_267":1667,"_275":1672,"_277":312},"1d6c6865362e",[1668],{"_20":1669,"_90":271,"_272":1670,"_141":1671},"2586e6d9e8b9",[],"The Future of Codebases: Built and Managed by AI?",[],{"_20":1674,"_90":266,"_267":1675,"_275":1680,"_277":278},"ab8bfa185ba6",[1676],{"_20":1677,"_90":271,"_272":1678,"_141":1679},"067bce70e73c",[],"From his experience building the project, Avagyan advises anyone working in a technical capacity to learn how to orchestrate. “Software engineering is changing faster than people think. The ability to orchestrate things will be important no matter what. By that, I mean knowing many different technologies: Their strengths and weaknesses, the scalability of each thing, and when to use what.”",[],{"_20":1682,"_90":266,"_267":1683,"_275":1696,"_277":278},"afb3c879f969",[1684,1688,1692],{"_20":1685,"_90":271,"_272":1686,"_141":1687},"49750bd53c8c",[],"“For example, in order to deploy applications at scale, you might need a Kubernetes cluster. You shouldn’t have to be a highly skilled DevOps engineer to know that as AI can help you there, but you would need to know ",{"_20":1689,"_90":271,"_272":1690,"_141":1691},"7f5989d240bd",[458],"when",{"_20":1693,"_90":271,"_272":1694,"_141":1695},"56e7f92d1742",[]," to use it, and what the pros and cons are. That will help you to have this overarching, holistic view of what it's like to build a highly scalable software engineering system.”",[],{"_20":1698,"_90":266,"_267":1699,"_275":1704,"_277":278},"dc75d7ad8097",[1700],{"_20":1701,"_90":271,"_272":1702,"_141":1703},"c6a9980c57d2",[],"“You need to understand how the distributed systems work, and the code-graph-rag project fits really well here to help you ask architectural questions and locate potential bottlenecks, like dead code or other opportunities for optimization.”",[],{"_20":1706,"_90":266,"_267":1707,"_275":1720,"_277":278},"628421de9da8",[1708,1712,1716],{"_20":1709,"_90":271,"_272":1710,"_141":1711},"0cc52c592750",[],"“It’s about having this type of ‘systems thinking,’ an understanding of system design architecture and the underlying principles of the technologies. Not just knowing ",{"_20":1713,"_90":271,"_272":1714,"_141":1715},"d4b8c4a82cf1",[458],"what Redis is",{"_20":1717,"_90":271,"_272":1718,"_141":1719},"ad91e713dbfb",[],", but also understanding what caching is and why it’s important. You’re going to need to apply that understanding when you think your system actually needs caching!”",[],{"_20":1722,"_90":266,"_267":1723,"_275":1728,"_277":278},"8eaf5dda166e",[1724],{"_20":1725,"_90":271,"_272":1726,"_141":1727},"7fccb285f275",[],"“So I think orchestration will be important for quite some time, because AI is still not at the level of fully, autonomously orchestrating large-scale systems that will work smoothly and without any problems. We are not there yet.”",[],17,"https://cdn.sanity.io/images/50q6fr1p/production/28801f05c737cc57c3c04d467a0a54838fc70ad0-1200x630.png","9d8377ac-25e6-47b7-8e40-22ee9a7cb43e","2026-08-27T16:38:00.000Z","understanding-code-by-treating-it-as-a-graph","default","Is the Key to Understanding Code Treating it as a Graph?",{"_302":1737,"_1077":2162,"_127":2163,"_1080":2164,"_1082":2165,"_1084":2166,"_1086":1734,"_32":2167,"_1088":1089},[1738,1746,1754,1801,1809,1813,1821,1829,1858,1866,1874,1882,1890,1898,1906,1940,1952,1964,1976,1988,1996,2004,2012,2020,2028,2036,2044,2074,2082,2090,2098,2106,2114,2122,2130,2138,2146,2154],{"_20":1739,"_90":266,"_267":1740,"_275":1745,"_277":312},"9196fa5c1337",[1741],{"_20":1742,"_90":271,"_272":1743,"_141":1744},"daa6967db340",[],"The Unique Challenges of Mobile Compute",[],{"_20":1747,"_90":266,"_267":1748,"_275":1753,"_277":278},"76f45bfec570",[1749],{"_20":1750,"_90":271,"_272":1751,"_141":1752},"6c276d156142",[],"A significant focus in modern AI has been on large language models with billions of parameters. Billions more parameters mean bigger training investment, and presumably, more-impressive performance. But there are emerging use cases for mobile AI inference, like offline mobile text generation and the rise of general-purpose generative models for edge that hint at a growing need for reliable mobile AI performance.",[],{"_20":1755,"_90":266,"_267":1756,"_275":1792,"_277":278},"af0f0de9eaae",[1757,1762,1766,1771,1774,1779,1783,1788],{"_20":1758,"_90":271,"_272":1759,"_141":1761},"bce9edf18932",[1760],"01f586bb19c8","Chris Karani",{"_20":1763,"_90":271,"_272":1764,"_141":1765},"669b28ad0b26",[],", a longtime software engineer with specific expertise working in the mobile space, has built a variety of open-source, mobile-specific AI projects like ",{"_20":1767,"_90":271,"_272":1768,"_141":1770},"8b6f89ec9c39",[1769],"dc8c3118f86c","Wax",{"_20":1772,"_90":271,"_272":1773,"_141":385},"43437631fed2",[],{"_20":1775,"_90":271,"_272":1776,"_141":1778},"4fbb11a36374",[1777],"4edd5928711b","Swarm",{"_20":1780,"_90":271,"_272":1781,"_141":1782},"e59bd93b5314",[],", and is now focusing on building a mobile-specific monitoring and release-gating platform ",{"_20":1784,"_90":271,"_272":1785,"_141":1787},"aa0cafd1ebca",[1786],"a7fbf0b65b8d","RYNO",{"_20":1789,"_90":271,"_272":1790,"_141":1791},"4ab51806f777",[]," after observing the unique challenges that on-device inference poses to software performance on mobile.",[1793,1795,1797,1799],{"_20":1760,"_90":30,"_349":1794},"https://www.linkedin.com/in/chriskarani/",{"_20":1769,"_90":30,"_349":1796},"https://github.com/christopherkarani/Wax",{"_20":1777,"_90":30,"_349":1798},"https://github.com/christopherkarani/Swarm",{"_20":1786,"_90":30,"_349":1800},"https://www.youtube.com/watch?v=yL9lcmvSs7o",{"_20":1802,"_90":266,"_267":1803,"_275":1808,"_277":278},"6a7d245a9398",[1804],{"_20":1805,"_90":271,"_272":1806,"_141":1807},"d215b9275145",[],"Below, he explains why mobile inference needs a new breed of observability, and why on-device inference may necessitate a different way to think about app development: At the edge, moving inference on-device effectively introduces a variety of new hardware-specific variables, such as device generations, OS versions, and thermal and memory constraints.",[],{"_20":1810,"_90":127,"_92":1811},"68822d00ffc2",{"_94":1812,"_90":96},"image-1dd5e966858d3cb95946cf1b3d940b1c92f2c46a-2030x554-png",{"_20":1814,"_90":266,"_267":1815,"_275":1820,"_277":278},"23fa169cb01d",[1816],{"_20":1817,"_90":271,"_272":1818,"_141":1819},"57f6112d5429",[458],"The RYNO dashboard specifically monitors on-device inference.",[],{"_20":1822,"_90":266,"_267":1823,"_275":1828,"_277":312},"5d05e23e9ffc",[1824],{"_20":1825,"_90":271,"_272":1826,"_141":1827},"d169649fc0a1",[],"How Traditional Observability Can’t Cover AI at the Edge",[],{"_20":1830,"_90":266,"_267":1831,"_275":1853,"_277":278},"1e42fff1e07f",[1832,1836,1841,1844,1849],{"_20":1833,"_90":271,"_272":1834,"_141":1835},"c4f86ecd4a57",[],"Karani explains that as a builder in the mobile space, he initially relied on popular tools like ",{"_20":1837,"_90":271,"_272":1838,"_141":1840},"471e6fe7b6f2",[1839],"f7f739690c4b","LangFuse",{"_20":1842,"_90":271,"_272":1843,"_141":385},"4d633666c652",[],{"_20":1845,"_90":271,"_272":1846,"_141":1848},"a54412f6e2d3",[1847],"f42d14678a11","LangSmith",{"_20":1850,"_90":271,"_272":1851,"_141":1852},"ad6e4e5ad7b2",[]," for observability, but found that neither provided the monitoring coverage his projects needed. He found himself building RYNO, a flight recorder and release gate for AI in production on mobile, which focuses on privacy, safe execution traces from real devices, and turning production failures into regression tests to improve future releases.",[1854,1856],{"_20":1839,"_90":30,"_349":1855},"https://langfuse.com/",{"_20":1847,"_90":30,"_349":1857},"https://www.langchain.com/langsmith-platform",{"_20":1859,"_90":266,"_267":1860,"_275":1865,"_277":278},"4a79b7587c7b",[1861],{"_20":1862,"_90":271,"_272":1863,"_141":1864},"f0af90a0db90",[],"“The project combines the hardware state with the end AI behavior to let teams debug and figure out why model failures happen when the apps are actually in production,” the founder explains. As an example, Karani cites a team he spoke with whose mobile app that identifies plants out in the wild and diagnoses them for issues.",[],{"_20":1867,"_90":266,"_267":1868,"_275":1873,"_277":278},"f8c80b58c7b7",[1869],{"_20":1870,"_90":271,"_272":1871,"_141":1872},"756f14956665",[],"Over time, the app team noticed negative reviews in the App Store from frustrated users who found the app misidentifying plants, but when the app’s developers conducted the same tests internally, they noticed no errors. “What they didn't realize was that a lot of their users were out using their apps in high ambient temperatures. This causes a variety of issues that the existing tools just couldn't catch.”",[],{"_20":1875,"_90":266,"_267":1876,"_275":1881,"_277":278},"73f032e4b7e8",[1877],{"_20":1878,"_90":271,"_272":1879,"_141":1880},"4a033185569e",[],"“For instance, once inference is running on a phone in production, changes in thermal state, memory pressure, and available compute can change how that workload performs in production. You can see slower execution, memory-related failures, or different behavior from what you saw during testing. RYNO comes in and is able to combine the hardware state with the model output and help teams debug why their models are failing when they're out into production.”",[],{"_20":1883,"_90":266,"_267":1884,"_275":1889,"_277":312},"cc837e4097a6",[1885],{"_20":1886,"_90":271,"_272":1887,"_141":1888},"4be7b5ebf70f",[],"Closing the Loop for Mobile: Ragequitting, UX, and Regressions",[],{"_20":1891,"_90":266,"_267":1892,"_275":1897,"_277":278},"ac1a98cd7c62",[1893],{"_20":1894,"_90":271,"_272":1895,"_141":1896},"cb6169cdaa7e",[],"For consumer-level mobile applications, this type of failure can lead to the ultimate cost: Users ragequitting the app and immediately deleting it. While the founder’s aim is eventually to target optimization opportunities for any edge computing setup, he admits that mobile is his current focus. “In mobile, most apps have a high uninstall rate.”",[],{"_20":1899,"_90":266,"_267":1900,"_275":1905,"_277":278},"09e3da705404",[1901],{"_20":1902,"_90":271,"_272":1903,"_141":1904},"bb84bfa9efca",[],"To avoid grossly inaccurate outputs due to hallucinations, Karani explains that his platform runs AI jobs across five stages: Capture, explain, replay, promote, and release.",[],{"_20":1907,"_90":266,"_267":1908,"_423":424,"_425":426,"_275":1935,"_277":278},"a3c417f4e89a",[1909,1913,1917,1922,1926,1931],{"_20":1910,"_90":271,"_272":1911,"_141":1912},"14fdc7cba99a",[417],"Capture:",{"_20":1914,"_90":271,"_272":1915,"_141":1916},"aa00c58d638c",[]," Utilizing the open-source, lightweight, privacy-centric ",{"_20":1918,"_90":271,"_272":1919,"_141":1921},"62c5efc259aa",[1920],"39f4350f1285","Terra SDK",{"_20":1923,"_90":271,"_272":1924,"_141":1925},"faaca851a04b",[],", which can be embedded into mobile apps, and is built on ",{"_20":1927,"_90":271,"_272":1928,"_141":1930},"2b5210b4a320",[1929],"12ed0e79639f","open telemetry",{"_20":1932,"_90":271,"_272":1933,"_141":1934},"1cf657c9feb9",[]," to prevent vendor lock-in, and records OS version, memory pressure, thermal state, and records OS version, memory pressure, thermal state, device class, and the compute resources involved in inference, such as CPU, GPU, or Neural Engine where available",[1936,1938],{"_20":1920,"_90":30,"_349":1937},"https://github.com/tryterra/terra-react",{"_20":1929,"_90":30,"_349":1939},"https://opentelemetry.io",{"_20":1941,"_90":266,"_267":1942,"_423":424,"_425":426,"_275":1951,"_277":278},"6bfe8b8aa55b",[1943,1947],{"_20":1944,"_90":271,"_272":1945,"_141":1946},"531d8bfb8a6c",[417],"Explain:",{"_20":1948,"_90":271,"_272":1949,"_141":1950},"102c1b804637",[]," Utilizing the local macOS app’s ability to plot a visual timeline of device physics and model execution, identifying factors like correlating changes in device state with changes in model execution",[],{"_20":1953,"_90":266,"_267":1954,"_423":424,"_425":426,"_275":1963,"_277":278},"16a216eace7b",[1955,1959],{"_20":1956,"_90":271,"_272":1957,"_141":1958},"005091edab91",[417],"Replay:",{"_20":1960,"_90":271,"_272":1961,"_141":1962},"6e1f8d5da353",[]," Having captured structured metadata, template IDs, token counts, and the precise physical hardware context at the time of errors, developers can use a replay button to recreate the conditions surrounding the failure to artificially reproduce the same environmental stress for testing purposes",[],{"_20":1965,"_90":266,"_267":1966,"_423":424,"_425":426,"_275":1975,"_277":278},"64a94ff32346",[1967,1971],{"_20":1968,"_90":271,"_272":1969,"_141":1970},"c2cb193596ec",[417],"Promote:",{"_20":1972,"_90":271,"_272":1973,"_141":1974},"057867fa95b8",[]," Teams can turn a one-off production failure into a structured, repeatable evaluation",[],{"_20":1977,"_90":266,"_267":1978,"_423":424,"_425":426,"_275":1987,"_277":278},"523186127ddd",[1979,1983],{"_20":1980,"_90":271,"_272":1981,"_141":1982},"3f0388696f60",[417],"Release:",{"_20":1984,"_90":271,"_272":1985,"_141":1986},"3802f3236628",[]," From here, the platform connects to a team CI/CD pipeline to run various evaluations on different devices with that specific eval case prior to the next app release to prevent further regressions before they make it to production",[],{"_20":1989,"_90":266,"_267":1990,"_275":1995,"_277":278},"88dd59afe227",[1991],{"_20":1992,"_90":271,"_272":1993,"_141":1994},"6cf15725e554",[],"The founder adds: “A lot of these issues also come in simple OS updates. Everything was working fine, then Apple launches a new update that introduces unexpected changes. And sometimes getting that stress test when you're testing within a controlled environment is difficult. So a lot of these issues can only be caught when the AI is actually ‘out in the wild.’”",[],{"_20":1997,"_90":266,"_267":1998,"_275":2003,"_277":278},"a4d61e87b83e",[1999],{"_20":2000,"_90":271,"_272":2001,"_141":2002},"c63132c0118c",[],"“Sometimes you have to actually catch these failures once they're actually already out there because these systems are so non deterministic. They're not like traditional, deterministic software where you might have a really tight unit test case, so certain errors don’t reach production.”",[],{"_20":2005,"_90":266,"_267":2006,"_275":2011,"_277":278},"51f37cb77d65",[2007],{"_20":2008,"_90":271,"_272":2009,"_141":2010},"9098f81dfbb7",[],"“Because there's 100 million ways that the AI can actually fail. So, one of the best approaches is having this sort of loop that we've created to catch them quickly in production and ensure that they don't persist in the future.”",[],{"_20":2013,"_90":266,"_267":2014,"_275":2019,"_277":312},"6e814b0ff721",[2015],{"_20":2016,"_90":271,"_272":2017,"_141":2018},"07bb1c6dbf0f",[],"Building for Privacy and Memory Optimization",[],{"_20":2021,"_90":266,"_267":2022,"_275":2027,"_277":278},"564424a3cff6",[2023],{"_20":2024,"_90":271,"_272":2025,"_141":2026},"4c7d752edc34",[],"Karani explains that privacy was a key part of the project due to its initial focus on iOS. “We believe that the on-device AI on iOS is some of the best in the world right now. The Apple Neural Engine lets users run these really awesome models. But for iOS and mobile users, privacy is very important. It’s common even for experienced teams to accidentally violate user privacy while trying to collect the debugging context they desperately need.”",[],{"_20":2029,"_90":266,"_267":2030,"_275":2035,"_277":278},"1d35f785606e",[2031],{"_20":2032,"_90":271,"_272":2033,"_141":2034},"8a890a6f2be8",[],"“This is how teams are silently trying to capture model regression with internal tooling. They captured a prompt and its output, and those contain extremely sensitive data, so you end up violating user privacy. To fix that for RYNO, we actually have a ‘never log’ list. No screenshots, no user prompts, no contacts lists, no physical addresses, no chat transcripts.”",[],{"_20":2037,"_90":266,"_267":2038,"_275":2043,"_277":278},"846ca4d16b40",[2039],{"_20":2040,"_90":271,"_272":2041,"_141":2042},"8da126fcdaa0",[],"“We use a minimal telemetry schema. Instead of logging what the user has typed, we log a structured shape and the operational health of the mathematical transaction. We've been using a lot of tools to ensure that teams are able to get the metadata that they need to reproduce these failures while still avoiding violations of user privacy in the process. We think this is super-important for on-device AI.”",[],{"_20":2045,"_90":266,"_267":2046,"_275":2069,"_277":278},"ea0d0888f24d",[2047,2051,2056,2060,2065],{"_20":2048,"_90":271,"_272":2049,"_141":2050},"c484d592a241",[],"Aside from privacy, the founder notes that going lightweight was an important architectural consideration to run successfully on edge. “Because of the memory pressure issues with the ",{"_20":2052,"_90":271,"_272":2053,"_141":2055},"e39d37c1ea2c",[2054],"a281272b15f2","Jetson Linux",{"_20":2057,"_90":271,"_272":2058,"_141":2059},"7c27af004f44",[],", we made a key decision early on to build the core of the SDK in ",{"_20":2061,"_90":271,"_272":2062,"_141":2064},"a74babccffb6",[2063],"a0ea04ccc546","Zig",{"_20":2066,"_90":271,"_272":2067,"_141":2068},"49524083c6a5",[],". This gave us a lot of control over memory pressure and it gave us full control over how we actually want to architect memory use for the SDK, because on-device AI uses a lot of resources on mobile.”",[2070,2072],{"_20":2054,"_90":30,"_349":2071},"https://developer.nvidia.com/embedded/jetson-linux-r3643",{"_20":2063,"_90":30,"_349":2073},"https://ziglang.org/",{"_20":2075,"_90":266,"_267":2076,"_275":2081,"_277":278},"e379b830e1eb",[2077],{"_20":2078,"_90":271,"_272":2079,"_141":2080},"090dfbc5e2fc",[],"“So we needed to build an SDK that's extremely lightweight. Because we’re running an observability SDK on constrained hardware, it would need to stay lightweight enough to not create another source of memory pressure competing against the model it’s observing. We think that for edge AI, debugging is something that is going to become a big problem. And it's totally different from how current tools actually enable teams to receive telemetry.”",[],{"_20":2083,"_90":266,"_267":2084,"_275":2089,"_277":278},"d120d7d1c55e",[2085],{"_20":2086,"_90":271,"_272":2087,"_141":2088},"017f00a9c3a6",[],"The founder admits that part of his inspiration came from his own analysis of the AI space, which increasingly seems dominated by billion-dollar vendors pushing trillion-parameter models. “We believe wholeheartedly that edge AI is 100% the future. Smaller models are improving at a rapid rate. Vendors like Apple and Google keep pumping a lot of money in the space, and we think that the kind of benefits that edge AI provides are undeniable.”",[],{"_20":2091,"_90":266,"_267":2092,"_275":2097,"_277":278},"88259a07d188",[2093],{"_20":2094,"_90":271,"_272":2095,"_141":2096},"007257714f01",[],"“First, there’s privacy. Second, there’s improvements to latency, and third, is cost reduction. As the models keep improving, people are going to start realizing and taking this extremely seriously as we move forward. The space is still very early. But the world I see in the future is one in which we'll have on-device AI everywhere you go. There’s already on-device AI in cars, and we'll have robots that need on-device AI, for television, mobile, and so on.”",[],{"_20":2099,"_90":266,"_267":2100,"_275":2105,"_277":278},"fa06d141f9b1",[2101],{"_20":2102,"_90":271,"_272":2103,"_141":2104},"8c15a64289b3",[],"“Yes, the Cloud will always be where true intelligence, the larger computational intelligence, lives. But more and more companies are going to keep moving towards leveraging the on-device AI space as we move forward. And I think that's why Apple has also been putting a lot of effort year over year into the Apple Neural Engine and every chip that they release, prioritizing the inference power that they can actually pull out of the on device models.”",[],{"_20":2107,"_90":266,"_267":2108,"_275":2113,"_277":312},"bb08c1e33c67",[2109],{"_20":2110,"_90":271,"_272":2111,"_141":2112},"8225a1eccfb5",[],"Founder Lessons, Enterprise Adoption, and the Future of Compute",[],{"_20":2115,"_90":266,"_267":2116,"_275":2121,"_277":278},"0f1dca34a263",[2117],{"_20":2118,"_90":271,"_272":2119,"_141":2120},"5446dd4a7a6b",[],"For founders looking to build on-device AI products, Karani suggests that his own deep understanding of the space has been fundamental to his progress. “I think having a good understanding of how the hardware state actually works is important. The computation and the OS system attributes during inference time...that’s a different problem space that a lot of engineers need to understand if they're going into the on-device space.”",[],{"_20":2123,"_90":266,"_267":2124,"_275":2129,"_277":278},"2f49784b8a7f",[2125],{"_20":2126,"_90":271,"_272":2127,"_141":2128},"bd75e9744e7c",[],"“Inference is one thing, input and output of prompts is another. But for a lot of these models, you have to understand they're not in a problem space that is similar to running on a large data center in Atlanta with perfect resources all the time. These are models that are running on-device with varying, different pressures in constrained environments. Understanding those environments is key to actually building the right systems for AI solutions.”",[],{"_20":2131,"_90":266,"_267":2132,"_275":2137,"_277":278},"d217ab50b70d",[2133],{"_20":2134,"_90":271,"_272":2135,"_141":2136},"80ee14ec9ff8",[],"On the question of why device-specific telemetry still doesn’t seem to be mainstream at the enterprise level, the founder points back to how the tool ecosystem for edge computing is still in its early days. “There’s a lack of tooling. A lot of the enterprise companies we speak to are concerned over their lack of telemetry and understanding of how models behave when they’re actually out into production.”",[],{"_20":2139,"_90":266,"_267":2140,"_275":2145,"_277":278},"66d1e4b93387",[2141],{"_20":2142,"_90":271,"_272":2143,"_141":2144},"f3f842438cd8",[],"“For instance, we were speaking to a medical company that was trying to launch a very privacy-centric, on-device model to different customers. But what they found is that they just couldn’t risk getting an incorrect diagnosis from the model (and not being able to understand why it happened). Existing tools just didn't cater to them. The space is so early and emerging and a lot of the focus is on cloud models.”",[],{"_20":2147,"_90":266,"_267":2148,"_275":2153,"_277":278},"37063324755f",[2149],{"_20":2150,"_90":271,"_272":2151,"_141":2152},"345027e2fcbe",[],"The founder notes that while high-end cloud models get a lot of attention, they’re also currently leading to a lot of untenable costs. “I think that's what's going to push enterprises more into on-device AI. And as [on-device models] improve, the solutions will become simpler, but we need to provide them with the kind of tooling that enables them to ship on-device AI reliably to their users.”",[],{"_20":2155,"_90":266,"_267":2156,"_275":2161,"_277":278},"8e071c2e06f0",[2157],{"_20":2158,"_90":271,"_272":2159,"_141":2160},"6731a73112cc",[],"Will there be an inflection point at which everything flips from cloud to local? Maybe not. “I think people are going to have a balance of the two. Teams will leverage on-device models for certain tasks and route to cloud models when they need heavy computational work to be completed. I think the future will be a hybrid of the two. I don’t think on-device AI is going to completely negate the need for Cloud, but I think cost and performance are going to push people to try to understand how to leverage a hybrid of the two. And the majority of the work we do today will be pushed to the on-device models for inference.”",[],13,"https://cdn.sanity.io/images/50q6fr1p/production/b4042cb54ee8f7b7f4ab04864bd8e1cf7a157ff6-1200x630.png","f7ca2bf6-bf0c-4904-aa01-faede655849e","2026-08-25T16:22:00.000Z","why-on-device-inference-needs-custom-observability","Why On-Device Inference Needs Custom Observability",{"_302":2169,"_1077":2891,"_127":2892,"_1080":2893,"_1082":2894,"_1084":2895,"_1086":1734,"_32":2896,"_1088":1089},[2170,2178,2208,2216,2264,2268,2283,2291,2310,2329,2345,2353,2383,2391,2399,2407,2426,2445,2461,2469,2477,2485,2493,2501,2509,2528,2563,2571,2595,2614,2622,2630,2646,2654,2662,2677,2685,2693,2701,2720,2744,2770,2778,2805,2813,2821,2829,2837,2845,2864,2883],{"_20":2171,"_90":266,"_267":2172,"_275":2177,"_277":312},"7a24ac14d07e",[2173],{"_20":2174,"_90":271,"_272":2175,"_141":2176},"48d3443f20a8",[],"How Can Agents Be the Future if They’re not Reliable?",[],{"_20":2179,"_90":266,"_267":2180,"_275":2203,"_277":278},"ff2d4b14fb40",[2181,2185,2190,2194,2199],{"_20":2182,"_90":271,"_272":2183,"_141":2184},"4b0676b5544d",[],"Some reports suggest that as many as ",{"_20":2186,"_90":271,"_272":2187,"_141":2189},"7633cd943027",[2188],"72307ba313ce","25%",{"_20":2191,"_90":271,"_272":2192,"_141":2193},"103406d76f91",[]," of enterprises have adopted agentic workflows. Isn’t this the era of AI agents? Why isn’t the number higher? Aside from predictable enterprise-level concerns (",{"_20":2195,"_90":271,"_272":2196,"_141":2198},"01325be45d21",[2197],"c2f310fcffb7","49%",{"_20":2200,"_90":271,"_272":2201,"_141":2202},"c145585f2175",[]," of enterprises are reportedly holding off due to security concerns, for instance), agents haven’t proven to be reliable workhorses at scale, particularly for long-horizon tasks. Not yet, anyway.",[2204,2206],{"_20":2188,"_90":30,"_349":2205},"https://snyk.io/blog/agentic-development-lifecycle/",{"_20":2197,"_90":30,"_349":2207},"https://www.forrester.com/blogs/the-state-of-agentic-ai-in-2026-companies-are-chasing-few-are-catching/",{"_20":2209,"_90":266,"_267":2210,"_275":2215,"_277":278},"72359702785e",[2211],{"_20":2212,"_90":271,"_272":2213,"_141":2214},"c805aa2c35cb",[],"Agents still can’t “remember” their learnings from previous successful runs, still hallucinate when their context window becomes full, and still have a number of other issues. Wasn’t the perfect agentic future supposed to feature technical teams commanding fleets of infallible, well-trained agents to do their bidding?",[],{"_20":2217,"_90":266,"_267":2218,"_275":2257,"_277":278},"24ebf08aefeb",[2219,2223,2228,2232,2237,2240,2244,2248,2253],{"_20":2220,"_90":271,"_272":2221,"_141":2222},"0008706ad2a9",[],"Open-source creator ",{"_20":2224,"_90":271,"_272":2225,"_141":2227},"bfecdcdc2d8e",[2226],"f4b6e64c3af0","Mike Hostetler",{"_20":2229,"_90":271,"_272":2230,"_141":2231},"c55ab4189631",[]," believes such a future is possible if systems combine strong guardrails with fault-tolerant infrastructure designed for large-scale deployments. Here, he explains why he built his project, ",{"_20":2233,"_90":271,"_272":2234,"_141":2236},"1b8ae31d51e4",[2235],"b47c6c24e0ab","Jido",{"_20":2238,"_90":271,"_272":2239,"_141":1618},"4ac8dc73cc44",[],{"_20":2241,"_90":271,"_272":2242,"_141":2243},"2f226b078804",[458],"not",{"_20":2245,"_90":271,"_272":2246,"_141":2247},"7c9c7486bda0",[]," using Python, the de facto programming language for AI, but using ",{"_20":2249,"_90":271,"_272":2250,"_141":2252},"be12332c52e1",[2251],"f60c98fe6c05","Elixir",{"_20":2254,"_90":271,"_272":2255,"_141":2256},"1edf6210f9e9",[],", which was inherently designed to support maintainability and massive concurrency.",[2258,2260,2262],{"_20":2226,"_90":30,"_349":2259},"https://www.linkedin.com/in/mikehostetler/",{"_20":2235,"_90":30,"_349":2261},"https://github.com/agentjido/jido",{"_20":2251,"_90":30,"_349":2263},"https://elixir-lang.org/",{"_20":2265,"_90":127,"_92":2266},"4bd56dd08d69",{"_94":2267,"_90":96},"image-734e0114ac063680123b3471dd5f4f588375e8fa-1280x591-jpg",{"_20":2269,"_90":266,"_267":2270,"_275":2280,"_277":278},"b03c891ae9f1",[2271,2275],{"_20":2272,"_90":271,"_272":2273,"_141":2274},"0c2b5b8ee499",[458],"Mike Hostetler discusses how AI is changing the business of building software on the Software Without Borders podcast. Image courtesy ",{"_20":2276,"_90":271,"_272":2277,"_141":2279},"a2615c1217b2",[2278,458],"ea2317db32a8","Software \nWithout Borders",[2281],{"_20":2278,"_90":30,"_349":2282},"https://www.youtube.com/watch?v=zVYqx_FDTPU",{"_20":2284,"_90":266,"_267":2285,"_275":2290,"_277":312},"378419cb6538",[2286],{"_20":2287,"_90":271,"_272":2288,"_141":2289},"b95d6349db96",[],"Resolving Nondeterministic LLMs and Deterministic Algorithms",[],{"_20":2292,"_90":266,"_267":2293,"_275":2307,"_277":278},"87b25977462d",[2294,2298,2303],{"_20":2295,"_90":271,"_272":2296,"_141":2297},"2cb97e117942",[],"Hostetler, a veteran developer and founder, got his start working on the ",{"_20":2299,"_90":271,"_272":2300,"_141":2302},"5df850edc9eb",[2301],"e7748844a8dd","jQuery",{"_20":2304,"_90":271,"_272":2305,"_141":2306},"e6f957aa936c",[]," ecosystem, from which he built a career, a startup, and a lifelong curiosity about technology. He explains that his introduction to Elixir came from a tour of duty working in distributed systems, which inspired him to think through the challenge of being able to run multiple agents with real-world reliability. “I envisioned a future of ‘10,000 agents per human.’”",[2308],{"_20":2301,"_90":30,"_349":2309},"https://jquery.com",{"_20":2311,"_90":266,"_267":2312,"_275":2326,"_277":278},"0609f26003a6",[2313,2317,2322],{"_20":2314,"_90":271,"_272":2315,"_141":2316},"11d3ae480877",[],"The creator notes that his original vision was running 10,000 agents on a single ",{"_20":2318,"_90":271,"_272":2319,"_141":2321},"bd8a2854d8b0",[2320],"f82f2e7f077d","Raspberry Pi",{"_20":2323,"_90":271,"_272":2324,"_141":2325},"3efb34af94bf",[],". In other words, in a resource-constrained environment with no direct access to LLMs. Models were getting better, but capital expenditure from foundation labs continued to skyrocket.",[2327],{"_20":2320,"_90":30,"_349":2328},"https://www.raspberrypi.com/",{"_20":2330,"_90":266,"_267":2331,"_275":2344,"_277":278},"c7d5705ac245",[2332,2336,2340],{"_20":2333,"_90":271,"_272":2334,"_141":2335},"22d5ed497f6c",[],"Hostetler suggests his goal was to make the best of both worlds: Capitalize on the improving performance of state-of-the-art models, combined with performant infrastructure that wouldn’t break the bank. “The intersection of these two ideas is: Agents needed to be easy to build, easy to deploy, easy to orchestrate and coordinate. And then, 20% of your agents could use an LLM as the thinking step. But it needed to be just as easy to build 80% of your agents on what I call ",{"_20":2337,"_90":271,"_272":2338,"_141":2339},"ae3aaa793b3f",[458],"classical AI algorithms",{"_20":2341,"_90":271,"_272":2342,"_141":2343},"c8c502a31d12",[],".”",[],{"_20":2346,"_90":266,"_267":2347,"_275":2352,"_277":278},"5e457368ab04",[2348],{"_20":2349,"_90":271,"_272":2350,"_141":2351},"ed9ee37193c9",[],"The builder clarifies that ‘classic algorithms’ also include logic that predates LLMs: “Everything from a finite state machine to a behavior tree to hierarchical task networks. Video game AI as well as LLM AI. And those could work in concert and then be easily and inexpensively deployable for a user, which gets into the real pragmatic piece.”",[],{"_20":2354,"_90":266,"_267":2355,"_275":2378,"_277":278},"c843c70bed8d",[2356,2360,2365,2369,2374],{"_20":2357,"_90":271,"_272":2358,"_141":2359},"9425a3cff2d7",[],"Hostetler clarifies that despite Jido’s explicit “AI optional” clause, his approach to AI is less skeptical and more pragmatic. “I love the nondeterminism of LLMs. I love pushing those boundaries. Just by happenstance, having put my own open-source project out there, I got to know ",{"_20":2361,"_90":271,"_272":2362,"_141":2364},"9a152e0af025",[2363],"c4b6a4a077f8","Geoff Huntley",{"_20":2366,"_90":271,"_272":2367,"_141":2368},"f8941b7433c0",[]," and tried ",{"_20":2370,"_90":271,"_272":2371,"_141":2373},"b9826d2e81c5",[2372],"cfe50a2150c7","the Ralph Wiggum loop",{"_20":2375,"_90":271,"_272":2376,"_141":2377},"64bca5e784fe",[]," before it was publicly available. Loved it, and had Jido running loops.”",[2379,2381],{"_20":2363,"_90":30,"_349":2380},"https://ghuntley.com",{"_20":2372,"_90":30,"_349":2382},"https://ghuntley.com/loop/",{"_20":2384,"_90":266,"_267":2385,"_275":2390,"_277":278},"103cd73a04dd",[2386],{"_20":2387,"_90":271,"_272":2388,"_141":2389},"9d7796132c04",[],"“But I quickly observed that [with successive loops], you end up compounding quality issues and you run into these challenges.” The builder’s observations led him to start experimenting with combinations of agentic loops and classical, deterministic AI algorithms, based on a self-taught course of study.",[],{"_20":2392,"_90":266,"_267":2393,"_275":2398,"_277":278},"0291b7718207",[2394],{"_20":2395,"_90":271,"_272":2396,"_141":2397},"094ef0d4310f",[],"“I've used my computer science degree more in the last two years than I have in my entire career, which is just so much fun. So I’m not ‘anti-AI’ at all. What I want to put forward is something that lets you use both modes of tooling in the same context, instead of having a ‘developer mentality’ that requires a hard switch between ‘Here’s an agent that has an LLM as its brain’ versus an agent that has a finite state machine as its brain, and you have to build a bridge between the two. Jido makes both into first-class citizens. I knew as an engineer, as an architect, I would want both.”",[],{"_20":2400,"_90":266,"_267":2401,"_275":2406,"_277":312},"8b84eea66041",[2402],{"_20":2403,"_90":271,"_272":2404,"_141":2405},"66b38a3b5c20",[],"How Can Agents Master Long-Horizon Jobs?",[],{"_20":2408,"_90":266,"_267":2409,"_275":2423,"_277":278},"47dc737f6c1e",[2410,2414,2419],{"_20":2411,"_90":271,"_272":2412,"_141":2413},"867bc7833538",[],"Hostetler admits to reading, and speculating about, the news headlines about agents seemingly getting closer to the holy grail of succeeding on complicated, long-horizon tasks in a single, zero-shot run. “Yes, I read the headlines, and I see the ",{"_20":2415,"_90":271,"_272":2416,"_141":2418},"06d1b38bf227",[2417],"2ea1dd0e9cf9","METR reports",{"_20":2420,"_90":271,"_272":2421,"_141":2422},"8b4acaa0ac87",[],". I would say I have hypotheses about the topic, but I acknowledge the limits of my hypotheses.”",[2424],{"_20":2417,"_90":30,"_349":2425},"https://metr.org/time-horizons/",{"_20":2427,"_90":266,"_267":2428,"_275":2442,"_277":278},"8d0acac52e79",[2429,2433,2438],{"_20":2430,"_90":271,"_272":2431,"_141":2432},"2c71dc2a8678",[],"“We see new model drops like the [recalled-and-re-released] ",{"_20":2434,"_90":271,"_272":2435,"_141":2437},"45c1bbef0be9",[2436],"f0386e204d9f","Fable 5",{"_20":2439,"_90":271,"_272":2440,"_141":2441},"1fbdf1ad0724",[],", which can reportedly do a long-running task for eight hours, uninterrupted.” The creator notes that it can be hard to separate reality from hype, but notes that foundation labs may have different incentives than the rest of us.",[2443],{"_20":2436,"_90":30,"_349":2444},"https://www.anthropic.com/news/fable-mythos-access",{"_20":2446,"_90":266,"_267":2447,"_275":2460,"_277":278},"a2cf64eff7a9",[2448,2452,2456],{"_20":2449,"_90":271,"_272":2450,"_141":2451},"eaf492689fdb",[],"“I would think there is a lot of pressure to embed the capability ",{"_20":2453,"_90":271,"_272":2454,"_141":2455},"92183232c806",[458],"behind the walls",{"_20":2457,"_90":271,"_272":2458,"_141":2459},"8058aab96629",[]," of the model; to make Fable better via academic RL, or to have a lot of infrastructure running between the API call and the model itself. You see them pulling in things like their web API tools, and doing everything to make Anthropic a walled garden for any corporate customer.”",[],{"_20":2462,"_90":266,"_267":2463,"_275":2468,"_277":278},"41e706c870c8",[2464],{"_20":2465,"_90":271,"_272":2466,"_141":2467},"14797e507230",[],"Hostetler contrasts the lofty ambitions of foundation labs against his day-to-day work, which requires him to service ERP systems maintained by tech leaders across the country who need reliable performance, day in and day out.",[],{"_20":2470,"_90":266,"_267":2471,"_275":2476,"_277":278},"c04dd4c136d2",[2472],{"_20":2473,"_90":271,"_272":2474,"_141":2475},"840ee6f95b58",[],"The creator suggests starting from a strong harness, but also looking into reliability. “A really concrete step is to start with a very high-quality model harness if I'm going to build my own agent. Yes, admittedly, Elixir’s philosophy is ‘let it crash,’ but you don't really want that to happen in practice. Your focus should be: How do you recover from the crash? How do you deliver high-quality software from that?”",[],{"_20":2478,"_90":266,"_267":2479,"_275":2484,"_277":599},"5824b8efe53d",[2480],{"_20":2481,"_90":271,"_272":2482,"_141":2483},"16d38849b2d4",[],"“Elixir’s philosophy is ‘let it crash,’ but you don't really want that to happen in practice. Your focus should be: How do you recover from the crash? How do you deliver high-quality software from that?” -Mike Hostetler, Creator/Jido",[],{"_20":2486,"_90":266,"_267":2487,"_275":2492,"_277":312},"5959d9db324a",[2488],{"_20":2489,"_90":271,"_272":2490,"_141":2491},"9adc46734820",[],"Squaring the Circle: Autonomous Agents vs. Reliable Uptime",[],{"_20":2494,"_90":266,"_267":2495,"_275":2500,"_277":278},"b4fc4a69d6f3",[2496],{"_20":2497,"_90":271,"_272":2498,"_141":2499},"e958458240eb",[],"The builder recognizes the gap between LLM-powered agents that often fall down for a variety of reasons and Elixir-based systems like Discord and Pinterest that are expected to provide continuous uptime for millions of users. “Without getting too technical, this really was a key design decision of Jido itself.”",[],{"_20":2502,"_90":266,"_267":2503,"_275":2508,"_277":278},"40ebc9c9d5ad",[2504],{"_20":2505,"_90":271,"_272":2506,"_141":2507},"9b48defee678",[],"Hostetler explains that the project was designed to accommodate both builders of independent, solo agents as well as those building for coordinated, multi-agent use cases. “And also, there’s all the infrastructure concerns for when you go to deploy: Where does it live? How is it persisted? It's persisted with what I consider to be two schools of thought.”",[],{"_20":2510,"_90":266,"_267":2511,"_275":2525,"_277":278},"01a202626a6c",[2512,2516,2521],{"_20":2513,"_90":271,"_272":2514,"_141":2515},"6a69eed46b2a",[],"“The first [persistence story] is that I persist my ",{"_20":2517,"_90":271,"_272":2518,"_141":2520},"be252cd6a312",[2519],"e73f4b0caaf3","BEAM system",{"_20":2522,"_90":271,"_272":2523,"_141":2524},"888079506d54",[]," back to a durable store, like a Postgres database which is centralized. No issue there. And I would say that more than half our users use this. The other key story is what I call the fully distributed, fault-tolerant approach for scale that you get into with the really big deployments, such as WhatsApp or Facebook Messenger. Multi-region, multi-clustered...the big stuff.”",[2526],{"_20":2519,"_90":30,"_349":2527},"https://en.wikipedia.org/wiki/BEAM_(Erlang_virtual_machine)",{"_20":2529,"_90":266,"_267":2530,"_275":2560,"_277":278},"52380ad616db",[2531,2535,2539,2543,2547,2551,2556],{"_20":2532,"_90":271,"_272":2533,"_141":2534},"04e742e52dfa",[],"“Jido was geared for [large-scale deployments]. We could get into the specifics of how Jido was created to live within the BEAM itself, but it ",{"_20":2536,"_90":271,"_272":2537,"_141":2538},"7f2912ffef49",[458],"is",{"_20":2540,"_90":271,"_272":2541,"_141":2542},"f38721e57e56",[]," possible. And there are nuances here, too. People come to the project and they sometimes feel like it's overbuilt. Case in point, when you send a message into a Jido agent, we use a dedicated message envelope called a ",{"_20":2544,"_90":271,"_272":2545,"_141":2546},"360df35b7e3c",[458],"Jido signal",{"_20":2548,"_90":271,"_272":2549,"_141":2550},"51665ddc50c6",[],", which, when you're in a single-node use case is really and truly overkill. And yet it's the standard built on the ",{"_20":2552,"_90":271,"_272":2553,"_141":2555},"152bbc74099b",[2554],"0abd3bbf8418","CloudEvents spec",{"_20":2557,"_90":271,"_272":2558,"_141":2559},"ca903e998961",[]," (which is independent of, and exists outside of Elixir).”",[2561],{"_20":2554,"_90":30,"_349":2562},"https://cloudevents.io/",{"_20":2564,"_90":266,"_267":2565,"_275":2570,"_277":278},"46faea6fef5d",[2566],{"_20":2567,"_90":271,"_272":2568,"_141":2569},"e670c10a6d1b",[],"“We do this because you can't have it both ways: You can't have your typical simple JSON payload alongside a robust, enterprise-grade, transport-agnostic CloudEvents envelope and do the same thing. You've got to pick a lane. And so, Jido picked the ‘robust and scalable’ lane. I do work with some teams that are now asking me about the multi-cluster use case. I’m working toward a demo of a multi-region cluster independent of a durable store with resilient agents.”",[],{"_20":2572,"_90":266,"_267":2573,"_275":2594,"_277":278},"8a6207353488",[2574,2578,2582,2586,2590],{"_20":2575,"_90":271,"_272":2576,"_141":2577},"775967e75006",[],"“To clarify, everything I just discussed was ",{"_20":2579,"_90":271,"_272":2580,"_141":2581},"5cfaaca93e88",[458],"outside",{"_20":2583,"_90":271,"_272":2584,"_141":2585},"97a4f29363fe",[]," the agentic loop. ",{"_20":2587,"_90":271,"_272":2588,"_141":2589},"accaff227757",[458],"Within",{"_20":2591,"_90":271,"_272":2592,"_141":2593},"83672844a0e3",[]," the agentic loop of determinism versus nondeterminism, there are guardrails within which Jido has to operate. It can't just go do everything because it was meant for the Raspberry Pi use case. Because Elixir and BEAM can natively cluster, you could easily have a deploy to support a use case for something like, let’s say, Internet of Things (IoT).”",[],{"_20":2596,"_90":266,"_267":2597,"_275":2611,"_277":278},"36dc5918428c",[2598,2602,2607],{"_20":2599,"_90":271,"_272":2600,"_141":2601},"f9f53a4cd67b",[],"“For example, you’re going to have a central cluster that then works with Jido agents that are deployed in an IoT setting that talk back to the central cluster, and you'd have two agents communicating with one another. You could have that modality in a larger deployment. As an example, one of the big successes in the Elixir world is the robot control project ",{"_20":2603,"_90":271,"_272":2604,"_141":2606},"30ba83fa35f4",[2605],"74bfc2d507f3","Beam Bots",{"_20":2608,"_90":271,"_272":2609,"_141":2610},"ac05df5ad6aa",[],", which adopted Jido natively, so we’re seeing use cases for non-clustered low-level IoT all the way up to your typical Claude Code agent harness.”",[2612],{"_20":2605,"_90":30,"_349":2613},"https://github.com/beam-bots",{"_20":2615,"_90":266,"_267":2616,"_275":2621,"_277":312},"6471f5331431",[2617],{"_20":2618,"_90":271,"_272":2619,"_141":2620},"88eb6436d108",[],"How Startup Founders Should Think About Agentic Products",[],{"_20":2623,"_90":266,"_267":2624,"_275":2629,"_277":278},"b02dc08a377e",[2625],{"_20":2626,"_90":271,"_272":2627,"_141":2628},"71c2612c69e4",[],"When asked how he would advise startup founders thinking of launching agentic products, Hostetler offers a handful of suggestions. “For your first level of planning, I would say: Focus on the default. How you define what an ‘agent’ is could be colored by your Claude Code glasses! Make sure that as you're defining an agent in terms of what you bring into your infrastructure, and that you have a tighter, concrete definition of what that is.”",[],{"_20":2631,"_90":266,"_267":2632,"_275":2645,"_277":278},"322cd96a4729",[2633,2637,2641],{"_20":2634,"_90":271,"_272":2635,"_141":2636},"a7c7ebbd0554",[],"“Second, the other mistake that I've seen (and have made personally) is that everybody wants to let the LLM ‘take the wheel.’ At the point when you're designing infrastructure to build a company on, you have to draw clear boundaries between nondeterminism and determinism, and create gates to bring it back and forth. And this is ",{"_20":2638,"_90":271,"_272":2639,"_141":2640},"1ea4bce42d9f",[458],"outside of",{"_20":2642,"_90":271,"_272":2643,"_141":2644},"9510224427a1",[]," the agentic loop.”",[],{"_20":2647,"_90":266,"_267":2648,"_275":2653,"_277":278},"e80672b9d929",[2649],{"_20":2650,"_90":271,"_272":2651,"_141":2652},"61e9e3aed2a8",[],"The creator suggests that customer expectations for agentic projects have grown over time. “After a few years, we’ve seen the patterns that ‘work,’ whether it's natural language or dynamic UI, we know the patterns to create user value and what users expect. The innovation has shifted from, ‘We can do just about anything,’ to ‘The user expectations are pretty much there, so how do you deliver on that?’”",[],{"_20":2655,"_90":266,"_267":2656,"_275":2661,"_277":278},"32013237d35f",[2657],{"_20":2658,"_90":271,"_272":2659,"_141":2660},"95b20553d1ab",[],"Hostetler describes an interesting project a friend built in Elixir that effectively replicated Claude Code’s functionality, but for posting on LinkedIn, using a dedicated agent with a specific ID. “Persistence wasn't dynamically allocated, but the agent would persist, and then it would hibernate and rehydrate when you wanted to edit a piece of content.”",[],{"_20":2663,"_90":266,"_267":2664,"_275":2676,"_277":278},"3309765c1194",[2665,2669,2673],{"_20":2666,"_90":271,"_272":2667,"_141":2668},"6b38e3208d9b",[],"The project let users edit posts normally via the LinkedIn GUI, but also supported giving instructions to a coordinator agent. “The key is, he made that persistent throughout the entire experience. So, no matter which modality you were interacting with, there was a kind of predefined team structure for how the agents would interact with your content. And it just struck me as this really innovative mental leap, that any agentic products I see ",{"_20":2670,"_90":271,"_272":2671,"_141":2672},"302d71f6b276",[458],"aren't thinking big enough",{"_20":2674,"_90":271,"_272":2675,"_141":2343},"5e2c8479951c",[],[],{"_20":2678,"_90":266,"_267":2679,"_275":2684,"_277":278},"f9024ac9c07d",[2680],{"_20":2681,"_90":271,"_272":2682,"_141":2683},"090d9b48e200",[],"The creator suggests that the main challenge he’d offer to startup founders and product owners is to think bigger. “I don't see a lot of product owners thinking about [larger ideas] because they're really stuck in the mindset of, ‘Claude Code is my agent. It can do everything. I have to always use Claude Code.’ I say: ‘No, you really need to break out of that mental model.’”",[],{"_20":2686,"_90":266,"_267":2687,"_275":2692,"_277":599},"d279075f7c24",[2688],{"_20":2689,"_90":271,"_272":2690,"_141":2691},"1407b21e0ca6",[],"“It just struck me...that any agentic products I see aren't thinking big enough. I don't see a lot of product owners thinking about [larger ideas] because they're really stuck in the mindset of, ‘Claude Code is my agent. I say: ‘No, you really need to break out of that mental model.’”",[],{"_20":2694,"_90":266,"_267":2695,"_275":2700,"_277":312},"8bd22d228d02",[2696],{"_20":2697,"_90":271,"_272":2698,"_141":2699},"febccbe3dad6",[],"How to Think About Enterprise Adoption",[],{"_20":2702,"_90":266,"_267":2703,"_275":2717,"_277":278},"55d44d50b054",[2704,2708,2713],{"_20":2705,"_90":271,"_272":2706,"_141":2707},"140634326dd3",[],"When asked how he would advise ambitious startup founders looking to sell AI products to enterprises, Hostetler draws upon his own day-to-day experience. “I would say that we're early. I would acknowledge the timeline displacement, ",{"_20":2709,"_90":271,"_272":2710,"_141":2712},"60c8c804bc83",[2711],"395e4914b63e","Ray Kurzweil’s",{"_20":2714,"_90":271,"_272":2715,"_141":2716},"940cd7aa0849",[]," riff on the William Gibson quote about the future being here, just not evenly distributed.”",[2718],{"_20":2711,"_90":30,"_349":2719},"https://www.writingsbyraykurzweil.com/the-law-of-accelerating-returns",{"_20":2721,"_90":266,"_267":2722,"_275":2743,"_277":278},"69e3ce55443f",[2723,2727,2731,2735,2739],{"_20":2724,"_90":271,"_272":2725,"_141":2726},"c2cbce688847",[],"“What's happening in enterprises is that there are engineering teams who ",{"_20":2728,"_90":271,"_272":2729,"_141":2730},"8f43979ee05a",[458],"are ",{"_20":2732,"_90":271,"_272":2733,"_141":2734},"7386ba0066ca",[],"using the latest tools, and they're on that curve somewhere. But ",{"_20":2736,"_90":271,"_272":2737,"_141":2738},"80dac42e9497",[458],"the operations teams",{"_20":2740,"_90":271,"_272":2741,"_141":2742},"f3f8661263be",[]," are just starting.” The creator notes that he works with operations teams that are just starting to encounter the same challenges that engineering teams faced 18 months ago: Worrying whether AI will replace them, realizing AI likely won’t replace them, then trying to figure out how to fit into this brave new world.",[],{"_20":2745,"_90":266,"_267":2746,"_275":2767,"_277":278},"d6bdea3ba045",[2747,2751,2756,2760,2764],{"_20":2748,"_90":271,"_272":2749,"_141":2750},"6dbb3fee6f18",[],"“We're early in that phase [where people are trying to find] pragmatic use cases. The issue isn't so much the tools themselves. What's interesting to me is the layer above them, which I wrote about in ",{"_20":2752,"_90":271,"_272":2753,"_141":2755},"e44831398a0a",[2754],"ba5da2cbf968","my blog on orchestration",{"_20":2757,"_90":271,"_272":2758,"_141":2759},"3c012b7e05ac",[],". To me, orchestration is the cornerstone, because there's nothing actually orchestrating ",{"_20":2761,"_90":271,"_272":2762,"_141":2763},"c493d94c2769",[458],"the tools",{"_20":2765,"_90":271,"_272":2766,"_141":2343},"0ddedf4c18ed",[],[2768],{"_20":2754,"_90":30,"_349":2769},"https://mike-hostetler.com/blog/orchestrating-intelligence-why-every-company-will-have-a-chief-agent-officer-y4EQpjr9",{"_20":2771,"_90":266,"_267":2772,"_275":2777,"_277":278},"ffdaf5b669b5",[2773],{"_20":2774,"_90":271,"_272":2775,"_141":2776},"d9fbf1dd4f20",[],"“I think that corporations are set up in silos. Sales isn't talking to marketing, who isn't talking to ops. Ops isn't talking to product. So, the pragmatic things that we need are places to deploy AI experiments, because the thing we've emphasized internally isn't so much AI as an end state, but rather, AI as iteration.”",[],{"_20":2779,"_90":266,"_267":2780,"_275":2802,"_277":278},"86f97ec55b10",[2781,2785,2790,2794,2798],{"_20":2782,"_90":271,"_272":2783,"_141":2784},"9aad627e9643",[],"The creator recounts a story of a team he works with using Claude Code to build a dashboard. In a completely unintended consequence, Claude Code used the Google Workspace connect to deploy the dashboard to Google Sheets via ",{"_20":2786,"_90":271,"_272":2787,"_141":2789},"865280795f71",[2788],"2ed51292aa7f","Apps Script",{"_20":2791,"_90":271,"_272":2792,"_141":2793},"4e0143f6f2a2",[]," to create a public URL via, of all things, a Google Sheets backdoor. “I burst out laughing. There's an entire product ",{"_20":2795,"_90":271,"_272":2796,"_141":2797},"25776a76f3df",[458],"right there,",{"_20":2799,"_90":271,"_272":2800,"_141":2801},"8a472903f3c6",[]," in that space between a non-technical user and a model’s instincts to propose unexpected solutions.”",[2803],{"_20":2788,"_90":30,"_349":2804},"https://developers.google.com/apps-script/overview",{"_20":2806,"_90":266,"_267":2807,"_275":2812,"_277":278},"1afc27d42096",[2808],{"_20":2809,"_90":271,"_272":2810,"_141":2811},"19bdb5bfad0f",[],"Hostetler also points out a strong business opportunity in governance. “During my tour of duty in corporate, I spent two years as a VP of enterprise technology. I managed all the IT and compliance stuff for a publicly-traded company.” The creator notes that “AI governance” could easily go beyond basic compliance frameworks like SOC 2.",[],{"_20":2814,"_90":266,"_267":2815,"_275":2820,"_277":278},"8fe6ad610c6c",[2816],{"_20":2817,"_90":271,"_272":2818,"_141":2819},"9e450b53590e",[],"“Every LLM call in your organization will eventually be proxied and logged. But nobody's doing this at the moment. Think about just capturing it as one problem. Analyzing and fingerprinting.” The creator predicts that sooner than we think, organizations may need to be “AI compliant” with whatever new version of the SOC2 standard that arises.",[],{"_20":2822,"_90":266,"_267":2823,"_275":2828,"_277":278},"ef3eab6b09a3",[2824],{"_20":2825,"_90":271,"_272":2826,"_141":2827},"856e44ea47e3",[],"“Another [enterprise use case] I’d look at is insurance. Cybersecurity insurance with AI, mark my words, in the next two years, that will be a ‘thing.’ And this won’t just be your CrowdStrike-style, on-device stuff. It's all the things that get exfiltrated from what you build. Nobody's talking about that. The strategy there is very immature. Admittedly, this isn’t a very sexy space, but I think there's a lot of money to be made there.”",[],{"_20":2830,"_90":266,"_267":2831,"_275":2836,"_277":312},"8c5c46a705f3",[2832],{"_20":2833,"_90":271,"_272":2834,"_141":2835},"e55bcf804130",[],"The Future of Elixir and Durable Systems",[],{"_20":2838,"_90":266,"_267":2839,"_275":2844,"_277":278},"44f07555baa8",[2840],{"_20":2841,"_90":271,"_272":2842,"_141":2843},"83aa3f705283",[],"The creator is realistic about how his project and the greater Elixir ecosystem figure into the future of software development. “A non-Elixir shop is going to have a hard time adopting Elixir. So I can see Jido becoming an agentic sidecar that eventually makes it easy for TypeScript devs to spin up an agent with Elixir-level durability.”",[],{"_20":2846,"_90":266,"_267":2847,"_275":2861,"_277":278},"dbad934515e7",[2848,2852,2857],{"_20":2849,"_90":271,"_272":2850,"_141":2851},"ddf9d15e994c",[],"“This might be for a market segment that could be ‘one step below’ the ",{"_20":2853,"_90":271,"_272":2854,"_141":2856},"7929d2ffed1a",[2855],"f82cbe3cdbf8","Temporals",{"_20":2858,"_90":271,"_272":2859,"_141":2860},"82c9fc0d6074",[]," of the world. To clarify, I’ve used Temporal before, and it’s fantastic, though expensive. But thinking pragmatically, there are a lot more TypeScript devs in the world. Python will be its own thing. TypeScript devs are going to need an SDK to easily call Jido agents, and then Jido agents will operate async, and it'll just be self-contained with a dashboard to spin it up.”",[2862],{"_20":2855,"_90":30,"_349":2863},"https://temporal.io/",{"_20":2865,"_90":266,"_267":2866,"_275":2880,"_277":278},"b3f069861ce0",[2867,2871,2876],{"_20":2868,"_90":271,"_272":2869,"_141":2870},"7dc862f60a3b",[],"The creator acknowledges that popularly-used languages and frameworks like TypeScript and ",{"_20":2872,"_90":271,"_272":2873,"_141":2875},"a5ab9089e32a",[2874],"cc95782eb602","Node.js",{"_20":2877,"_90":271,"_272":2878,"_141":2879},"05d94911aeac",[]," are excellent for what they are, and that they solve specific problems. “But you need to be able to spin things up inexpensively that companies built on TypeScript can solve either by running in Docker via their infrastructure, or run out of a cloud where they can spin up platforms of agents, with one agent per customer (or dozens of agents per customer).”",[2881],{"_20":2874,"_90":30,"_349":2882},"http://node.js",{"_20":2884,"_90":266,"_267":2885,"_275":2890,"_277":278},"1c459bacc235",[2886],{"_20":2887,"_90":271,"_272":2888,"_141":2889},"4773952f0d8b",[],"“And to be able to do that practically is just going to be overwhelmingly expensive, even with Cloud costs. And you need to be able to stitch it all together, potentially using non-deterministic agents with deterministic agents to deliver a strong product experience. It’s a very early picture, but that's where I see the market now.”",[],19,"https://cdn.sanity.io/images/50q6fr1p/production/a119969ab98a18b0937559865cbc6ee4dee53c2d-1200x630.png","39be2ceb-4600-40c4-80a9-dc0c7b9504dd","2026-08-04T16:24:00.000Z","how-to-make-agents-durable-for-concurrent-systems","How to Make Agents Durable for Concurrent Systems","modules","moreFromLibrary",[2900,2902,2904],{"_20":2901,"_94":1081,"_90":96},"VSf4o39TUkfm",{"_20":2903,"_94":1102,"_90":96},"fFYnhAouKczT",{"_20":2905,"_94":1119,"_90":96},"Gin6nrrRRQRt","outlines",[2908],{"_20":2909,"_90":266,"_267":2910,"_275":2915,"_277":278},"eb954975585b",[2911],{"_20":2912,"_90":271,"_272":2913,"_141":2914},"46b6413f3260",[],"

Minimizing Human Intervention in Incident Response\n
- Which Future We Mean
- Automation Comes First
- Distributed Systems Are a Fluid
- Golden Paths and Paved Roads
- Sharing the On-Call Burden
- What Already Works
- Legacy Is Here to Stay
- Ready for the Future
\n

",[],"2022-11-17T18:05:00.000Z","seo",{"_90":2917,"_13":2919,"_2920":161,"_2921":161,"_32":2922,"_2923":295,"_125":2924},"PagerDuty DevOps Advocate Mandi Walls on automating first response, sustainable on-call rotations, paved roads, and post-incident reviews, from DevGuild: Incident Response.","nofollow","noindex","Minimizing Human Intervention in Incident Response w/ PagerDuty's Mandi Walls","OGdescription","https://image.mux.com/fj2dQK9fStwL6He1t2ndCDv02kVME500Ef/thumbnail.png","slideDeck",{"_90":1084,"_2927":2928},"current","incident-response-automation","speakers","speakersWithOrg",[2932],{"_20":2933,"_90":2934,"_2935":-5,"_2936":2937},"spk1mandi","speaker","organization","person",{"_239":2938,"_241":2939,"_243":2940,"_2941":2942,"_90":2936,"_246":2948,"_2949":2950,"_2959":138,"_282":746,"_2935":2960,"_3016":3017,"_3020":3021,"_2917":3022,"_1084":3023,"_3025":3026,"_3034":161},"2022-10-24T16:24:47Z","2fa91475-d6d0-43ee-92da-8f0ced726c4c","mp6XYBI2Ev1Nw6ci78XFNS","_system",{"_2943":2944},"base",{"_2945":2939,"_2946":2947},"id","rev","4Invl7jK0ymRlKBtu4th3k","2026-09-23T20:55:08Z","bio",[2951],{"_20":2952,"_90":266,"_267":2953,"_275":2958,"_277":278},"059520a85f55",[2954],{"_20":2955,"_90":271,"_272":2956,"_141":2957},"0bd9c867bee20",[],"For the past several years, Mandi has concentrated on technical communities within DevOps. Everyone has their own definition of DevOps and what they want or need out of this reevaluation of technology workflows. For Mandi, DevOps has meant thinking about the lifecycle of applications beyond a successful code checkin: how does it perform? how does it interact? how do your users *really* use what you've developed? With a more holistic approach, we can embrace security concerns, business logic, customer success, and other stakeholder needs.",[],"generate",{"_239":2961,"_241":2962,"_243":2963,"_2941":2964,"_90":2935,"_246":2967,"_2968":2969,"_2976":2977,"_13":2985,"_2959":138,"_2986":2987,"_30":2988,"_2989":2990,"_2993":2994,"_282":2995,"_2996":138,"_2997":2998,"_2917":3007,"_1084":3008,"_3010":3011,"_3012":3013,"_3014":3015},"2022-04-29T20:41:45Z","recf6aYZrIOdTKDMZ","qjAKFm8zjdcUeuDjprdx0E",{"_2943":2965},{"_2945":2962,"_2946":2966},"MFtZJMAHHa7XPGTVf7HYUV","2026-09-11T23:38:56Z","about",[2970],{"_20":2968,"_90":266,"_267":2971,"_275":2975,"_277":278},[2972],{"_20":2968,"_90":271,"_272":2973,"_141":2974},[],"PagerDuty is an operations performance platform delivering visibility and actionable intelligence across the entire incident lifecycle. Their SaaS-based solution empowers small, mid-size and enterprise global customers with the insight to intelligently respond to critical disruptions for exceptional customer experience. They provide reliable notifications, automatic escalations, on-call scheduling, and other functionality to help teams detect and address unplanned work in real-time.\n",[],"altLogo",{"_90":91,"_2978":2979,"_92":2980,"_2982":2983,"_2984":2979},"alt","Logo of PagerDuty",{"_94":2981,"_90":96},"image-e7b8df6f3530912209aa0d427cf75e926cff1896-975x200-svg","attribution","","caption","real-time incidence response","joined","2013-05-20","https://www.pagerduty.com","logo",{"_90":91,"_2978":2979,"_92":2991,"_2982":2983,"_2984":2979},{"_94":2992,"_90":96},"image-3b2c3f410674c0475b6392b1a5edc8326f2706af-975x200-svg","memberNumber",5,"PagerDuty","portfolioCompany","resources",[2999,3003],{"_20":3000,"_90":3001,"_28":3002,"_30":2988,"_32":3002},"website","resource","Website",{"_20":3004,"_90":3001,"_28":3005,"_30":3006,"_32":3005},"2c5b11c0966b","Work at PagerDuty","https://careers.pagerduty.com/home",{"_90":2917,"_13":2995,"_2920":161,"_2921":161,"_32":2995},{"_90":1084,"_2927":3009},"pagerduty","stage","public","status","ipo","tags",[],"photo",{"_90":91,"_92":3018},{"_94":3019,"_90":96},"image-367588ba9a2c9a55a6fa190e72587dd71dfdca91-2118x1562-png","position","DevOps Advocate",{"_90":2917,"_13":2957,"_2920":161,"_2921":161,"_32":746},{"_90":1084,"_2927":3024},"mandi-walls","socials",[3027,3031],{"_20":3028,"_90":84,"_30":3029,"_282":3030},"f9d50237a9cc","https://twitter.com/lnxchk","twitter",{"_20":3032,"_90":84,"_30":753,"_282":3033},"0e8d84d449f3","linkedin","teamMember","startDate",[3037,3043,3050],{"_239":3038,"_241":3039,"_243":3040,"_90":3041,"_246":3038,"_282":3042},"2022-08-23T19:28:33Z","7e93d635-2341-4db2-9cb6-141475b9232c","410YivtbVgNW6RgQxPP59n","tag","Incident Response",{"_239":3044,"_241":3045,"_243":3046,"_90":3041,"_246":3044,"_13":2983,"_282":3047,"_1084":3048},"2022-09-09T00:17:13Z","sGpx8ZAtjp3y9Udp02ELau","TfMkf5gVJ7UWv1vMi5R04L","Site Reliability Engineer (SRE)",{"_90":1084,"_2927":3049},"site-reliability-engineer-(sre)",{"_239":3051,"_241":3052,"_243":3053,"_90":3041,"_246":3051,"_13":2983,"_282":3054,"_1084":3055},"2022-09-09T00:13:21Z","v6woxsuuKgC7emJ9vwJKve","aeN0todGvIjkWyTywFzd0a","DevOps",{"_90":1084,"_2927":3056},"devops","Minimizing Human Intervention in Incident Response","transcript",[3060,3114,3122,3163,3194,3225,3233,3254,3285,3336,3367,3408,3449,3500,3531,3582,3613,3644,3685,3736,3744,3775,3806,3827,3868,3889,3940,3961,3992,4023,4064,4072,4103,4144,4185,4216,4237,4258,4289,4320,4361,4392,4413,4421,4452,4483,4514,4545,4596,4637,4658,4699,4707,4738,4769,4790,4811,4852,4893,4934,4975,5006,5014,5065,5106,5147,5188,5229,5270,5291,5299,5310,5341],{"_20":3061,"_90":266,"_267":3062,"_275":3101,"_277":278},"0060b4936170",[3063,3068,3072,3077,3080,3085,3088,3093,3096],{"_20":3064,"_90":271,"_272":3065,"_141":3067},"09289e4a2c38",[3066],"c104b36","All right, thanks for joining me for this session.",{"_20":3069,"_90":271,"_272":3070,"_141":3071},"8db4d6475cb9",[]," ",{"_20":3073,"_90":271,"_272":3074,"_141":3076},"e298a548b533",[3075],"f6a2050","I am Mandi Walls, DevOps Advocate at PagerDuty.",{"_20":3078,"_90":271,"_272":3079,"_141":3071},"329ce198e4e0",[],{"_20":3081,"_90":271,"_272":3082,"_141":3084},"ba6ad9e2fa39",[3083],"74a47d6","If you want to get in touch with me, I'm LNXCHK, currently still on Twitter, and I'm MWalls at PagerDuty.com.",{"_20":3086,"_90":271,"_272":3087,"_141":3071},"914744571075",[],{"_20":3089,"_90":271,"_272":3090,"_141":3092},"6cfb9bbe2cf7",[3091],"0a8aaf2","I'm going to be talking about minimizing the human intervention in incident response and sort of how we can get there",{"_20":3094,"_90":271,"_272":3095,"_141":3071},"209da59c4ff4",[],{"_20":3097,"_90":271,"_272":3098,"_141":3100},"1a327a60c868",[3099],"c117ec9","and a bit about where things are now and what we're doing well.",[3102,3106,3108,3110,3112],{"_20":3066,"_90":3103,"_3104":3105},"timestamp","time","00:00:00.480",{"_20":3075,"_90":3103,"_3104":3107},"00:00:03.120",{"_20":3083,"_90":3103,"_3104":3109},"00:00:06.640",{"_20":3091,"_90":3103,"_3104":3111},"00:00:12.960",{"_20":3099,"_90":3103,"_3104":3113},"00:00:21.720",{"_20":3115,"_90":266,"_267":3116,"_275":3121,"_277":312},"c650e6db2842",[3117],{"_20":3118,"_90":271,"_272":3119,"_141":3120},"d086bda550b0",[],"Which Future We Mean",[],{"_20":3123,"_90":266,"_267":3124,"_275":3154,"_277":278},"45e0c0f1de68",[3125,3130,3133,3138,3141,3146,3149],{"_20":3126,"_90":271,"_272":3127,"_141":3129},"25e596baf790",[3128],"39a2826","So if I'm talking about the future, thinking about how that's going to impact incident response, I want to kind of set a baseline for",{"_20":3131,"_90":271,"_272":3132,"_141":3071},"9a394288cf05",[],{"_20":3134,"_90":271,"_272":3135,"_141":3137},"52c85543c34f",[3136],"918123c","what future I'm actually talking about.",{"_20":3139,"_90":271,"_272":3140,"_141":3071},"454a781a6d6e",[],{"_20":3142,"_90":271,"_272":3143,"_141":3145},"4b32d1c757e8",[3144],"1c4a2a9","If you're listening to this talk, there's a pretty good chance that the things you're doing now are the future for folks who work in large enterprises or government agencies",{"_20":3147,"_90":271,"_272":3148,"_141":3071},"b66e0d4d4b27",[],{"_20":3150,"_90":271,"_272":3151,"_141":3153},"6d3fda29b6ee",[3152],"4b435c1","or conservative industries like, say, manufacturing, where things move at a slower pace. And that's okay, right?",[3155,3157,3159,3161],{"_20":3128,"_90":3103,"_3104":3156},"00:00:25.640",{"_20":3136,"_90":3103,"_3104":3158},"00:00:37.559",{"_20":3144,"_90":3103,"_3104":3160},"00:00:40.359",{"_20":3152,"_90":3103,"_3104":3162},"00:00:52.339",{"_20":3164,"_90":266,"_267":3165,"_275":3187,"_277":278},"d00be9449694",[3166,3171,3174,3179,3182],{"_20":3167,"_90":271,"_272":3168,"_141":3170},"cf7d10eabc4d",[3169],"3fc257d","We don't all need to be at the same stage of the journey, but we kind of need to know that there are sort of multiple trains on the tracks. So",{"_20":3172,"_90":271,"_272":3173,"_141":3071},"c4560c190d16",[],{"_20":3175,"_90":271,"_272":3176,"_141":3178},"2d3ccf39c96c",[3177],"0a9ee04","bifurcate a little bit and talk about the future sort of for both, for folks that are more the tip of the spear and the folks who are following along.",{"_20":3180,"_90":271,"_272":3181,"_141":3071},"fd05039da692",[],{"_20":3183,"_90":271,"_272":3184,"_141":3186},"26c68fd5fbbf",[3185],"f64f9fb","Hopefully across all of those companies, everyone wants to build the best products for their users and their customers.",[3188,3190,3192],{"_20":3169,"_90":3103,"_3104":3189},"00:01:00.820",{"_20":3177,"_90":3103,"_3104":3191},"00:01:09.780",{"_20":3185,"_90":3103,"_3104":3193},"00:01:19.820",{"_20":3195,"_90":266,"_267":3196,"_275":3218,"_277":278},"76321b10e221",[3197,3202,3205,3210,3213],{"_20":3198,"_90":271,"_272":3199,"_141":3201},"e19e37cebe4d",[3200],"f3885b8","We've come to accept though that at least in some places, doing that, building that amazing customer experience, means that some folks are going to be taking on incident responsibilities",{"_20":3203,"_90":271,"_272":3204,"_141":3071},"d95e66d45939",[],{"_20":3206,"_90":271,"_272":3207,"_141":3209},"921ef4a70456",[3208],"21147a8","and a lot more ownership of the operational performance of the components they build.",{"_20":3211,"_90":271,"_272":3212,"_141":3071},"143e50c7368e",[],{"_20":3214,"_90":271,"_272":3215,"_141":3217},"92277ff9a077",[3216],"311acfa","That means different things for different people and different kinds of teams, right?",[3219,3221,3223],{"_20":3200,"_90":3103,"_3104":3220},"00:01:27.920",{"_20":3208,"_90":3103,"_3104":3222},"00:01:39.540",{"_20":3216,"_90":3103,"_3104":3224},"00:01:45.860",{"_20":3226,"_90":266,"_267":3227,"_275":3232,"_277":312},"b6a9b6107742",[3228],{"_20":3229,"_90":271,"_272":3230,"_141":3231},"316858cf8d4b",[],"Automation Comes First",[],{"_20":3234,"_90":266,"_267":3235,"_275":3249,"_277":278},"f8e8e8e5d914",[3236,3241,3244],{"_20":3237,"_90":271,"_272":3238,"_141":3240},"f75c64da1ccf",[3239],"315db9b","So something we can all be looking at though is automation.",{"_20":3242,"_90":271,"_272":3243,"_141":3071},"dfaf5f6d68d7",[],{"_20":3245,"_90":271,"_272":3246,"_141":3248},"1814290a4d11",[3247],"49216e3","The future of incident response includes a lot less human intervention across all kinds of incidents.",[3250,3252],{"_20":3239,"_90":3103,"_3104":3251},"00:01:51.740",{"_20":3247,"_90":3103,"_3104":3253},"00:01:56.700",{"_20":3255,"_90":266,"_267":3256,"_275":3278,"_277":278},"b1d9d2346a14",[3257,3262,3265,3270,3273],{"_20":3258,"_90":271,"_272":3259,"_141":3261},"4a974170e05a",[3260],"d17b731","In everyone's future is going to be automation.",{"_20":3263,"_90":271,"_272":3264,"_141":3071},"4d4565e2d069",[],{"_20":3266,"_90":271,"_272":3267,"_141":3269},"bf7d9a92feff",[3268],"62b558e","Capital A, Automation. We're used to seeing automation mentioned.",{"_20":3271,"_90":271,"_272":3272,"_141":3071},"160536a937c3",[],{"_20":3274,"_90":271,"_272":3275,"_141":3277},"4235cf554b2a",[3276],"9a4743e","In some industries it's lauded, in other industries it's reviled.",[3279,3281,3283],{"_20":3260,"_90":3103,"_3104":3280},"00:02:04.759",{"_20":3268,"_90":3103,"_3104":3282},"00:02:07.640",{"_20":3276,"_90":3103,"_3104":3284},"00:02:11.080",{"_20":3286,"_90":266,"_267":3287,"_275":3325,"_277":278},"0514d51efa91",[3288,3293,3296,3301,3304,3309,3312,3317,3320],{"_20":3289,"_90":271,"_272":3290,"_141":3292},"d574cf2a06b3",[3291],"ea587b4","But if we're living in a digitally driven environment, automation is how we keep our heads above water, right?",{"_20":3294,"_90":271,"_272":3295,"_141":3071},"2e26afc27faf",[],{"_20":3297,"_90":271,"_272":3298,"_141":3300},"d62d4d6fae9a",[3299],"f374f73","The things around us are gaining complexity every day, so we have to stay ahead of the game.",{"_20":3302,"_90":271,"_272":3303,"_141":3071},"c4d4b93dff6b",[],{"_20":3305,"_90":271,"_272":3306,"_141":3308},"d69c00bab596",[3307],"cebe6e0","So if we put automation in the context of incident response, one of the things that PagerDuty can do,",{"_20":3310,"_90":271,"_272":3311,"_141":3071},"69c19f8e6d1b",[],{"_20":3313,"_90":271,"_272":3314,"_141":3316},"6b50230d4d72",[3315],"6f11bf2","since it's a SaaS platform, is take a look at how users behave across the platform.",{"_20":3318,"_90":271,"_272":3319,"_141":3071},"f4faeb6655e7",[],{"_20":3321,"_90":271,"_272":3322,"_141":3324},"3c53394158f8",[3323],"5176c99","What kinds of workflows they build, what kinds of features they're using, all these cohorts and all this nice slice and dice.",[3326,3328,3330,3332,3334],{"_20":3291,"_90":3103,"_3104":3327},"00:02:15.180",{"_20":3299,"_90":3103,"_3104":3329},"00:02:22.860",{"_20":3307,"_90":3103,"_3104":3331},"00:02:28.040",{"_20":3315,"_90":3103,"_3104":3333},"00:02:36.120",{"_20":3323,"_90":3103,"_3104":3335},"00:02:41.960",{"_20":3337,"_90":266,"_267":3338,"_275":3360,"_277":278},"3bc6636861e7",[3339,3344,3347,3352,3355],{"_20":3340,"_90":271,"_272":3341,"_141":3343},"3a202b29a6f3",[3342],"6f249c1","And something that has jumped out",{"_20":3345,"_90":271,"_272":3346,"_141":3071},"32da80209391",[],{"_20":3348,"_90":271,"_272":3349,"_141":3351},"e5a500cf12f1",[3350],"6a879bf","for us recently is that there is an astounding number of incidents that are triggered, notify a human responder, and are resolved in under five minutes.",{"_20":3353,"_90":271,"_272":3354,"_141":3071},"5e8a1a03e3f2",[],{"_20":3356,"_90":271,"_272":3357,"_141":3359},"1909094d2bc9",[3358],"9539896","And that's an incredible waste of time for anyone.",[3361,3363,3365],{"_20":3342,"_90":3103,"_3104":3362},"00:02:48.120",{"_20":3350,"_90":3103,"_3104":3364},"00:02:50.940",{"_20":3358,"_90":3103,"_3104":3366},"00:03:02.580",{"_20":3368,"_90":266,"_267":3369,"_275":3399,"_277":278},"55b7dbadaab0",[3370,3375,3378,3383,3386,3391,3394],{"_20":3371,"_90":271,"_272":3372,"_141":3374},"6c6f8b648dd6",[3373],"825c83c","A five-minute resolution isn't an incident that needs deep triage.",{"_20":3376,"_90":271,"_272":3377,"_141":3071},"9a2a153aa615",[],{"_20":3379,"_90":271,"_272":3380,"_141":3382},"c752e8591cc7",[3381],"7cf22d4","It's not even that much time to check monitors or telemetry or your traces.",{"_20":3384,"_90":271,"_272":3385,"_141":3071},"ca69dd724a3f",[],{"_20":3387,"_90":271,"_272":3388,"_141":3390},"d0e723ce8853",[3389],"c7fd30e","There are known errors, known problems.",{"_20":3392,"_90":271,"_272":3393,"_141":3071},"b2213a7a468b",[],{"_20":3395,"_90":271,"_272":3396,"_141":3398},"7a2053e98f4e",[3397],"21febb0","What's worse, it could have been something transient, and the notification to human responders should have been suppressed.",[3400,3402,3404,3406],{"_20":3373,"_90":3103,"_3104":3401},"00:03:05.540",{"_20":3381,"_90":3103,"_3104":3403},"00:03:11.620",{"_20":3389,"_90":3103,"_3104":3405},"00:03:16.900",{"_20":3397,"_90":3103,"_3104":3407},"00:03:19.860",{"_20":3409,"_90":266,"_267":3410,"_275":3440,"_277":278},"79fd4b50e2ed",[3411,3416,3419,3424,3427,3432,3435],{"_20":3412,"_90":271,"_272":3413,"_141":3415},"dbe17cab5470",[3414],"9c2da4a","So why is it still going to a human?",{"_20":3417,"_90":271,"_272":3418,"_141":3071},"3fb405395f2a",[],{"_20":3420,"_90":271,"_272":3421,"_141":3423},"a148da980b03",[3422],"edb9348","Why are we dragging people away from their work or their beds or their kids' birthday party",{"_20":3425,"_90":271,"_272":3426,"_141":3071},"b7a8ec294d37",[],{"_20":3428,"_90":271,"_272":3429,"_141":3431},"2fc5037d834a",[3430],"ea2b8c0","for an alert we don't need their human brain for?",{"_20":3433,"_90":271,"_272":3434,"_141":3071},"1d2fd4f36471",[],{"_20":3436,"_90":271,"_272":3437,"_141":3439},"4e9d5edc114f",[3438],"16df54c","And why are organizations wasting this most expensive resource, these human employees, on problems that don't require them?",[3441,3443,3445,3447],{"_20":3414,"_90":3103,"_3104":3442},"00:03:28.120",{"_20":3422,"_90":3103,"_3104":3444},"00:03:30.280",{"_20":3430,"_90":3103,"_3104":3446},"00:03:35.400",{"_20":3438,"_90":3103,"_3104":3448},"00:03:38.140",{"_20":3450,"_90":266,"_267":3451,"_275":3489,"_277":278},"e31c718762f5",[3452,3457,3460,3465,3468,3473,3476,3481,3484],{"_20":3453,"_90":271,"_272":3454,"_141":3456},"e19f8ef230b5",[3455],"cf2e1a2","So automation is going to help us get out in front of these low-value alerts, help us prevent them, and help us get to a platform for the next generation of operations.",{"_20":3458,"_90":271,"_272":3459,"_141":3071},"60e7f5bf61c8",[],{"_20":3461,"_90":271,"_272":3462,"_141":3464},"062000e1ef8d",[3463],"3393836","What is the next generation of operations?",{"_20":3466,"_90":271,"_272":3467,"_141":3071},"c796af3ec45f",[],{"_20":3469,"_90":271,"_272":3470,"_141":3472},"28b4e942ef68",[3471],"1952414","Well, analysts are telling us that AIOps is next.",{"_20":3474,"_90":271,"_272":3475,"_141":3071},"0d88d74956fb",[],{"_20":3477,"_90":271,"_272":3478,"_141":3480},"631daac4c813",[3479],"8ea4139","But thinking about incorporating machine learning solutions",{"_20":3482,"_90":271,"_272":3483,"_141":3071},"1938e6556e44",[],{"_20":3485,"_90":271,"_272":3486,"_141":3488},"436055ae2bc4",[3487],"e66f45d","into your incident response process means that somewhere there needs to be a huge corpus of data to train that AI with.",[3490,3492,3494,3496,3498],{"_20":3455,"_90":3103,"_3104":3491},"00:03:46.140",{"_20":3463,"_90":3103,"_3104":3493},"00:03:56.879",{"_20":3471,"_90":3103,"_3104":3495},"00:03:59.280",{"_20":3479,"_90":3103,"_3104":3497},"00:04:03.599",{"_20":3487,"_90":3103,"_3104":3499},"00:04:07.440",{"_20":3501,"_90":266,"_267":3502,"_275":3524,"_277":278},"09a34185bc1a",[3503,3508,3511,3516,3519],{"_20":3504,"_90":271,"_272":3505,"_141":3507},"2fecd602a969",[3506],"31b9744","Unfortunately, we're years away from having that volume of useful data for any specific production ecosystem.",{"_20":3509,"_90":271,"_272":3510,"_141":3071},"e3f31128e3b7",[],{"_20":3512,"_90":271,"_272":3513,"_141":3515},"a7a491abf928",[3514],"d31620f","For now, AIOps is really good at scratching the surface, gathering up well-defined data in certain constrained categories.",{"_20":3517,"_90":271,"_272":3518,"_141":3071},"3c032e5d2452",[],{"_20":3520,"_90":271,"_272":3521,"_141":3523},"f40cd1d98d71",[3522],"9d01ff5","Things like alerts. It's really good at producing rules for placing like with like.",[3525,3527,3529],{"_20":3506,"_90":3103,"_3104":3526},"00:04:17.640",{"_20":3514,"_90":3103,"_3104":3528},"00:04:25.760",{"_20":3522,"_90":3103,"_3104":3530},"00:04:35.220",{"_20":3532,"_90":266,"_267":3533,"_275":3571,"_277":278},"3ab39f453f68",[3534,3539,3542,3547,3550,3555,3558,3563,3566],{"_20":3535,"_90":271,"_272":3536,"_141":3538},"257e3d77bdbb",[3537],"0689e91","So AI won't be helping us anytime soon",{"_20":3540,"_90":271,"_272":3541,"_141":3071},"4a45f6c235bb",[],{"_20":3543,"_90":271,"_272":3544,"_141":3546},"491261028008",[3545],"a847981","with the intricacies to fix our services and our environment.",{"_20":3548,"_90":271,"_272":3549,"_141":3071},"61dff1cd47b0",[],{"_20":3551,"_90":271,"_272":3552,"_141":3554},"2c4b0eb2b2bf",[3553],"6d36ab6","It has to learn how to get things done.",{"_20":3556,"_90":271,"_272":3557,"_141":3071},"5f4a8acdb115",[],{"_20":3559,"_90":271,"_272":3560,"_141":3562},"749ddab0e5c8",[3561],"d0ac357","How are we going to train that algorithm in the future?",{"_20":3564,"_90":271,"_272":3565,"_141":3071},"6569cc113452",[],{"_20":3567,"_90":271,"_272":3568,"_141":3570},"b1111c490d2c",[3569],"a96d888","We're going to pave the way for it with strong, comprehensive automation.",[3572,3574,3576,3578,3580],{"_20":3537,"_90":3103,"_3104":3573},"00:04:41.300",{"_20":3545,"_90":3103,"_3104":3575},"00:04:45.560",{"_20":3553,"_90":3103,"_3104":3577},"00:04:48.840",{"_20":3561,"_90":3103,"_3104":3579},"00:04:52.040",{"_20":3569,"_90":3103,"_3104":3581},"00:04:56.940",{"_20":3583,"_90":266,"_267":3584,"_275":3606,"_277":599},"270f4e5928b5",[3585,3590,3593,3598,3601],{"_20":3586,"_90":271,"_272":3587,"_141":3589},"4954485ab476",[3588],"f211ac8","Our job is to get the human component out of situations where we already know what's wrong and how to fix it.",{"_20":3591,"_90":271,"_272":3592,"_141":3071},"f9e282148a2d",[],{"_20":3594,"_90":271,"_272":3595,"_141":3597},"c715145cff93",[3596],"4240ff6","In those cases, the truth is the humans are slowing things down.",{"_20":3599,"_90":271,"_272":3600,"_141":3071},"83a1622270cb",[],{"_20":3602,"_90":271,"_272":3603,"_141":3605},"445987944301",[3604],"1861c76","They are delaying the resolution of incidents.",[3607,3609,3611],{"_20":3588,"_90":3103,"_3104":3608},"00:05:01.659",{"_20":3596,"_90":3103,"_3104":3610},"00:05:10.199",{"_20":3604,"_90":3103,"_3104":3612},"00:05:14.919",{"_20":3614,"_90":266,"_267":3615,"_275":3637,"_277":278},"404d783219ce",[3616,3621,3624,3629,3632],{"_20":3617,"_90":271,"_272":3618,"_141":3620},"104f27fd635e",[3619],"4a9c70a","So we automate as much as you can when it comes to how your team is handling what we call unplanned work.",{"_20":3622,"_90":271,"_272":3623,"_141":3071},"117f6195e193",[],{"_20":3625,"_90":271,"_272":3626,"_141":3628},"50073439e936",[3627],"2556b35","Automating the collection of diagnostic logs, the first attempted restart, things like allocating a little bit more resources.",{"_20":3630,"_90":271,"_272":3631,"_141":3071},"6d58c1e2e9b3",[],{"_20":3633,"_90":271,"_272":3634,"_141":3636},"74630773be37",[3635],"9b8d108","We can automate all those kinds of things.",[3638,3640,3642],{"_20":3619,"_90":3103,"_3104":3639},"00:05:18.500",{"_20":3627,"_90":3103,"_3104":3641},"00:05:25.220",{"_20":3635,"_90":3103,"_3104":3643},"00:05:32.760",{"_20":3645,"_90":266,"_267":3646,"_275":3676,"_277":278},"64e3f64463c9",[3647,3652,3655,3660,3663,3668,3671],{"_20":3648,"_90":271,"_272":3649,"_141":3651},"853b369d251c",[3650],"65af2f2","When human creativity is needed, all the first tasks are done.",{"_20":3653,"_90":271,"_272":3654,"_141":3071},"643656cc757b",[],{"_20":3656,"_90":271,"_272":3657,"_141":3659},"bb7d26ef31ca",[3658],"cc72132","The human responder joins the call.",{"_20":3661,"_90":271,"_272":3662,"_141":3071},"9e1c8ea5fe7f",[],{"_20":3664,"_90":271,"_272":3665,"_141":3667},"bcd863815981",[3666],"4586a42","It's like a luxury response experience.",{"_20":3669,"_90":271,"_272":3670,"_141":3071},"4dcd597ebe54",[],{"_20":3672,"_90":271,"_272":3673,"_141":3675},"d83f82db96ca",[3674],"09cfb3c","Everything they need is right there, and you save the first minutes of that human triage.",[3677,3679,3681,3683],{"_20":3650,"_90":3103,"_3104":3678},"00:05:35.080",{"_20":3658,"_90":3103,"_3104":3680},"00:05:40.400",{"_20":3666,"_90":3103,"_3104":3682},"00:05:42.639",{"_20":3674,"_90":3103,"_3104":3684},"00:05:44.880",{"_20":3686,"_90":266,"_267":3687,"_275":3725,"_277":278},"9b82899c5d07",[3688,3693,3696,3701,3704,3709,3712,3717,3720],{"_20":3689,"_90":271,"_272":3690,"_141":3692},"e1760442b54e",[3691],"fb1b2ee","The challenge, unfortunately, for automation in many organizations isn't even technical.",{"_20":3694,"_90":271,"_272":3695,"_141":3071},"cb4b2c5653ba",[],{"_20":3697,"_90":271,"_272":3698,"_141":3700},"cbc51a513d59",[3699],"27a1851","It's cultural and often political. Managers fretting about loss of control or influence, individual contributors worried about having obsolete skills.",{"_20":3702,"_90":271,"_272":3703,"_141":3071},"242831ece785",[],{"_20":3705,"_90":271,"_272":3706,"_141":3708},"0cd177cafc3c",[3707],"ad2f74c","But if we're building products customers love,",{"_20":3710,"_90":271,"_272":3711,"_141":3071},"b9ebaf6f1c2d",[],{"_20":3713,"_90":271,"_272":3714,"_141":3716},"f6d7021a8b6f",[3715],"ec16187","we aren't going to keep up when we're using manual processes and hoarding all of this toil to make ourselves relevant. So",{"_20":3718,"_90":271,"_272":3719,"_141":3071},"9706e0f277ab",[],{"_20":3721,"_90":271,"_272":3722,"_141":3724},"005899ea7243",[3723],"07e84f5","if we want to preserve and improve our reliability, automation is the first priority.",[3726,3728,3730,3732,3734],{"_20":3691,"_90":3103,"_3104":3727},"00:05:50.420",{"_20":3699,"_90":3103,"_3104":3729},"00:05:56.660",{"_20":3707,"_90":3103,"_3104":3731},"00:06:08.080",{"_20":3715,"_90":3103,"_3104":3733},"00:06:12.760",{"_20":3723,"_90":3103,"_3104":3735},"00:06:21.280",{"_20":3737,"_90":266,"_267":3738,"_275":3743,"_277":312},"11f305f65297",[3739],{"_20":3740,"_90":271,"_272":3741,"_141":3742},"8cc593941254",[],"Distributed Systems Are a Fluid",[],{"_20":3745,"_90":266,"_267":3746,"_275":3768,"_277":278},"fb15af6565ee",[3747,3752,3755,3760,3763],{"_20":3748,"_90":271,"_272":3749,"_141":3751},"a2ad4c8ef91e",[3750],"e280ff5","Other things that we can be thinking about when we're looking at reliability, resiliency, operability, all the ilities,",{"_20":3753,"_90":271,"_272":3754,"_141":3071},"89f2e6b3ed5c",[],{"_20":3756,"_90":271,"_272":3757,"_141":3759},"3148f31bdbca",[3758],"d1385c7","is that it feels like it's hard, right?",{"_20":3761,"_90":271,"_272":3762,"_141":3071},"228cbc5941e6",[],{"_20":3764,"_90":271,"_272":3765,"_141":3767},"6fe4ea86c05e",[3766],"52b3063","We have all these complex things.",[3769,3771,3773],{"_20":3750,"_90":3103,"_3104":3770},"00:06:28.440",{"_20":3758,"_90":3103,"_3104":3772},"00:06:36.520",{"_20":3766,"_90":3103,"_3104":3774},"00:06:39.160",{"_20":3776,"_90":266,"_267":3777,"_275":3799,"_277":278},"883e7c051aa9",[3778,3783,3786,3791,3794],{"_20":3779,"_90":271,"_272":3780,"_141":3782},"93d6e3199c61",[3781],"28e7550","Having on-call and responder duties on top of that feels like an even bigger burden because by their nature, distributed systems are like a fluid.",{"_20":3784,"_90":271,"_272":3785,"_141":3071},"7718a905ebe4",[],{"_20":3787,"_90":271,"_272":3788,"_141":3790},"fbc7396bfae4",[3789],"303cfd4","They expand to fill every unbounded space.",{"_20":3792,"_90":271,"_272":3793,"_141":3071},"c56bb7f70b63",[],{"_20":3795,"_90":271,"_272":3796,"_141":3798},"d9a415694287",[3797],"6456d04","And they become more complex as they do so.",[3800,3802,3804],{"_20":3781,"_90":3103,"_3104":3801},"00:06:40.840",{"_20":3789,"_90":3103,"_3104":3803},"00:06:49.280",{"_20":3797,"_90":3103,"_3104":3805},"00:06:53.520",{"_20":3807,"_90":266,"_267":3808,"_275":3822,"_277":278},"fd3cfcfa3c8b",[3809,3814,3817],{"_20":3810,"_90":271,"_272":3811,"_141":3813},"99ee41d37d99",[3812],"2366668","It's part of how our innovation works.",{"_20":3815,"_90":271,"_272":3816,"_141":3071},"67d2a991423a",[],{"_20":3818,"_90":271,"_272":3819,"_141":3821},"3b07f8986c59",[3820],"2b135e8","We find a vacuum, a blank space, a dark corner that represents some kind of unmet need, and we fill it with a service.",[3823,3825],{"_20":3812,"_90":3103,"_3104":3824},"00:06:56.139",{"_20":3820,"_90":3103,"_3104":3826},"00:06:58.139",{"_20":3828,"_90":266,"_267":3829,"_275":3859,"_277":278},"8423dd111d8d",[3830,3835,3838,3843,3846,3851,3854],{"_20":3831,"_90":271,"_272":3832,"_141":3834},"dd351ac323a9",[3833],"0d687d8","But when we do that, we're stretching the organization's resources.",{"_20":3836,"_90":271,"_272":3837,"_141":3071},"731c968702f1",[],{"_20":3839,"_90":271,"_272":3840,"_141":3842},"f854a58a366a",[3841],"98c90c8","We stretch the ability to rationalize the system behaviors.",{"_20":3844,"_90":271,"_272":3845,"_141":3071},"0d0bdda999aa",[],{"_20":3847,"_90":271,"_272":3848,"_141":3850},"e0184debb96a",[3849],"18235c7","We stretch the capabilities of our tooling.",{"_20":3852,"_90":271,"_272":3853,"_141":3071},"64b3540aea31",[],{"_20":3855,"_90":271,"_272":3856,"_141":3858},"8001cdf6058b",[3857],"3b1ad3a","We stretch the bounds of what our team can handle.",[3860,3862,3864,3866],{"_20":3833,"_90":3103,"_3104":3861},"00:07:06.820",{"_20":3841,"_90":3103,"_3104":3863},"00:07:09.780",{"_20":3849,"_90":3103,"_3104":3865},"00:07:13.220",{"_20":3857,"_90":3103,"_3104":3867},"00:07:15.940",{"_20":3869,"_90":266,"_267":3870,"_275":3884,"_277":599},"78a44964eadd",[3871,3876,3879],{"_20":3872,"_90":271,"_272":3873,"_141":3875},"7fa9c7a8338b",[3874],"205e56e","So part of what we also want to be doing is putting up some guardrails to encourage innovation in the directions that we can best support.",{"_20":3877,"_90":271,"_272":3878,"_141":3071},"1a1b17f11b2f",[],{"_20":3880,"_90":271,"_272":3881,"_141":3883},"96e4a61be527",[3882],"dafae81","Our fluid should look more like irrigation canals than a flood.",[3885,3887],{"_20":3874,"_90":3103,"_3104":3886},"00:07:19.100",{"_20":3882,"_90":3103,"_3104":3888},"00:07:26.860",{"_20":3890,"_90":266,"_267":3891,"_275":3929,"_277":278},"205ee21302f8",[3892,3897,3900,3905,3908,3913,3916,3921,3924],{"_20":3893,"_90":271,"_272":3894,"_141":3896},"0a4367e3a856",[3895],"6afa879","To meet our goal of taking humans out of incident response, we have to be realistic about what matters and what can kind of slide.",{"_20":3898,"_90":271,"_272":3899,"_141":3071},"fcbadcb409d3",[],{"_20":3901,"_90":271,"_272":3902,"_141":3904},"b47fe560cee0",[3903],"6621d78","Leading teams are already setting their services up with SLOs, learning how to work with their error budgets, but our laggard peers haven't even heard of SLOs yet because",{"_20":3906,"_90":271,"_272":3907,"_141":3071},"115f57fa96a1",[],{"_20":3909,"_90":271,"_272":3910,"_141":3912},"ac9f090de545",[3911],"7477e6f","I've asked them, right? I'll give a couple of talks about SLOs.",{"_20":3914,"_90":271,"_272":3915,"_141":3071},"d25b3241d183",[],{"_20":3917,"_90":271,"_272":3918,"_141":3920},"03ff681fa633",[3919],"96d3ed4","Raise your hand if you've heard about them.",{"_20":3922,"_90":271,"_272":3923,"_141":3071},"fbfd86930fc6",[],{"_20":3925,"_90":271,"_272":3926,"_141":3928},"4102b2ddd89a",[3927],"86f41c1","Silence, right? So",[3930,3932,3934,3936,3938],{"_20":3895,"_90":3103,"_3104":3931},"00:07:32.840",{"_20":3903,"_90":3103,"_3104":3933},"00:07:41.660",{"_20":3911,"_90":3103,"_3104":3935},"00:07:53.220",{"_20":3919,"_90":3103,"_3104":3937},"00:07:56.420",{"_20":3927,"_90":3103,"_3104":3939},"00:07:58.340",{"_20":3941,"_90":266,"_267":3942,"_275":3956,"_277":278},"74c2e5ef6503",[3943,3948,3951],{"_20":3944,"_90":271,"_272":3945,"_141":3947},"52e7a81247d9",[3946],"9d0c997","unfortunately then, on one side we've got great teams who have a lot of data about what their users love, what their behaviors are, what their tolerances and preferences are, and they can create budgets off of that data.",{"_20":3949,"_90":271,"_272":3950,"_141":3071},"8bfd5f79e78a",[],{"_20":3952,"_90":271,"_272":3953,"_141":3955},"55dd3804fb6f",[3954],"61aee8e","They can reduce or suppress response processes when the stakes are low and escalate when the stakes are high.",[3957,3959],{"_20":3946,"_90":3103,"_3104":3958},"00:08:00.180",{"_20":3954,"_90":3103,"_3104":3960},"00:08:12.920",{"_20":3962,"_90":266,"_267":3963,"_275":3985,"_277":278},"f7cc595c0316",[3964,3969,3972,3977,3980],{"_20":3965,"_90":271,"_272":3966,"_141":3968},"8354f65bb321",[3967],"610b138","And on the other side, those teams don't have that information.",{"_20":3970,"_90":271,"_272":3971,"_141":3071},"e46cb52f6035",[],{"_20":3973,"_90":271,"_272":3974,"_141":3976},"b3f3db29a0a0",[3975],"93006dc","It might exist somewhere in their organization, but it might be sequestered, might belong to marketing or to product, but doesn't reach engineering.",{"_20":3978,"_90":271,"_272":3979,"_141":3071},"861de400444a",[],{"_20":3981,"_90":271,"_272":3982,"_141":3984},"b592c807991d",[3983],"3de6f07","It's in some other silo. So they're still treating their systems like we treated the monolith.",[3986,3988,3990],{"_20":3967,"_90":3103,"_3104":3987},"00:08:20.840",{"_20":3975,"_90":3103,"_3104":3989},"00:08:24.900",{"_20":3983,"_90":3103,"_3104":3991},"00:08:34.500",{"_20":3993,"_90":266,"_267":3994,"_275":4016,"_277":278},"13f45f887a09",[3995,4000,4003,4008,4011],{"_20":3996,"_90":271,"_272":3997,"_141":3999},"b9e9949299c8",[3998],"31d67c3","When the monolith wobbles, the world shakes.",{"_20":4001,"_90":271,"_272":4002,"_141":3071},"68e3b815848d",[],{"_20":4004,"_90":271,"_272":4005,"_141":4007},"3df64aef4325",[4006],"25c8da7","So the response is comparable. Full mobilization, all hands on deck,",{"_20":4009,"_90":271,"_272":4010,"_141":3071},"0193ad17e7d0",[],{"_20":4012,"_90":271,"_272":4013,"_141":4015},"60fcee1bb32e",[4014],"eaa5f6c","18 people on a conference call while a small team does all the work.",[4017,4019,4021],{"_20":3998,"_90":3103,"_3104":4018},"00:08:40.160",{"_20":4006,"_90":3103,"_3104":4020},"00:08:43.680",{"_20":4014,"_90":3103,"_3104":4022},"00:08:48.620",{"_20":4024,"_90":266,"_267":4025,"_275":4055,"_277":278},"530aaa27ece2",[4026,4031,4034,4039,4042,4047,4050],{"_20":4027,"_90":271,"_272":4028,"_141":4030},"bb06be412852",[4029],"73a4006","Those are fascinating customer discussions. But there's no claim that all components are important, only that smart teams will know how important their",{"_20":4032,"_90":271,"_272":4033,"_141":3071},"0b1a30cbfbcb",[],{"_20":4035,"_90":271,"_272":4036,"_141":4038},"e628eea0910d",[4037],"9afc396","components are in a way that reflects the real-world data that they've accumulated. So we have to be more rational",{"_20":4040,"_90":271,"_272":4041,"_141":3071},"48cb33574f81",[],{"_20":4043,"_90":271,"_272":4044,"_141":4046},"c85587b6f618",[4045],"253cf0e","about the services we create, what they mean for the users, and what they mean for the business versus",{"_20":4048,"_90":271,"_272":4049,"_141":3071},"7a87afcac4e2",[],{"_20":4051,"_90":271,"_272":4052,"_141":4054},"662539e32a05",[4053],"d3b2508","what they mean for us personally, as the humans who sort of created them and gave them life, as it were, right?",[4056,4058,4060,4062],{"_20":4029,"_90":3103,"_3104":4057},"00:08:52.700",{"_20":4037,"_90":3103,"_3104":4059},"00:09:04.560",{"_20":4045,"_90":3103,"_3104":4061},"00:09:13.500",{"_20":4053,"_90":3103,"_3104":4063},"00:09:20.459",{"_20":4065,"_90":266,"_267":4066,"_275":4071,"_277":312},"075057c71f8d",[4067],{"_20":4068,"_90":271,"_272":4069,"_141":4070},"fd9f4a0de6db",[],"Golden Paths and Paved Roads",[],{"_20":4073,"_90":266,"_267":4074,"_275":4096,"_277":278},"d7a2062e3024",[4075,4080,4083,4088,4091],{"_20":4076,"_90":271,"_272":4077,"_141":4079},"6a245921c356",[4078],"488fcad","So we look for other ways to conserve complexity.",{"_20":4081,"_90":271,"_272":4082,"_141":3071},"cfa5ca74d034",[],{"_20":4084,"_90":271,"_272":4085,"_141":4087},"4b5d9ef1d8e1",[4086],"2df1064","Some folks like a production tooling free-for-all.",{"_20":4089,"_90":271,"_272":4090,"_141":3071},"767c833bd09d",[],{"_20":4092,"_90":271,"_272":4093,"_141":4095},"90863d31ed5c",[4094],"26de4b5","It might be an interesting experiment for a while, but it quickly becomes exhausting and it hinders our ability to respond to incidents when everybody's using something different.",[4097,4099,4101],{"_20":4078,"_90":3103,"_3104":4098},"00:09:28.540",{"_20":4086,"_90":3103,"_3104":4100},"00:09:32.800",{"_20":4094,"_90":3103,"_3104":4102},"00:09:35.440",{"_20":4104,"_90":266,"_267":4105,"_275":4135,"_277":278},"6e845272f67f",[4106,4111,4114,4119,4122,4127,4130],{"_20":4107,"_90":271,"_272":4108,"_141":4110},"72ee83bd3663",[4109],"9c3ab6d","In some organizations, they constrain what engineers are doing.",{"_20":4112,"_90":271,"_272":4113,"_141":3071},"0d431242e408",[],{"_20":4115,"_90":271,"_272":4116,"_141":4118},"916acbbdb88b",[4117],"e82f05d","Here's your IDE, here's your allowed plugins.",{"_20":4120,"_90":271,"_272":4121,"_141":3071},"9764d9a63266",[],{"_20":4123,"_90":271,"_272":4124,"_141":4126},"edc4418fbee6",[4125],"2c5f069","Other places are a lot more",{"_20":4128,"_90":271,"_272":4129,"_141":3071},"a979334a07bc",[],{"_20":4131,"_90":271,"_272":4132,"_141":4134},"a89a05a63eec",[4133],"84d4e56","YOLO, whatever you do to ship, just do it, right?",[4136,4138,4140,4142],{"_20":4109,"_90":3103,"_3104":4137},"00:09:45.020",{"_20":4117,"_90":3103,"_3104":4139},"00:09:48.620",{"_20":4125,"_90":3103,"_3104":4141},"00:09:51.180",{"_20":4133,"_90":3103,"_3104":4143},"00:09:53.240",{"_20":4145,"_90":266,"_267":4146,"_275":4176,"_277":278},"9b25a89d3b0e",[4147,4152,4155,4160,4163,4168,4171],{"_20":4148,"_90":271,"_272":4149,"_141":4151},"236d71d5452c",[4150],"01dc072","So to preserve a little bit of our sanity, we want to encourage development teams to use the golden path or",{"_20":4153,"_90":271,"_272":4154,"_141":3071},"a093b054fe9e",[],{"_20":4156,"_90":271,"_272":4157,"_141":4159},"9fcdbe0a99f6",[4158],"e43bbde","paved roads or the express lane, whatever you call it. Choose your adventure, but it's a little from column A and a little from column B, and",{"_20":4161,"_90":271,"_272":4162,"_141":3071},"0e75ce3be643",[],{"_20":4164,"_90":271,"_272":4165,"_141":4167},"b2e70288f8ab",[4166],"d4d81d0","you get to production faster. If you deviate,",{"_20":4169,"_90":271,"_272":4170,"_141":3071},"ac46eda9c0fd",[],{"_20":4172,"_90":271,"_272":4173,"_141":4175},"517a55755c66",[4174],"84ac5be","you start to lose support and the path gets a little dark and scary.",[4177,4179,4181,4183],{"_20":4150,"_90":3103,"_3104":4178},"00:09:57.000",{"_20":4158,"_90":3103,"_3104":4180},"00:10:06.199",{"_20":4166,"_90":3103,"_3104":4182},"00:10:17.200",{"_20":4174,"_90":3103,"_3104":4184},"00:10:20.540",{"_20":4186,"_90":266,"_267":4187,"_275":4209,"_277":278},"28217ad57cd5",[4188,4193,4196,4201,4204],{"_20":4189,"_90":271,"_272":4190,"_141":4192},"5d29567a233f",[4191],"1563612","Folks are fighting about what to call this, right?",{"_20":4194,"_90":271,"_272":4195,"_141":3071},"fdd62853ff3b",[],{"_20":4197,"_90":271,"_272":4198,"_141":4200},"69659494dc83",[4199],"2450d42","Is it platform engineering? Is it developer experience?",{"_20":4202,"_90":271,"_272":4203,"_141":3071},"23a117c92eea",[],{"_20":4205,"_90":271,"_272":4206,"_141":4208},"ab14778dc5f2",[4207],"75afed3","Is it rebranding of release engineering, or is it just what SRE should be doing anyway?",[4210,4212,4214],{"_20":4191,"_90":3103,"_3104":4211},"00:10:24.300",{"_20":4199,"_90":3103,"_3104":4213},"00:10:27.100",{"_20":4207,"_90":3103,"_3104":4215},"00:10:31.020",{"_20":4217,"_90":266,"_267":4218,"_275":4232,"_277":278},"b10e50e18cdc",[4219,4224,4227],{"_20":4220,"_90":271,"_272":4221,"_141":4223},"d8463b04940b",[4222],"1435536","So we know folks are thinking about solidifying their approaches and benefiting from this efficiency.",{"_20":4225,"_90":271,"_272":4226,"_141":3071},"bfa54f2ddd6e",[],{"_20":4228,"_90":271,"_272":4229,"_141":4231},"e7e892ecc5c4",[4230],"c3c348f","Because when teams cluster their projects together around a set stack, knowledge sharing is easier.",[4233,4235],{"_20":4222,"_90":3103,"_3104":4234},"00:10:36.860",{"_20":4230,"_90":3103,"_3104":4236},"00:10:42.920",{"_20":4238,"_90":266,"_267":4239,"_275":4253,"_277":599},"86da9338109d",[4240,4245,4248],{"_20":4241,"_90":271,"_272":4242,"_141":4244},"e918efe2a7a3",[4243],"d9e28b3","Learning the sharp edges, finding unusual behaviors,",{"_20":4246,"_90":271,"_272":4247,"_141":3071},"0d90c2172b9a",[],{"_20":4249,"_90":271,"_272":4250,"_141":4252},"e6b2bbb1a615",[4251],"5e25a78","looking for those resource limitations or weird bugs becomes a group sport instead of an individual safari.",[4254,4256],{"_20":4243,"_90":3103,"_3104":4255},"00:10:49.880",{"_20":4251,"_90":3103,"_3104":4257},"00:10:53.620",{"_20":4259,"_90":266,"_267":4260,"_275":4282,"_277":278},"004bf10ff437",[4261,4266,4269,4274,4277],{"_20":4262,"_90":271,"_272":4263,"_141":4265},"26e864fc1653",[4264],"b6be497","Teams that use the same environments can share more production responsibilities.",{"_20":4267,"_90":271,"_272":4268,"_141":3071},"bf7cea538600",[],{"_20":4270,"_90":271,"_272":4271,"_141":4273},"f5e02b4177c5",[4272],"dc24365","They have this knowledge extension across teams.",{"_20":4275,"_90":271,"_272":4276,"_141":3071},"45e209dc826c",[],{"_20":4278,"_90":271,"_272":4279,"_141":4281},"149386be03aa",[4280],"58bd06b","We can also prevent decision fatigue on our teams and channel that energy into innovating for our users.",[4283,4285,4287],{"_20":4264,"_90":3103,"_3104":4284},"00:11:02.140",{"_20":4272,"_90":3103,"_3104":4286},"00:11:06.540",{"_20":4280,"_90":3103,"_3104":4288},"00:11:09.820",{"_20":4290,"_90":266,"_267":4291,"_275":4313,"_277":278},"a28a5976e6df",[4292,4297,4300,4305,4308],{"_20":4293,"_90":271,"_272":4294,"_141":4296},"97951c50f098",[4295],"3d3b9f4","Are there developers out there searching for nirvana, looking for a place where all the tools in the workflow are beautiful and perfect?",{"_20":4298,"_90":271,"_272":4299,"_141":3071},"226a0cf2aaf1",[],{"_20":4301,"_90":271,"_272":4302,"_141":4304},"bf05735c77a8",[4303],"e0468e6","And when their code runs through the pipeline, angels sing.",{"_20":4306,"_90":271,"_272":4307,"_141":3071},"f73a00c55490",[],{"_20":4309,"_90":271,"_272":4310,"_141":4312},"875e2708fbdd",[4311],"3726747","I'm sure there are, but while they're off on that side quest, their customers are in another castle, right?",[4314,4316,4318],{"_20":4295,"_90":3103,"_3104":4315},"00:11:16.779",{"_20":4303,"_90":3103,"_3104":4317},"00:11:24.459",{"_20":4311,"_90":3103,"_3104":4319},"00:11:27.899",{"_20":4321,"_90":266,"_267":4322,"_275":4352,"_277":278},"03d320ce9e48",[4323,4328,4331,4336,4339,4344,4347],{"_20":4324,"_90":271,"_272":4325,"_141":4327},"06ae858f6b65",[4326],"f5212c1","That doesn't mean there isn't still room to play and experiment.",{"_20":4329,"_90":271,"_272":4330,"_141":3071},"d278a19e7d65",[],{"_20":4332,"_90":271,"_272":4333,"_141":4335},"3606bc0e2420",[4334],"cf0ebdc","Intentional experimentation that focuses on improving user experience is great.",{"_20":4337,"_90":271,"_272":4338,"_141":3071},"5742339df950",[],{"_20":4340,"_90":271,"_272":4341,"_141":4343},"b7dc61cc3d6f",[4342],"477b6fe","Wild experimentation because your engineers are bored is a problem.",{"_20":4345,"_90":271,"_272":4346,"_141":3071},"75a4c630f1a2",[],{"_20":4348,"_90":271,"_272":4349,"_141":4351},"47e8612ead9a",[4350],"f98d1c9","And it makes responding to errors more difficult because no one's sure what's under the hood.",[4353,4355,4357,4359],{"_20":4326,"_90":3103,"_3104":4354},"00:11:34.759",{"_20":4334,"_90":3103,"_3104":4356},"00:11:38.440",{"_20":4342,"_90":3103,"_3104":4358},"00:11:43.820",{"_20":4350,"_90":3103,"_3104":4360},"00:11:49.100",{"_20":4362,"_90":266,"_267":4363,"_275":4385,"_277":278},"c20a858ddd3b",[4364,4369,4372,4377,4380],{"_20":4365,"_90":271,"_272":4366,"_141":4368},"c4eb837c9051",[4367],"108754d","Leaning a little bit towards predictable, towards a little bit more boring, gives us a leg up on automation and efficiency.",{"_20":4370,"_90":271,"_272":4371,"_141":3071},"50da53cbf0f2",[],{"_20":4373,"_90":271,"_272":4374,"_141":4376},"c7b685f3d9c3",[4375],"fe79ba2","We share more of the things that we learn.",{"_20":4378,"_90":271,"_272":4379,"_141":3071},"8d04394d1c4c",[],{"_20":4381,"_90":271,"_272":4382,"_141":4384},"1ccac1581807",[4383],"0aced4b","It allows us to expand our on-call rotations when different teams are using the same stack.",[4386,4388,4390],{"_20":4367,"_90":3103,"_3104":4387},"00:11:56.160",{"_20":4375,"_90":3103,"_3104":4389},"00:12:06.319",{"_20":4383,"_90":3103,"_3104":4391},"00:12:08.740",{"_20":4393,"_90":266,"_267":4394,"_275":4408,"_277":278},"b1a0613420cb",[4395,4400,4403],{"_20":4396,"_90":271,"_272":4397,"_141":4399},"5fc88cf82dd5",[4398],"f905069","Conversion down to a monoculture isn't required.",{"_20":4401,"_90":271,"_272":4402,"_141":3071},"5378f3ef4a2d",[],{"_20":4404,"_90":271,"_272":4405,"_141":4407},"fdaacc82ba90",[4406],"5764b0d","But we can create clusters of well-defined packages and dependencies and stacks that can give folks some breathing room and help us learn faster.",[4409,4411],{"_20":4398,"_90":3103,"_3104":4410},"00:12:15.540",{"_20":4406,"_90":3103,"_3104":4412},"00:12:18.500",{"_20":4414,"_90":266,"_267":4415,"_275":4420,"_277":312},"7b7baa62a2f6",[4416],{"_20":4417,"_90":271,"_272":4418,"_141":4419},"9a1ca0a9eaae",[],"Sharing the On-Call Burden",[],{"_20":4422,"_90":266,"_267":4423,"_275":4445,"_277":278},"6f8ad650319c",[4424,4429,4432,4437,4440],{"_20":4425,"_90":271,"_272":4426,"_141":4428},"9cf162d77b7a",[4427],"80a5b1e","So we can treat our production responsibilities not as a punishment to be borne, but a shared opportunity to celebrate a successful deployment",{"_20":4430,"_90":271,"_272":4431,"_141":3071},"dfaa4cbb1003",[],{"_20":4433,"_90":271,"_272":4434,"_141":4436},"d1853ce3f432",[4435],"63379b5","and good customer experience. And then we learn from that and continue to improve.",{"_20":4438,"_90":271,"_272":4439,"_141":3071},"0e02dced4c46",[],{"_20":4441,"_90":271,"_272":4442,"_141":4444},"781530b5875e",[4443],"e5d2743","As teams are sharing more and learning together, the burden for incidents spreads out, right?",[4446,4448,4450],{"_20":4427,"_90":3103,"_3104":4447},"00:12:30.280",{"_20":4435,"_90":3103,"_3104":4449},"00:12:41.940",{"_20":4443,"_90":3103,"_3104":4451},"00:12:47.460",{"_20":4453,"_90":266,"_267":4454,"_275":4476,"_277":278},"c932a153aa48",[4455,4460,4463,4468,4471],{"_20":4456,"_90":271,"_272":4457,"_141":4459},"3b72541ee575",[4458],"ff29b7b","It gets better for everyone. We still have a lot to learn about how to best support teams that are very small",{"_20":4461,"_90":271,"_272":4462,"_141":3071},"d529c64af093",[],{"_20":4464,"_90":271,"_272":4465,"_141":4467},"2accc6c74a7d",[4466],"f299a74","or are dealing with a particularly rough production environment",{"_20":4469,"_90":271,"_272":4470,"_141":3071},"82b3f24497ed",[],{"_20":4472,"_90":271,"_272":4473,"_141":4475},"27b2ed79a597",[4474],"ce909f5","with a lot of incidents that require human intervention.",[4477,4479,4481],{"_20":4458,"_90":3103,"_3104":4478},"00:12:53.600",{"_20":4466,"_90":3103,"_3104":4480},"00:13:01.759",{"_20":4474,"_90":3103,"_3104":4482},"00:13:05.640",{"_20":4484,"_90":266,"_267":4485,"_275":4507,"_277":278},"096b978ba6d2",[4486,4491,4494,4499,4502],{"_20":4487,"_90":271,"_272":4488,"_141":4490},"3b6d4d7d4160",[4489],"9a61ed7","Fortunately, something we're seeing across the industry is more engineering management teams have been having more",{"_20":4492,"_90":271,"_272":4493,"_141":3071},"90dba6d92e4c",[],{"_20":4495,"_90":271,"_272":4496,"_141":4498},"3c89c6b72096",[4497],"0a21932","mature discussions with their engineers, with their leadership teams, about what good on-call practices can look like.",{"_20":4500,"_90":271,"_272":4501,"_141":3071},"c231d214d30a",[],{"_20":4503,"_90":271,"_272":4504,"_141":4506},"1ad62120612f",[4505],"8e3b6fc","More teams are looking for flexible scheduling and they're stepping back from that seven-day rotation that's kind of been the cultural default in a lot of places.",[4508,4510,4512],{"_20":4489,"_90":3103,"_3104":4509},"00:13:09.400",{"_20":4497,"_90":3103,"_3104":4511},"00:13:17.360",{"_20":4505,"_90":3103,"_3104":4513},"00:13:26.779",{"_20":4515,"_90":266,"_267":4516,"_275":4538,"_277":278},"0d7d1fdf59ca",[4517,4522,4525,4530,4533],{"_20":4518,"_90":271,"_272":4519,"_141":4521},"8afc9b116eb5",[4520],"f21000f","Because seven days in a noisy environment is awful.",{"_20":4523,"_90":271,"_272":4524,"_141":3071},"eb30398ac301",[],{"_20":4526,"_90":271,"_272":4527,"_141":4529},"e40df91df94f",[4528],"1bb4f8d","It is a burnout experience. So looking at more creative ways to spread the load, to share responsibilities, to lighten everyone else's load",{"_20":4531,"_90":271,"_272":4532,"_141":3071},"515257b3ec3d",[],{"_20":4534,"_90":271,"_272":4535,"_141":4537},"2d31e66f7ea2",[4536],"3c4186e","on that is definitely a thing that we want for folks.",[4539,4541,4543],{"_20":4520,"_90":3103,"_3104":4540},"00:13:37.160",{"_20":4528,"_90":3103,"_3104":4542},"00:13:41.720",{"_20":4536,"_90":3103,"_3104":4544},"00:13:53.839",{"_20":4546,"_90":266,"_267":4547,"_275":4585,"_277":278},"e3feab8d2c2e",[4548,4553,4556,4561,4564,4569,4572,4577,4580],{"_20":4549,"_90":271,"_272":4550,"_141":4552},"859cfe3c696c",[4551],"4b92a7b","And we can make the processes better.",{"_20":4554,"_90":271,"_272":4555,"_141":3071},"f91ab158fbbc",[],{"_20":4557,"_90":271,"_272":4558,"_141":4560},"b55b2dcafc2a",[4559],"b2a8246","We can improve this relationship engineers have to the demands of the production environment.",{"_20":4562,"_90":271,"_272":4563,"_141":3071},"d7687dfde0f0",[],{"_20":4565,"_90":271,"_272":4566,"_141":4568},"6ad099dda9d2",[4567],"57eee11","We can be getting more defensive around the things that we know are sort of continual problems, whether that is just ignoring the thing or caching responses",{"_20":4570,"_90":271,"_272":4571,"_141":3071},"a9054cb39309",[],{"_20":4573,"_90":271,"_272":4574,"_141":4576},"b76407894d41",[4575],"ee3dc3f","or holding it down or red buttoning.",{"_20":4578,"_90":271,"_272":4579,"_141":3071},"3198a7a58aa6",[],{"_20":4581,"_90":271,"_272":4582,"_141":4584},"35051c360f07",[4583],"f86f2c5","We save our collective sanity when we place bad components in a bubble as much as we can.",[4586,4588,4590,4592,4594],{"_20":4551,"_90":3103,"_3104":4587},"00:13:57.959",{"_20":4559,"_90":3103,"_3104":4589},"00:13:59.959",{"_20":4567,"_90":3103,"_3104":4591},"00:14:05.520",{"_20":4575,"_90":3103,"_3104":4593},"00:14:16.800",{"_20":4583,"_90":3103,"_3104":4595},"00:14:19.640",{"_20":4597,"_90":266,"_267":4598,"_275":4628,"_277":599},"abefbf7e9cf6",[4599,4604,4607,4612,4615,4620,4623],{"_20":4600,"_90":271,"_272":4601,"_141":4603},"db5fdd557237",[4602],"008575a","The challenge still exists that things don't get fixed until the right person has had the hard conversation",{"_20":4605,"_90":271,"_272":4606,"_141":3071},"5a87cfd549c6",[],{"_20":4608,"_90":271,"_272":4609,"_141":4611},"8ec1764faf2e",[4610],"66cd54a","about performance and prioritization of the work required from a socio-technical perspective. A service that drags",{"_20":4613,"_90":271,"_272":4614,"_141":3071},"26e4d30cba99",[],{"_20":4616,"_90":271,"_272":4617,"_141":4619},"6d2f0c71e612",[4618],"4a70f3e","everyone away from user-facing work is costing the organization time and money.",{"_20":4621,"_90":271,"_272":4622,"_141":3071},"a4903565733b",[],{"_20":4624,"_90":271,"_272":4625,"_141":4627},"a784c4f826ca",[4626],"55c10d9","Why are you willing to let it do that?",[4629,4631,4633,4635],{"_20":4602,"_90":3103,"_3104":4630},"00:14:25.720",{"_20":4610,"_90":3103,"_3104":4632},"00:14:34.079",{"_20":4618,"_90":3103,"_3104":4634},"00:14:44.080",{"_20":4626,"_90":3103,"_3104":4636},"00:14:49.440",{"_20":4638,"_90":266,"_267":4639,"_275":4653,"_277":278},"45d9cc9012f9",[4640,4645,4648],{"_20":4641,"_90":271,"_272":4642,"_141":4644},"3858d3af29fd",[4643],"4e65d96","So we want to include our partners from product management and the business side of the organization, bring the whole process full circle.",{"_20":4646,"_90":271,"_272":4647,"_141":3071},"d79d655a7f29",[],{"_20":4649,"_90":271,"_272":4650,"_141":4652},"f0921f6f5977",[4651],"da86ca0","We can illustrate the core benefit of having product development teams involved in the incident response.",[4654,4656],{"_20":4643,"_90":3103,"_3104":4655},"00:14:52.320",{"_20":4651,"_90":3103,"_3104":4657},"00:15:00.940",{"_20":4659,"_90":266,"_267":4660,"_275":4690,"_277":278},"ca369b9fd091",[4661,4666,4669,4674,4677,4682,4685],{"_20":4662,"_90":271,"_272":4663,"_141":4665},"62a73c42c798",[4664],"0a8d2d4","When something needs to be fixed, they can document the work, add it to their backlog, and advocate for it to be prioritized. That's harder to do when your production operations",{"_20":4667,"_90":271,"_272":4668,"_141":3071},"9a7bed8b2bc2",[],{"_20":4670,"_90":271,"_272":4671,"_141":4673},"35f8d4d0a43e",[4672],"1803eb0","lives in another team. It's harder to prioritize or advocate across departments",{"_20":4675,"_90":271,"_272":4676,"_141":3071},"e999b70727ee",[],{"_20":4678,"_90":271,"_272":4679,"_141":4681},"414734abf13f",[4680],"dc79713","for the folks who are feeling the pain versus the folks who might be sort of creating the pain, right?",{"_20":4683,"_90":271,"_272":4684,"_141":3071},"7ccc520fb48d",[],{"_20":4686,"_90":271,"_272":4687,"_141":4689},"ba01fb1f72e3",[4688],"3fa4aac","So we want our engineering teams to get that full blast of what's going on with their stuff in production.",[4691,4693,4695,4697],{"_20":4664,"_90":3103,"_3104":4692},"00:15:07.339",{"_20":4672,"_90":3103,"_3104":4694},"00:15:19.160",{"_20":4680,"_90":3103,"_3104":4696},"00:15:25.920",{"_20":4688,"_90":3103,"_3104":4698},"00:15:32.160",{"_20":4700,"_90":266,"_267":4701,"_275":4706,"_277":312},"9e8d6ba017de",[4702],{"_20":4703,"_90":271,"_272":4704,"_141":4705},"73e298b03a1a",[],"What Already Works",[],{"_20":4708,"_90":266,"_267":4709,"_275":4731,"_277":278},"ac70646466b2",[4710,4715,4718,4723,4726],{"_20":4711,"_90":271,"_272":4712,"_141":4714},"07d84a5d8229",[4713],"871d190","So there's all this difficult stuff going forward, right?",{"_20":4716,"_90":271,"_272":4717,"_141":3071},"2ba181c4f8e2",[],{"_20":4719,"_90":271,"_272":4720,"_141":4722},"c31185454e1d",[4721],"7db1aad","But there's a bunch of stuff that already works well",{"_20":4724,"_90":271,"_272":4725,"_141":3071},"fdd7f06f534f",[],{"_20":4727,"_90":271,"_272":4728,"_141":4730},"1a52c6d442bc",[4729],"69c2c57","across lots of places, right? One of the major cultural processes in incident response that's really crossed a chasm is the post-incident review.",[4732,4734,4736],{"_20":4713,"_90":3103,"_3104":4733},"00:15:41.220",{"_20":4721,"_90":3103,"_3104":4735},"00:15:43.699",{"_20":4729,"_90":3103,"_3104":4737},"00:15:46.339",{"_20":4739,"_90":266,"_267":4740,"_275":4762,"_277":278},"c4a8332aa282",[4741,4746,4749,4754,4757],{"_20":4742,"_90":271,"_272":4743,"_141":4745},"6ae2d94ff431",[4744],"5205cf6","We called it a bunch of things, right?",{"_20":4747,"_90":271,"_272":4748,"_141":3071},"0b12c0a0f479",[],{"_20":4750,"_90":271,"_272":4751,"_141":4753},"574747cdf3c1",[4752],"8f0c8f9","And interestingly enough, we get email, folks really don't like post-mortem.",{"_20":4755,"_90":271,"_272":4756,"_141":3071},"a3d9d34a8722",[],{"_20":4758,"_90":271,"_272":4759,"_141":4761},"e9ad4658c242",[4760],"7046a00","So we'll go with post-incident review.",[4763,4765,4767],{"_20":4744,"_90":3103,"_3104":4764},"00:15:55.720",{"_20":4752,"_90":3103,"_3104":4766},"00:15:57.480",{"_20":4760,"_90":3103,"_3104":4768},"00:16:02.440",{"_20":4770,"_90":266,"_267":4771,"_275":4785,"_277":278},"87bba3ba4c68",[4772,4777,4780],{"_20":4773,"_90":271,"_272":4774,"_141":4776},"3ba1a6b12aee",[4775],"f26fbd9","For teams operating in these process-heavy environments, whether it's ITIL or whatever else, having a review after any significant event totally fits in their project management workflow.",{"_20":4778,"_90":271,"_272":4779,"_141":3071},"bd5bf45c6e9a",[],{"_20":4781,"_90":271,"_272":4782,"_141":4784},"b6fbb1ea33a6",[4783],"06a46f0","They love this stuff, right? There's increased willingness from big companies, little companies to publish their incident reviews and really get into the account of what happened.",[4786,4788],{"_20":4775,"_90":3103,"_3104":4787},"00:16:05.220",{"_20":4783,"_90":3103,"_3104":4789},"00:16:16.340",{"_20":4791,"_90":266,"_267":4792,"_275":4806,"_277":278},"3e72ec9fbfa2",[4793,4798,4801],{"_20":4794,"_90":271,"_272":4795,"_141":4797},"6ecefa87b97e",[4796],"a42c7c5","Create these sometimes beautiful artifacts that can be referenced by various stakeholders after the event.",{"_20":4799,"_90":271,"_272":4800,"_141":3071},"d0c09fed09a9",[],{"_20":4802,"_90":271,"_272":4803,"_141":4805},"5be977174d06",[4804],"b454102","Plus, they're fun for all of us to read too, right?",[4807,4809],{"_20":4796,"_90":3103,"_3104":4808},"00:16:26.920",{"_20":4804,"_90":3103,"_3104":4810},"00:16:33.079",{"_20":4812,"_90":266,"_267":4813,"_275":4843,"_277":278},"b9f9c9910a75",[4814,4819,4822,4827,4830,4835,4838],{"_20":4815,"_90":271,"_272":4816,"_141":4818},"fa328a6a9ab6",[4817],"332d462","One other thing I've seen emerging is the presence of dedicated teams for dealing with incident response practices.",{"_20":4820,"_90":271,"_272":4821,"_141":3071},"ee9b1fc847e6",[],{"_20":4823,"_90":271,"_272":4824,"_141":4826},"ecf5a4c9c3ca",[4825],"2f3a799","I know we hated this when big companies claimed they were starting DevOps teams, right?",{"_20":4828,"_90":271,"_272":4829,"_141":3071},"a958d1031011",[],{"_20":4831,"_90":271,"_272":4832,"_141":4834},"04c6585b7548",[4833],"30f0c5d","Now, surprise, they're starting response teams.",{"_20":4836,"_90":271,"_272":4837,"_141":3071},"33303bdb6172",[],{"_20":4839,"_90":271,"_272":4840,"_141":4842},"e389c099bbcc",[4841],"6fae6e0","Honestly, where I've seen this, it looks great, right?",[4844,4846,4848,4850],{"_20":4817,"_90":3103,"_3104":4845},"00:16:35.820",{"_20":4825,"_90":3103,"_3104":4847},"00:16:45.220",{"_20":4833,"_90":3103,"_3104":4849},"00:16:50.500",{"_20":4841,"_90":3103,"_3104":4851},"00:16:55.040",{"_20":4853,"_90":266,"_267":4854,"_275":4884,"_277":278},"ed9b8bbe9ac7",[4855,4860,4863,4868,4871,4876,4879],{"_20":4856,"_90":271,"_272":4857,"_141":4859},"f3a42f160e13",[4858],"25c34af","When you have a sufficiently large organization and they have a lot of incidents that require humans, that's a lot of overhead.",{"_20":4861,"_90":271,"_272":4862,"_141":3071},"ad24f499a687",[],{"_20":4864,"_90":271,"_272":4865,"_141":4867},"f7f2b8f96a38",[4866],"b32608d","It's a lot of investment. And the other things that come with these processes are expensive.",{"_20":4869,"_90":271,"_272":4870,"_141":3071},"487d1517855b",[],{"_20":4872,"_90":271,"_272":4873,"_141":4875},"8f0f51c4f3ae",[4874],"d2f7603","If you're following sort of an incident commander program, you have to find the candidates, train them, create all the processes, document all that stuff.",{"_20":4877,"_90":271,"_272":4878,"_141":3071},"b03b2799f63b",[],{"_20":4880,"_90":271,"_272":4881,"_141":4883},"005c7c837cc1",[4882],"a132f33","Maybe you've got checklists to create, all that good stuff.",[4885,4887,4889,4891],{"_20":4858,"_90":3103,"_3104":4886},"00:16:59.680",{"_20":4866,"_90":3103,"_3104":4888},"00:17:06.640",{"_20":4874,"_90":3103,"_3104":4890},"00:17:13.900",{"_20":4882,"_90":3103,"_3104":4892},"00:17:25.020",{"_20":4894,"_90":266,"_267":4895,"_275":4925,"_277":278},"96b68916e59a",[4896,4901,4904,4909,4912,4917,4920],{"_20":4897,"_90":271,"_272":4898,"_141":4900},"3a65178f79df",[4899],"f0df879","You evangelize it out to the rest of the organization.",{"_20":4902,"_90":271,"_272":4903,"_141":3071},"907ba8d6afa3",[],{"_20":4905,"_90":271,"_272":4906,"_141":4908},"52f64d1a43f2",[4907],"cbe9b0e","And then you train all the other teams on how to make use of this process.",{"_20":4910,"_90":271,"_272":4911,"_141":3071},"3d67bee4324b",[],{"_20":4913,"_90":271,"_272":4914,"_141":4916},"73cbdfb31f08",[4915],"0e0fb85","When we've encountered the dedicated response teams at various organizations, though, the first parts are all given.",{"_20":4918,"_90":271,"_272":4919,"_141":3071},"57ce25d76785",[],{"_20":4921,"_90":271,"_272":4922,"_141":4924},"13f6191620e4",[4923],"82417e8","We hired this team specifically to do all this stuff for us.",[4926,4928,4930,4932],{"_20":4899,"_90":3103,"_3104":4927},"00:17:27.660",{"_20":4907,"_90":3103,"_3104":4929},"00:17:30.380",{"_20":4915,"_90":3103,"_3104":4931},"00:17:35.100",{"_20":4923,"_90":3103,"_3104":4933},"00:17:41.340",{"_20":4935,"_90":266,"_267":4936,"_275":4966,"_277":278},"46448047f659",[4937,4942,4945,4950,4953,4958,4961],{"_20":4938,"_90":271,"_272":4939,"_141":4941},"bf76357ae982",[4940],"ab3c94a","They're going to be our incident commanders.",{"_20":4943,"_90":271,"_272":4944,"_141":3071},"2083ef0dfa84",[],{"_20":4946,"_90":271,"_272":4947,"_141":4949},"f17426490f8b",[4948],"d201d36","They're going to be our customer liaisons.",{"_20":4951,"_90":271,"_272":4952,"_141":3071},"703a1a71f81d",[],{"_20":4954,"_90":271,"_272":4955,"_141":4957},"9bd4d841dea9",[4956],"0f4f5fc","They're going to be our scribes.",{"_20":4959,"_90":271,"_272":4960,"_141":3071},"c2488097dfc7",[],{"_20":4962,"_90":271,"_272":4963,"_141":4965},"f72a6061a255",[4964],"09513ca","Then they're going to shepherd the post-incident review.",[4967,4969,4971,4973],{"_20":4940,"_90":3103,"_3104":4968},"00:17:44.400",{"_20":4948,"_90":3103,"_3104":4970},"00:17:46.320",{"_20":4956,"_90":3103,"_3104":4972},"00:17:48.160",{"_20":4964,"_90":3103,"_3104":4974},"00:17:50.960",{"_20":4976,"_90":266,"_267":4977,"_275":4999,"_277":599},"86b2cd6ecf46",[4978,4983,4986,4991,4994],{"_20":4979,"_90":271,"_272":4980,"_141":4982},"f633c24d7c85",[4981],"e79ac3b","So they're like our professional career firefighters.",{"_20":4984,"_90":271,"_272":4985,"_141":3071},"0135cf86afdd",[],{"_20":4987,"_90":271,"_272":4988,"_141":4990},"87cd91721dc3",[4989],"ecdca75","They know all this stuff inside and out and can help everybody else get better.",{"_20":4992,"_90":271,"_272":4993,"_141":3071},"c57d9bda8d3d",[],{"_20":4995,"_90":271,"_272":4996,"_141":4998},"f21fd37db528",[4997],"de45f3a","So there's less hesitancy, there's less uncertainty, and there's a lot more working together.",[5000,5002,5004],{"_20":4981,"_90":3103,"_3104":5001},"00:17:53.840",{"_20":4989,"_90":3103,"_3104":5003},"00:17:57.380",{"_20":4997,"_90":3103,"_3104":5005},"00:18:02.900",{"_20":5007,"_90":266,"_267":5008,"_275":5013,"_277":312},"eeab798d5b0c",[5009],{"_20":5010,"_90":271,"_272":5011,"_141":5012},"743beef8354a",[],"Legacy Is Here to Stay",[],{"_20":5015,"_90":266,"_267":5016,"_275":5054,"_277":278},"b7c74af92ca8",[5017,5022,5025,5030,5033,5038,5041,5046,5049],{"_20":5018,"_90":271,"_272":5019,"_141":5021},"abfa9a2ea89f",[5020],"93659e7","There's other stuff that might not get there, right?",{"_20":5023,"_90":271,"_272":5024,"_141":3071},"11c5a7fbe20b",[],{"_20":5026,"_90":271,"_272":5027,"_141":5029},"d8e0b0ef7100",[5028],"cc9c1e4","Seen a lot of hesitancy in some organizations around certain things, right?",{"_20":5031,"_90":271,"_272":5032,"_141":3071},"724141e8cc3e",[],{"_20":5034,"_90":271,"_272":5035,"_141":5037},"9096a533720c",[5036],"52de326","One thing for us all to remember, especially when we're working with large organizations, maybe they're partners, maybe they're prospective customers,",{"_20":5039,"_90":271,"_272":5040,"_141":3071},"679a99067e88",[],{"_20":5042,"_90":271,"_272":5043,"_141":5045},"f54aacf1a5cf",[5044],"4ef8973","because they have the big budget,",{"_20":5047,"_90":271,"_272":5048,"_141":3071},"bd5e7b26cd30",[],{"_20":5050,"_90":271,"_272":5051,"_141":5053},"81362b1170fb",[5052],"8c610fd","is that their scale and the legacy they have exists for a reason.",[5055,5057,5059,5061,5063],{"_20":5020,"_90":3103,"_3104":5056},"00:18:10.519",{"_20":5028,"_90":3103,"_3104":5058},"00:18:14.519",{"_20":5036,"_90":3103,"_3104":5060},"00:18:19.480",{"_20":5044,"_90":3103,"_3104":5062},"00:18:28.440",{"_20":5052,"_90":3103,"_3104":5064},"00:18:30.660",{"_20":5066,"_90":266,"_267":5067,"_275":5097,"_277":278},"e007dc17a7fb",[5068,5073,5076,5081,5084,5089,5092],{"_20":5069,"_90":271,"_272":5070,"_141":5072},"216771c3d696",[5071],"35d7645","The old code that's still hanging around is probably directly in their revenue stream somewhere, right?",{"_20":5074,"_90":271,"_272":5075,"_141":3071},"76b0dc358f99",[],{"_20":5077,"_90":271,"_272":5078,"_141":5080},"560707061125",[5079],"0ce5fc7","And it's just kind of sitting in the corner generating real cash money, right?",{"_20":5082,"_90":271,"_272":5083,"_141":3071},"d718af686ed5",[],{"_20":5085,"_90":271,"_272":5086,"_141":5088},"a5143de5537c",[5087],"de54bb5","So we might be living in a lean, purpose-built, cloud-native environment.",{"_20":5090,"_90":271,"_272":5091,"_141":3071},"7b68dfde873f",[],{"_20":5093,"_90":271,"_272":5094,"_141":5096},"fa5d31caec1a",[5095],"0055784","Big multinationals have stuff kind of all over the place.",[5098,5100,5102,5104],{"_20":5071,"_90":3103,"_3104":5099},"00:18:36.740",{"_20":5079,"_90":3103,"_3104":5101},"00:18:44.720",{"_20":5087,"_90":3103,"_3104":5103},"00:18:50.040",{"_20":5095,"_90":3103,"_3104":5105},"00:18:55.800",{"_20":5107,"_90":266,"_267":5108,"_275":5138,"_277":278},"b4e41834f734",[5109,5114,5117,5122,5125,5130,5133],{"_20":5110,"_90":271,"_272":5111,"_141":5113},"bfe9f40158f1",[5112],"d6e2caa","It's like walking into one of those big warehouses at the end of Indiana Jones or whatever.",{"_20":5115,"_90":271,"_272":5116,"_141":3071},"5d8d9f2dd91b",[],{"_20":5118,"_90":271,"_272":5119,"_141":5121},"7ec5de975ffa",[5120],"7fb6656","Some of it might be recent, some of it might be older than the intern.",{"_20":5123,"_90":271,"_272":5124,"_141":3071},"688f061d6c1f",[],{"_20":5126,"_90":271,"_272":5127,"_141":5129},"c61053456398",[5128],"720c68d","And that's okay.",{"_20":5131,"_90":271,"_272":5132,"_141":3071},"564c2826c60a",[],{"_20":5134,"_90":271,"_272":5135,"_141":5137},"01b86a34af49",[5136],"df93e7a","But even components that are only off by one major revision are probably going to be outside of the scope for a big effort that requires changes to code.",[5139,5141,5143,5145],{"_20":5112,"_90":3103,"_3104":5140},"00:18:59.360",{"_20":5120,"_90":3103,"_3104":5142},"00:19:05.279",{"_20":5128,"_90":3103,"_3104":5144},"00:19:09.760",{"_20":5136,"_90":3103,"_3104":5146},"00:19:12.120",{"_20":5148,"_90":266,"_267":5149,"_275":5179,"_277":278},"08d9957a69a2",[5150,5155,5158,5163,5166,5171,5174],{"_20":5151,"_90":271,"_272":5152,"_141":5154},"53832ea0dfea",[5153],"d4b7b13","Maybe pieces were built by a contractor or another outside firm.",{"_20":5156,"_90":271,"_272":5157,"_141":3071},"5b6f31b1588b",[],{"_20":5159,"_90":271,"_272":5160,"_141":5162},"b47f0a96629e",[5161],"7188696","They might have been brought in during a corporate merger or an acquisition.",{"_20":5164,"_90":271,"_272":5165,"_141":3071},"905936d0ffc3",[],{"_20":5167,"_90":271,"_272":5168,"_141":5170},"dd805b04f2c5",[5169],"3037842","They have all this baggage that accumulates.",{"_20":5172,"_90":271,"_272":5173,"_141":3071},"30d1b497e50b",[],{"_20":5175,"_90":271,"_272":5176,"_141":5178},"9d2a5b280be9",[5177],"19728e2","And they're not going to get resources to get updated for modern tooling.",[5180,5182,5184,5186],{"_20":5153,"_90":3103,"_3104":5181},"00:19:24.580",{"_20":5161,"_90":3103,"_3104":5183},"00:19:28.900",{"_20":5169,"_90":3103,"_3104":5185},"00:19:32.900",{"_20":5177,"_90":3103,"_3104":5187},"00:19:36.919",{"_20":5189,"_90":266,"_267":5190,"_275":5220,"_277":278},"207ea132fb38",[5191,5196,5199,5204,5207,5212,5215],{"_20":5192,"_90":271,"_272":5193,"_141":5195},"e56f5f668692",[5194],"b7494a4","I've worked with teams who have legitimately lost their source code in some major disaster.",{"_20":5197,"_90":271,"_272":5198,"_141":3071},"4c305ea152f4",[],{"_20":5200,"_90":271,"_272":5201,"_141":5203},"a00e31d4d360",[5202],"50fdc4e","So they're never going to be able to go back and step back into that code and update it for anything, right?",{"_20":5205,"_90":271,"_272":5206,"_141":3071},"d26be34d9b87",[],{"_20":5208,"_90":271,"_272":5209,"_141":5211},"b0e255cc3926",[5210],"2c2f593","So hybrid legacy environments, they're gonna persist.",{"_20":5213,"_90":271,"_272":5214,"_141":3071},"d68072ed3c4a",[],{"_20":5216,"_90":271,"_272":5217,"_141":5219},"78ea3a9d726f",[5218],"e3ba9ff","They're gonna keep running for a while.",[5221,5223,5225,5227],{"_20":5194,"_90":3103,"_3104":5222},"00:19:42.200",{"_20":5202,"_90":3103,"_3104":5224},"00:19:48.360",{"_20":5210,"_90":3103,"_3104":5226},"00:19:56.620",{"_20":5218,"_90":3103,"_3104":5228},"00:20:00.460",{"_20":5230,"_90":266,"_267":5231,"_275":5261,"_277":278},"430b14c82028",[5232,5237,5240,5245,5248,5253,5256],{"_20":5233,"_90":271,"_272":5234,"_141":5236},"ba50419f1a4a",[5235],"c3549bd","And we need to be mindful of that",{"_20":5238,"_90":271,"_272":5239,"_141":3071},"e91759956233",[],{"_20":5241,"_90":271,"_272":5242,"_141":5244},"9f7982228970",[5243],"3dbbce7","rather than dismissive of them. I think that was one of the things that early DevOps kind of misstepped on.",{"_20":5246,"_90":271,"_272":5247,"_141":3071},"4521a1a186f5",[],{"_20":5249,"_90":271,"_272":5250,"_141":5252},"973739fb9ef1",[5251],"753743d","We got a lot of questions like, well, can I DevOps the mainframe?",{"_20":5254,"_90":271,"_272":5255,"_141":3071},"88a876ebbef8",[],{"_20":5257,"_90":271,"_272":5258,"_141":5260},"5e258e81be1a",[5259],"f6ee732","And we didn't do a good job of answering that.",[5262,5264,5266,5268],{"_20":5235,"_90":3103,"_3104":5263},"00:20:02.380",{"_20":5243,"_90":3103,"_3104":5265},"00:20:04.820",{"_20":5251,"_90":3103,"_3104":5267},"00:20:10.420",{"_20":5259,"_90":3103,"_3104":5269},"00:20:14.880",{"_20":5271,"_90":266,"_267":5272,"_275":5286,"_277":278},"7aebd201922f",[5273,5278,5281],{"_20":5274,"_90":271,"_272":5275,"_141":5277},"d39eeb988b61",[5276],"3430081","So being aware that there are all these other components out there that are stable and are kind of set in stone,",{"_20":5279,"_90":271,"_272":5280,"_141":3071},"347511269445",[],{"_20":5282,"_90":271,"_272":5283,"_141":5285},"390d473a556d",[5284],"a854163","there's not gonna be a lot of enthusiasm for going back and adjusting them for new tooling.",[5287,5289],{"_20":5276,"_90":3103,"_3104":5288},"00:20:17.040",{"_20":5284,"_90":3103,"_3104":5290},"00:20:27.419",{"_20":5292,"_90":266,"_267":5293,"_275":5298,"_277":312},"75a5a146a3d2",[5294],{"_20":5295,"_90":271,"_272":5296,"_141":5297},"878be4524709",[],"Ready for the Future",[],{"_20":5300,"_90":266,"_267":5301,"_275":5307,"_277":278},"638cd3cb0534",[5302],{"_20":5303,"_90":271,"_272":5304,"_141":5306},"8f25be210d5c",[5305],"915d20d","So the future is gonna get here for all of us eventually, right?",[5308],{"_20":5305,"_90":3103,"_3104":5309},"00:20:32.460",{"_20":5311,"_90":266,"_267":5312,"_275":5334,"_277":599},"8683d186d3c0",[5313,5318,5321,5326,5329],{"_20":5314,"_90":271,"_272":5315,"_141":5317},"1bd0cd1d5ea0",[5316],"5343bf4","We want to get ourselves ready by focusing on what will make the best use of our scarcest resources, which are our human beings.",{"_20":5319,"_90":271,"_272":5320,"_141":3071},"72c0f73e8c27",[],{"_20":5322,"_90":271,"_272":5323,"_141":5325},"1719e5413f7a",[5324],"96e63af","We're going to stop wasting their time on things that we don't need creativity or lateral thinking for.",{"_20":5327,"_90":271,"_272":5328,"_141":3071},"d1c3836ed098",[],{"_20":5330,"_90":271,"_272":5331,"_141":5333},"6c89256f9cd2",[5332],"045177b","We want to focus their efforts on innovating for the customer.",[5335,5337,5339],{"_20":5316,"_90":3103,"_3104":5336},"00:20:36.100",{"_20":5324,"_90":3103,"_3104":5338},"00:20:43.460",{"_20":5332,"_90":3103,"_3104":5340},"00:20:49.940",{"_20":5342,"_90":266,"_267":5343,"_275":5373,"_277":278},"c763917b2c18",[5344,5349,5352,5357,5360,5365,5368],{"_20":5345,"_90":271,"_272":5346,"_141":5348},"b6d173f79116",[5347],"f0d845b","So we want to be planning for automation first, making that our first-class citizen, and redeploying all those people resources to focus on innovation",{"_20":5350,"_90":271,"_272":5351,"_141":3071},"ab406e68690e",[],{"_20":5353,"_90":271,"_272":5354,"_141":5356},"cba976c7063a",[5355],"11c16a8","for our customers. So hopefully you found that interesting.",{"_20":5358,"_90":271,"_272":5359,"_141":3071},"74f51f3d391d",[],{"_20":5361,"_90":271,"_272":5362,"_141":5364},"a1b381101f43",[5363],"bdeed40","I am looking forward to seeing all of you in our future.",{"_20":5366,"_90":271,"_272":5367,"_141":3071},"3c2a4b23d72e",[],{"_20":5369,"_90":271,"_272":5370,"_141":5372},"e99bf18431fe",[5371],"ccd8331","So thank you.",[5374,5376,5378,5380],{"_20":5347,"_90":3103,"_3104":5375},"00:20:54.659",{"_20":5355,"_90":3103,"_3104":5377},"00:21:03.740",{"_20":5363,"_90":3103,"_3104":5379},"00:21:08.140",{"_20":5371,"_90":3103,"_3104":5381},"00:21:11.340",{"_90":5383,"_92":5384},"mux.video",{"_239":5385,"_241":5386,"_243":5387,"_90":5388,"_246":5389,"_5390":5391,"_7":5392,"_5446":5415,"_3012":5423,"_5447":5443,"_97":5448},"2026-09-23T20:47:19Z","930dc51e-6d41-4ad4-87df-6f0a9cb69279","UE8PQ8Bn1WR78tvU7u9hUl","mux.videoAsset","2026-09-23T20:48:26Z","assetId","XTtAo8Z202Fq3kEtswE4UrJVoeeJeZBAL",{"_5393":5394,"_5395":5396,"_1077":5397,"_5398":5399,"_2945":5391,"_5400":5401,"_5402":5403,"_5404":5405,"_5406":5407,"_5408":5409,"_5410":5403,"_5411":5386,"_5412":5413,"_5417":5418,"_5422":5405,"_3012":5423,"_5424":5425,"_5442":5443,"_5444":5445},"aspect_ratio","16:9","created_at","1790196482",1272.633333,"encoding_tier","smart","ingest_type","on_demand_direct_upload","master_access","none","max_resolution_tier","1080p","max_stored_frame_rate",30,"max_stored_resolution","HD","mp4_support","passthrough","playback_ids",[5414],{"_2945":5415,"_5416":3011},"fj2dQK9fStwL6He1t2ndCDv02kVME500Ef","policy","progress",{"_5417":5419,"_5420":5421},100,"state","completed","resolution_tier","ready","tracks",[5426,5435],{"_1077":5427,"_2945":5428,"_5429":5430,"_5431":540,"_5432":138,"_5433":5434},1272.566667,"vSNi00L4pmXQR4VQ01y01eV2pua02W4dBvgn","max_channel_layout","stereo","max_channels","primary","type","audio",{"_1077":5427,"_2945":5436,"_5437":5407,"_5438":5439,"_5440":5441,"_5433":245},"81yHd7NtmKwcP6hcAnpBOomk02ISA4CbQ7rW9XofNwMM","max_frame_rate","max_height",1080,"max_width",1920,"upload_id","2vP2T2mfCFFsfTCFHjHSh3k2D9QQK9wO","video_quality","plus","playbackId","uploadId","https://stream.mux.com/fj2dQK9fStwL6He1t2ndCDv02kVME500Ef","preview","pageNumber","actionData","errors"]

Minimizing Human Intervention in Incident Response

Speakers