[{"_1":2,"_9501":-5,"_9502":-5},"loaderData",{"_3":4,"_244":245},"root",{"_5":6,"_7":8,"_226":227},"ua","Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","data",{"_9":10,"_118":119,"_135":136},"footer",{"_11":12,"_13":9,"_14":15,"_16":9,"_17":18,"_19":20,"_27":28,"_104":105},"_createdAt","2022-05-17T11:04:58Z","_id","_rev","rROGsNu2aOWyG6N46TlATL","_type","_updatedAt","2025-08-13T23:00:00Z","emailSubscription",{"_21":22,"_23":24,"_25":26},"description","Unlock success from technical pioneers in software infrastructure. We send the best strategies, tools, and industry updates directly to your inbox.","targetURL","https://share.hsforms.com/1_KYaR4lAQm-C8_9H7r4MIwqdoyu","title","Stay in the loop","footerColumns",[29,61,85],{"_30":31,"_16":32,"_33":34,"_35":36,"_42":59,"_25":60},"_key","30a87ef6e7c1","footerColumn","heading","About","items",[37,45,50,55],{"_30":38,"_16":39,"_40":41,"_42":43,"_25":44},"3dfef1283ab5","footerLink","label","Portfolio","link","/portfolio","Navigate to portfolio page",{"_30":46,"_16":39,"_40":47,"_42":48,"_25":49},"42666d0b3987","Team","/team","Navigate to team page",{"_30":51,"_16":39,"_40":52,"_42":53,"_25":54},"398a9b50e214","News","/press","Navigate to press page",{"_30":56,"_16":39,"_40":57,"_42":58,"_25":57},"6df053133fb8","Apply","https://heavybit.typeform.com/to/tP7Lh7","/about","Navigate to about page",{"_30":62,"_16":32,"_33":63,"_35":64,"_42":83,"_25":84},"a584a6acf07b","Community",[65,69,74,79],{"_30":66,"_16":39,"_40":67,"_42":68,"_25":67},"4358699f4ca4","Events","/events",{"_30":70,"_16":39,"_40":71,"_42":72,"_25":73},"e2d5641ef7de","DevGuild","https://www.heavybit.com/devguild","Navigate to devguild page",{"_30":75,"_16":39,"_40":76,"_42":77,"_25":78},"feb038ce46a1","Jobs","/jobs","Portfolio Company Jobs",{"_30":80,"_16":39,"_40":81,"_42":82,"_25":81},"e22bfa6b2330","Inclusion","/community/diversity-and-inclusion","https://www.heavybit.com/community","Navigate to community page",{"_30":86,"_16":32,"_33":87,"_35":88,"_42":92,"_25":103},"ea618aa34610","Library",[89,94,99],{"_30":90,"_16":39,"_40":91,"_42":92,"_25":93},"c82a64edc4dd","Blog","https://www.heavybit.com/library/","Navigate to browse on library page",{"_30":95,"_16":39,"_40":96,"_42":97,"_25":98},"e1673433290e","Podcast Network","https://www.heavybit.com/library/podcasts","Navigate to podcasts page",{"_30":100,"_16":39,"_40":101,"_42":102,"_25":101},"96a97ceba3a2","DevToolsDigest","https://www.heavybit.com/devtoolsdigest","Navigate to library page","socialMedia",[106],{"_30":107,"_108":109,"_116":117},"ee025c96397c","icon",{"_16":110,"_111":112},"mainImage","asset",{"_113":114,"_16":115},"_ref","image-9d618da0948299428b29e5d499af3fa5369a24d2-20x20-svg","reference","url","https://twitter.com/heavybit","globalConfig",{"_120":121,"_11":125,"_13":118,"_14":126,"_127":128,"_16":118,"_17":134},"OGimage",{"_16":122,"_111":123},"image",{"_113":124,"_16":115},"image-32fdfae4fd13b8f9a3c5bf6b13a635068cb0ecbb-1332x529-png","2022-05-17T11:15:28Z","RDgKg7yO6ZYtH7Ot8nWSxU","_system",{"_129":130},"base",{"_131":118,"_132":133},"id","rev","2ZxAYmIihJuAs2TKuKQs3J","2025-08-19T13:26:15Z","header",{"_11":137,"_13":135,"_14":15,"_16":135,"_17":18,"_138":139,"_149":150},"2022-05-17T11:04:33Z","announcementBar",{"_140":141,"_142":143,"_144":145,"_146":147,"_116":148},"buttonText","Read Now","show",true,"start","2025-07-08T13:00:00.000Z","text","Why Heavybit raised its largest fund to date","https://heavybit.com/press/heavybit-reinventing-enterprise-infrastructure","headerNavLinks",[151,171,181,194,222],{"_30":152,"_16":153,"_154":143,"_155":156,"_40":34,"_42":59,"_169":170,"_25":60},"a8d641babed2","navLink","addDropdown","dropdown",[157,160,165,167],{"_30":158,"_16":159,"_40":47,"_42":48,"_25":49},"964627410473","dropdownLink",{"_30":161,"_16":159,"_40":162,"_42":163,"_25":164},"3d279775d73b","FAQ","/faq","Navigate to FAQ page",{"_30":166,"_16":159,"_40":52,"_42":53,"_25":54},"84e68f9f5c33",{"_30":168,"_16":159,"_40":57,"_42":58,"_25":57},"716c250fe888","mobileOnly",false,{"_30":172,"_16":153,"_154":143,"_155":173,"_40":41,"_42":43,"_169":170,"_25":44},"90b68b861c06",[174,179],{"_30":175,"_16":159,"_40":176,"_42":177,"_25":178},"d481fcbbd3af","Spotlights","/portfolio/spotlights","Navigate to spotlights page",{"_30":180,"_16":159,"_40":76,"_42":77,"_25":78},"e50fa43ad2e7",{"_30":182,"_16":153,"_154":143,"_155":183,"_40":63,"_42":193,"_169":170,"_25":84},"716b76469486",[184,186,190],{"_30":185,"_16":159,"_40":67,"_42":68,"_25":67},"3a177efc4649",{"_30":187,"_16":159,"_40":188,"_42":189,"_25":188},"1fb7c9c465d0","Advisors","/community/advisors",{"_30":191,"_16":159,"_40":71,"_42":192,"_25":73},"23077fca0e2f","/devguild","/community",{"_30":195,"_16":153,"_154":143,"_155":196,"_40":87,"_42":221,"_169":170,"_25":103},"4bec0dd665d4",[197,202,207,210,214,218],{"_30":198,"_16":159,"_40":199,"_42":200,"_25":201},"63e48d609548","Articles","/library?filter=articles","Read educational articles in Heavybit's Library",{"_30":203,"_16":159,"_40":204,"_42":205,"_25":206},"6526c69f5260","Videos","/library?filter=videos","Watch educational videos in Heavybit's Library",{"_30":208,"_16":159,"_40":96,"_42":209,"_25":98},"1f3fb936f3b9","/library/podcasts",{"_30":211,"_16":159,"_40":212,"_42":213,"_25":212},"ac06751f0372","Collections","/library/collections",{"_30":215,"_16":159,"_40":216,"_42":217,"_25":216},"2ead3bd14715","Reports","/reports",{"_30":219,"_16":159,"_40":101,"_42":220,"_25":101},"66773779036c","/devtoolsdigest","/library",{"_30":223,"_16":153,"_154":170,"_40":224,"_42":53,"_169":143,"_25":225},"3702498a502c","Contact","Navigate to contact page","ENV",{"_228":229,"_230":231,"_232":233,"_234":235,"_236":237,"_238":239,"_240":241,"_242":229,"_243":229},"SANITY_DATASET","production","ALGOLIA_INDEX","heavybit-library","ALGOLIA_APPLICATION_ID","0QCKBC1FMK","ALGOLIA_SITEWIDE_INDEX","heavybit-sitewide","ALGOLIA_JOBS_INDEX","heavybit-jobs","MUX_ENV_KEY","dRI98bL8ze03gYDL6-Ogle2Js","ALGOLIA_SEARCH_API_KEY","7733fdd199ec7af7ee2923ec39c3f036","MODE","VITE_CONTEXT","routes/library.video.$slug",{"_7":246,"_9499":170,"_9500":-5,"_301":3448},{"_247":248},"routeData",{"_11":249,"_13":250,"_14":251,"_16":252,"_17":253,"_254":143,"_255":-5,"_256":257,"_258":-5,"_259":260,"_261":262,"_278":279,"_280":281,"_3411":-5,"_3412":3413,"_3420":3421,"_3430":3431,"_299":260,"_3434":3435,"_3439":3440,"_301":3446,"_3449":-5,"_3450":3451,"_3485":260,"_3486":3487,"_25":3528,"_3529":3530,"_252":9401},"2025-09-19T15:26:21Z","8749e33b-9611-4108-9548-1fd8241c3edb","zFdk8ICwNOz1zwk2Oi7WsG","video","2025-09-24T18:00:24Z","afterEvent","agenda","collections",[],"cta","endDate","2025-09-24T17:30:00.000Z","excerpt",[263],{"_30":264,"_16":265,"_266":267,"_274":275,"_276":277},"c47b28479a51","block","children",[268],{"_30":269,"_16":270,"_271":272,"_146":273},"dcb7028600bf","span","marks",[],"Raza Habib argues that successful LLM-powered products depend far more on people, evaluation, and product practices than on model size alone.","markDefs",[],"style","normal","filmedOn","2025-08-21","library",{"_35":282,"_337":338},[283,306,322],{"_284":285,"_294":295,"_122":296,"_297":298,"_299":300,"_301":302,"_25":303,"_304":305},"content",[286],{"_30":287,"_16":265,"_266":288,"_274":293,"_276":277},"5209d0346f17",[289],{"_30":290,"_16":270,"_271":291,"_146":292},"c4c9c88c5ed3",[],"In episode 21 of Open Source Ready, Brian and John sit down with Chad Metcalf, CEO of Continue, to explore the rise of continuous AI. Chad shares his journey from early embedded systems and platform engineering to leading an open-source-focused company building coding agents. They discuss automation, developer trust, open source attribution, and how AI is reshaping workflows across the entire software lifecycle.",[],"duration","56:38","https://cdn.sanity.io/images/50q6fr1p/production/79201f7b95d6f360cc0df6a5a8678b607df6d5ed-3000x3000.jpg","objectID","37fbf445-0037-4c81-ba81-afe7498c47b1","publishDate","2025-09-10T07:10:00.000Z","slug","open-source-ready/ep-21-continuous-ai-with-chad-metcalf-of-continue","Open Source Ready Ep. #21, Continuous AI with Chad Metcalf of Continue","typeName","podcastEpisode",{"_284":307,"_294":316,"_122":317,"_297":318,"_299":319,"_301":320,"_25":321,"_304":305},[308],{"_30":309,"_16":265,"_266":310,"_274":315,"_276":277},"fb87126465f4",[311],{"_30":312,"_16":270,"_271":313,"_146":314},"804e0edaccdd",[],"In episode 43 of Generationship, Rachel explores the frontier of AI-driven cybersecurity with John Amaral, co-founder of Root.io. Together they unpack the promise and challenges of agentic systems that detect, patch, and remediate vulnerabilities automatically. This installment offers a mix of technical deep dives, AI optimism, and a vision of security that works more like an immune system than a fire alarm.",[],"36:55","https://cdn.sanity.io/images/50q6fr1p/production/20f1ba4a8726f5453a36db0c061b7373ec74981e-3000x3000.jpg","14aee4e7-4e62-4471-8662-7488afebb10d","2025-09-04T07:10:00.000Z","generationship/ep-43-security-woven-in-with-john-amaral","Generationship Ep. #43, Security Woven In with John Amaral",{"_284":323,"_294":332,"_122":333,"_297":334,"_299":319,"_301":335,"_25":336,"_304":305},[324],{"_30":325,"_16":265,"_266":326,"_274":331,"_276":277},"4f9548228bf8",[327],{"_30":328,"_16":270,"_271":329,"_146":330},"0456264a33f6",[],"In episode 13 of Platform Builders, Christine and Isaac explore the intersection of AI and medicine with ER physician Nathan Murray. Nathan shares how and why he founded DocAssistant, an AI-powered scribe and decision-support tool for emergency medicine. This conversation examines the challenges of integrating AI into clinical workflows and offers a unique perspective on the future of healthcare tech.",[],"38:19","https://cdn.sanity.io/images/50q6fr1p/production/7ea1ac95f080e818e315f3d91637dfc40a2b5855-3000x3000.jpg","1f1732c4-5a4d-4575-9beb-e61d56d16ec1","platform-builders/ep-13-mirroring-the-er-brain-with-ai-with-nathan-murray-of-docassistant","Platform Builders Ep. #13, Mirroring the ER Brain with AI with Nathan Murray of DocAssistant","recent",[339,2109,2710],{"_284":340,"_294":2102,"_122":2103,"_297":2104,"_299":2105,"_301":2106,"_25":2107,"_304":2108},[341,350,392,411,439,447,466,496,500,509,524,534,545,556,559,567,597,605,618,630,642,654,666,678,697,705,713,721,728,736,748,781,815,827,835,843,851,859,867,875,883,894,905,916,927,935,943,951,959,978,986,994,1002,1010,1018,1026,1034,1042,1050,1057,1065,1073,1081,1089,1096,1104,1112,1164,1172,1179,1186,1194,1202,1209,1228,1236,1244,1252,1260,1268,1276,1284,1292,1299,1307,1315,1322,1330,1337,1344,1352,1360,1367,1375,1383,1391,1406,1421,1436,1444,1451,1458,1466,1474,1482,1490,1498,1506,1514,1522,1530,1538,1546,1554,1561,1569,1576,1584,1603,1610,1617,1625,1633,1640,1648,1656,1663,1670,1678,1685,1693,1705,1717,1771,1779,1787,1795,1810,1818,1826,1834,1842,1850,1858,1866,1878,1890,1902,1914,1926,1938,1950,1962,2003,2011,2018,2025,2033,2040,2048,2055,2063,2071,2078,2086,2094],{"_30":342,"_16":265,"_266":343,"_274":348,"_276":349},"a6399587d8a1",[344],{"_30":345,"_16":270,"_271":346,"_146":347},"8ce5c5b51263",[],"Introduction to Training an LLM",[],"h2",{"_30":351,"_16":265,"_266":352,"_274":384,"_276":277},"0e69c3910248",[353,357,362,366,371,375,380],{"_30":354,"_16":270,"_271":355,"_146":356},"b5ac2b6b3fd3",[],"Organizations of all sizes and maturity levels are adopting artificial intelligence (AI) quickly. While enterprises are notorious for",{"_30":358,"_16":270,"_271":359,"_146":361},"65ae6e8a770d",[360],"977b250d5e29"," slower adoption",{"_30":363,"_16":270,"_271":364,"_146":365},"71d7c15b714b",[],", their retention rates for AI applications are now hitting",{"_30":367,"_16":270,"_271":368,"_146":370},"4b5d25a9ee77",[369],"0d11a753d96f"," 63% annually",{"_30":372,"_16":270,"_271":373,"_146":374},"ad3f903a3771",[],", up from 41% the previous year. So while there are arguably overhyped, generic use cases that could fizzle out, AI is",{"_30":376,"_16":270,"_271":377,"_146":379},"1ea7e8b7eef9",[378],"3539ba7296a9"," becoming stickier",{"_30":381,"_16":270,"_271":382,"_146":383},"4d41aef2957c",[]," in some places.",[385,388,390],{"_30":360,"_16":42,"_386":387},"href","https://www.heavybit.com/library/article/enterprise-ai-infrastructure-compliance-risks-adoption",{"_30":369,"_16":42,"_386":389},"https://www.forbes.com/councils/forbestechcouncil/2025/02/05/why-enterprise-ai-adoption-is-finally-reaching-its-tipping-point-a-guide-for-corporate-leaders/",{"_30":378,"_16":42,"_386":391},"https://www.heavybit.com/library/article/prompting-chatgpt-programming-llm-software-development",{"_30":393,"_16":265,"_266":394,"_274":408,"_276":277},"b6739aa2a6c2",[395,399,404],{"_30":396,"_16":270,"_271":397,"_146":398},"97b07bade036",[],"If you’re at an",{"_30":400,"_16":270,"_271":401,"_146":403},"b9622a6c7387",[402],"c62a39db8c0a"," organization that’s already leveraging generative AI",{"_30":405,"_16":270,"_271":406,"_146":407},"a77be8063224",[],", you may be tasked with exploring innovative applications for it. Large Language Models (LLMs) offer a range of opportunities, but while pre-trained LLMs like the ChatGPT family from OpenAI and the Llama family from META are excellent for general purposes, there’s a good chance your company will need custom LLM models that are a bit more bespoke.",[409],{"_30":402,"_16":42,"_386":410},"https://www.heavybit.com/library/article/future-of-ai-coding-genai",{"_30":412,"_16":110,"_413":414,"_111":415,"_417":418,"_419":414,"_420":421},"8681d46038f3","alt","Heavybit: Train LLMs on your own data",{"_113":416,"_16":115},"image-0266788f35bc1118319d84bbc32bd1b7b715dcb8-4000x2250-jpg","attribution","Heavybit","caption","detailedCaption",[422],{"_30":423,"_16":265,"_266":424,"_274":436,"_276":277},"15c937478e0a",[425,429],{"_30":426,"_16":270,"_271":427,"_146":428},"8cd616692996",[],"Training LLMs on your own data requires processing, model engineering, setup, testing, and iteration. ",{"_30":430,"_16":270,"_271":431,"_146":435},"3341ccdb16bb",[432,433,434],"ed5d975da551","underline","em","Click to zoom in on this infographic.",[437],{"_30":432,"_16":42,"_386":438},"https://cdn.sanity.io/images/50q6fr1p/production/0266788f35bc1118319d84bbc32bd1b7b715dcb8-4000x2250.jpg",{"_30":440,"_16":265,"_266":441,"_274":446,"_276":277},"b3e8ad3f66f9",[442],{"_30":443,"_16":270,"_271":444,"_146":445},"09110967b1ff",[],"",[],{"_30":448,"_16":265,"_266":449,"_274":463,"_276":277},"017e0464b625",[450,454,459],{"_30":451,"_16":270,"_271":452,"_146":453},"0d88d17d13d4",[],"Teams can optimize models for better on-the-fly performance using techniques like",{"_30":455,"_16":270,"_271":456,"_146":458},"72bca12cee19",[457],"cd448e36593c"," retrieval-augmented generation (RAG)",{"_30":460,"_16":270,"_271":461,"_146":462},"1c0f3bc08941",[],", or they can take the approach of training their LLM on their own data. (Note that a training project, the initialization of an AI model’s weights against a dataset, differs from a fine-tuning LLM project, which further trains pre-trained models and is often even more costly in terms of computational resources).",[464],{"_30":457,"_16":42,"_386":465},"https://www.heavybit.com/library/article/rag-vs-fine-tuning",{"_30":467,"_16":265,"_266":468,"_274":491,"_276":277},"9071a9e4fb58",[469,473,478,482,487],{"_30":470,"_16":270,"_271":471,"_146":472},"d37f5bff4703",[],"Whether you're building a chatbot with expertise fueled by internal documentation, generating domain-specific content, building a",{"_30":474,"_16":270,"_271":475,"_146":477},"ff69fa245d49",[476],"8a8f6d620c5d"," natural-language processing (NLP)",{"_30":479,"_16":270,"_271":480,"_146":481},"0724750bc08d",[],"-focused",{"_30":483,"_16":270,"_271":484,"_146":486},"5f30dce4a8ef",[485],"7f220939a0fd"," BERT model",{"_30":488,"_16":270,"_271":489,"_146":490},"cce76ef5fe75",[]," for a text data or text generation use case, or enhancing existing models with proprietary knowledge, this tutorial will walk you through the practical steps required to train a language model on your own dataset.",[492,494],{"_30":476,"_16":42,"_386":493},"https://en.wikipedia.org/wiki/Natural_language_processing",{"_30":485,"_16":42,"_386":495},"https://en.wikipedia.org/wiki/BERT_(language_model)",{"_30":497,"_16":122,"_111":498},"71f3d6c2eb40",{"_113":499,"_16":115},"image-57880beea577987aaefcbac3658955424d2dabbd-1200x2-jpg",{"_30":501,"_16":265,"_266":502,"_274":507,"_276":508},"9980872013aa",[503],{"_30":504,"_16":270,"_271":505,"_146":506},"8f0616078cf0",[],"More Resources on LLM Engineering",[],"h3",{"_30":510,"_16":265,"_266":511,"_517":518,"_519":520,"_274":521,"_276":277},"52d136e50b86",[512],{"_30":513,"_16":270,"_271":514,"_146":516},"3a1e7e239739",[515],"51f7019d065f","Article: LLM Fine-Tuning: A Guide for Engineering Teams in 2025","level",1,"listItem","bullet",[522],{"_30":515,"_16":42,"_386":523},"https://www.heavybit.com/library/article/llm-fine-tuning",{"_30":525,"_16":265,"_266":526,"_517":518,"_519":520,"_274":532,"_276":277},"b90365c48cb8",[527],{"_30":528,"_16":270,"_271":529,"_146":531},"0211029c56af",[530],"774350bf476d","Article: RAG vs. Fine-Tuning: What Dev Teams Need to Know",[533],{"_30":530,"_16":42,"_386":465},{"_30":535,"_16":265,"_266":536,"_517":518,"_519":520,"_274":542,"_276":277},"7a83b822584d",[537],{"_30":538,"_16":270,"_271":539,"_146":541},"8dcbb68adbde",[540],"be97ab70f307","Article: Synthetic Data for AI: Purpose and Use Cases",[543],{"_30":540,"_16":42,"_386":544},"https://www.heavybit.com/library/article/synthetic-data-for-ai",{"_30":546,"_16":265,"_266":547,"_517":518,"_519":520,"_274":553,"_276":277},"54d3ce84a16e",[548],{"_30":549,"_16":270,"_271":550,"_146":552},"d98b34615398",[551],"c886ace6e24a","Article: How to Create Data Pipelines",[554],{"_30":551,"_16":42,"_386":555},"https://www.heavybit.com/library/article/how-to-create-data-pipelines",{"_30":557,"_16":122,"_111":558},"2989adc68ef0",{"_113":499,"_16":115},{"_30":560,"_16":265,"_266":561,"_274":566,"_276":508},"3dc8452f6bb0",[562],{"_30":563,"_16":270,"_271":564,"_146":565},"0ef22361eadf",[],"What Is an LLM and Its Key Components?",[],{"_30":568,"_16":265,"_266":569,"_274":592,"_276":277},"4ab475bdf11e",[570,574,579,583,588],{"_30":571,"_16":270,"_271":572,"_146":573},"dfdf24648eb6",[],"If you’re reading this, there’s already a good chance you know about LLMs, here’s a brief recap. An LLM is a deep learning model trained to understand and generate human language. LLMs are commonly built on the",{"_30":575,"_16":270,"_271":576,"_146":578},"a42d067fb740",[577],"f10bc77397f6"," Transformer architecture",{"_30":580,"_16":270,"_271":581,"_146":582},"cbef3de8d5e2",[],", introduced in the seminal \"",{"_30":584,"_16":270,"_271":585,"_146":587},"de9052028c45",[586],"a7769067698b","Attention is All You Need",{"_30":589,"_16":270,"_271":590,"_146":591},"00adc4ff135f",[],"\" paper.",[593,595],{"_30":577,"_16":42,"_386":594},"https://en.wikipedia.org/wiki/Transformer_(deep_learning_architecture)",{"_30":586,"_16":42,"_386":596},"https://arxiv.org/abs/1706.03762",{"_30":598,"_16":265,"_266":599,"_274":604,"_276":277},"79e56bc43c57",[600],{"_30":601,"_16":270,"_271":602,"_146":603},"8589ac125a91",[],"Key components of an LLM include:",[],{"_30":606,"_16":265,"_266":607,"_517":518,"_519":520,"_274":617,"_276":277},"d16b8cd3ddfa",[608,613],{"_30":609,"_16":270,"_271":610,"_146":612},"91329b0ce3d4",[611],"strong","Tokenization:",{"_30":614,"_16":270,"_271":615,"_146":616},"24c7f1508da4",[]," Breaks inputted text into manageable units/tokens (such as words or subwords) so it’s machine-readable.",[],{"_30":619,"_16":265,"_266":620,"_517":518,"_519":520,"_274":629,"_276":277},"df1c73b6c0eb",[621,625],{"_30":622,"_16":270,"_271":623,"_146":624},"258fd9bced23",[611],"Embedding Layer:",{"_30":626,"_16":270,"_271":627,"_146":628},"8d813909c24d",[]," Maps tokens to high-dimensional vectors. These embeddings are updated during model training to capture task-specific meaning.",[],{"_30":631,"_16":265,"_266":632,"_517":518,"_519":520,"_274":641,"_276":277},"ce34cd758ab4",[633,637],{"_30":634,"_16":270,"_271":635,"_146":636},"1bca2ab083ee",[611],"Positional Encoding:",{"_30":638,"_16":270,"_271":639,"_146":640},"b8841f98c482",[]," Added to token embeddings to give the model information about the position of each token in the sequence.",[],{"_30":643,"_16":265,"_266":644,"_517":518,"_519":520,"_274":653,"_276":277},"76d55bc27217",[645,649],{"_30":646,"_16":270,"_271":647,"_146":648},"34f8cfebe47f",[611],"Attention Mechanisms:",{"_30":650,"_16":270,"_271":651,"_146":652},"cbb723a92424",[]," Allows the model to weigh the relationships between all tokens in a sequence, and dynamically focus on the most relevant components of the context for each token.",[],{"_30":655,"_16":265,"_266":656,"_517":518,"_519":520,"_274":665,"_276":277},"8b720d3c6d00",[657,661],{"_30":658,"_16":270,"_271":659,"_146":660},"7c253cfba82e",[611],"Feed-Forward Networks:",{"_30":662,"_16":270,"_271":663,"_146":664},"cf9a2b6d5a47",[]," Process each position independently, and enrich the model’s capacity to detect and process complex patterns per the model’s algorithms.",[],{"_30":667,"_16":265,"_266":668,"_517":518,"_519":520,"_274":677,"_276":277},"b2400571263c",[669,673],{"_30":670,"_16":270,"_271":671,"_146":672},"0366238f7b35",[611],"Output Layer:",{"_30":674,"_16":270,"_271":675,"_146":676},"cb097c884067",[]," Converts the final token representations into probabilities over the vocabulary, which then generates predictions, such as the next token in a sequence.",[],{"_30":679,"_16":265,"_266":680,"_274":694,"_276":277},"6c4571fec1e1",[681,685,690],{"_30":682,"_16":270,"_271":683,"_146":684},"d4c21dc4f2c0",[],"When combined, these elements of an LLM create a pipeline that transforms your inputted raw text into outputted",{"_30":686,"_16":270,"_271":687,"_146":689},"bfcf7e0e534a",[688],"955440818a39"," meaningful predictions",{"_30":691,"_16":270,"_271":692,"_146":693},"a648856fd6d8",[]," with nuanced understanding and context-aware sensitivity. Now let’s talk about how to apply that to your organization.",[695],{"_30":688,"_16":42,"_386":696},"https://www.heavybit.com/library/article/ai-inference",{"_30":698,"_16":265,"_266":699,"_274":704,"_276":349},"8012c53cf0be",[700],{"_30":701,"_16":270,"_271":702,"_146":703},"26dabdc33c73",[],"Why Train an LLM on Your Own Data?",[],{"_30":706,"_16":265,"_266":707,"_274":712,"_276":277},"d73ea082ccdf",[708],{"_30":709,"_16":270,"_271":710,"_146":711},"a2c65cd84ae4",[],"If you have a common medical problem, such as a cough or headache, you’re probably fine going to see a doctor who is a general practitioner. On the other hand, if there are recurring concerns about specific parts of your body, such as your knees or shoulders, you might want to schedule an appointment with someone who specializes in that field, such as an orthopedic surgeon. The value of your own LLM, trained to focus on your users’ needs, can be seen in a similar way.",[],{"_30":714,"_16":265,"_266":715,"_274":720,"_276":277},"02c6fe3d89a4",[716],{"_30":717,"_16":270,"_271":718,"_146":719},"5bf9c0a71c24",[],"While GPT-4, for instance, might be able to provide a broad overview of what your organization does at a high level — based on publicly available information — it’s not going to be able to do so with the same level of granular knowledge that is common amongst your co-workers.",[],{"_30":722,"_16":265,"_266":723,"_274":727,"_276":277},"c1b7a7e56c46",[724],{"_30":725,"_16":270,"_271":726,"_146":445},"a4ce084fe742",[],[],{"_30":729,"_16":265,"_266":730,"_274":735,"_276":277},"fdd766dfd996",[731],{"_30":732,"_16":270,"_271":733,"_146":734},"9dcf25f638a4",[],"By training an LLM on your own dataset, you can achieve:",[],{"_30":737,"_16":265,"_266":738,"_274":747,"_276":277},"2babe1df0a62",[739,743],{"_30":740,"_16":270,"_271":741,"_146":742},"c2345bf86f48",[611],"Domain Knowledge:",{"_30":744,"_16":270,"_271":745,"_146":746},"354423d39a3e",[]," General models will struggle with niche terminology or specialized workflows. By training your own LLM on your own specific data tailored to your use case (like support tickets, codebases, or datasheets), you ensure the model internalizes domain-specific vocabulary, abbreviations, and context.",[],{"_30":749,"_16":265,"_266":750,"_274":777,"_276":277},"eff652765c59",[751,755,759,764,768,773],{"_30":752,"_16":270,"_271":753,"_146":754},"c0dc64eaf562",[611],"Better Performance:",{"_30":756,"_16":270,"_271":757,"_146":758},"80720adb7a32",[]," Generic models can be imprecise on narrow tasks.",{"_30":760,"_16":270,"_271":761,"_146":763},"ba195bedbcbe",[762],"9810080ef9f3"," Fine-tuning",{"_30":765,"_16":270,"_271":766,"_146":767},"58145107b894",[]," on task-relevant examples, such as customer questions, code reviews, or internal documentation, significantly boosts accuracy, relevance, and reliability. For use cases where data freshness is critical, updating your data (such as via a",{"_30":769,"_16":270,"_271":770,"_146":772},"6e48850fe2cf",[771],"4a6eadff5615"," data pipeline",{"_30":774,"_16":270,"_271":775,"_146":776},"c317a37d875b",[],") using a real-time cadence may be necessary.",[778,780],{"_30":762,"_16":42,"_386":779},"https://www.heavybit.com/library/article/how-to-make-open-llms-work-in-practice",{"_30":771,"_16":42,"_386":555},{"_30":782,"_16":265,"_266":783,"_274":810,"_276":277},"fb7c76ae74a0",[784,788,792,797,801,806],{"_30":785,"_16":270,"_271":786,"_146":787},"10b3cc973f92",[611],"Privacy and Control:",{"_30":789,"_16":270,"_271":790,"_146":791},"4598d730d126",[]," If you choose to host and train your own LLM on-premise or within a secure cloud environment, you’ll ensure that sensitive, regulated, or proprietary data stays within your",{"_30":793,"_16":270,"_271":794,"_146":796},"47188695285e",[795],"408d06df4f75"," infrastructure",{"_30":798,"_16":270,"_271":799,"_146":800},"3e2d44731d4c",[],". This is especially",{"_30":802,"_16":270,"_271":803,"_146":805},"11676f23b056",[804],"673cf4527d26"," important for industries",{"_30":807,"_16":270,"_271":808,"_146":809},"f6289cade65e",[]," with compliance requirements or IP concerns.",[811,813],{"_30":795,"_16":42,"_386":812},"https://www.heavybit.com/library/collections/ai-infrastructure",{"_30":804,"_16":42,"_386":814},"https://www.heavybit.com/library/article/how-llm-guardrails-reduce-ai-risk-in-software-development",{"_30":816,"_16":265,"_266":817,"_274":826,"_276":277},"28a6bec21ee5",[818,822],{"_30":819,"_16":270,"_271":820,"_146":821},"c1c9b97d8529",[611],"Customization:",{"_30":823,"_16":270,"_271":824,"_146":825},"ce693571ded0",[]," By fine-tuning the model's highly-sensitive behavior, you can better match your organization’s brand voice, formatting standards, or response strategies. This leads to a more consistent and aligned user experience across your product or platform.",[],{"_30":828,"_16":265,"_266":829,"_274":834,"_276":277},"3c7b494ec018",[830],{"_30":831,"_16":270,"_271":832,"_146":833},"60eb9ac95a33",[],"Training an LLM on your own training data turns a general-purpose tool into a purpose-built solution aligned exclusively with your environment, goals, and constraints. That said, anyone who has successfully built, launched, and maintained in-house tools knows it can all seem easier said than done.",[],{"_30":836,"_16":265,"_266":837,"_274":842,"_276":349},"435fd9b87290",[838],{"_30":839,"_16":270,"_271":840,"_146":841},"8bbcff7894d2",[],"How to Train an LLM on Your Own Data: A Step-by-Step Guide",[],{"_30":844,"_16":265,"_266":845,"_274":850,"_276":277},"54d2393ad9f5",[846],{"_30":847,"_16":270,"_271":848,"_146":849},"8faaf6301622",[],"The success of training your own LLM using training data you own hinges on a variety of factors, but let’s outline the broad steps, using Python and open-source tools where available for simplicity.",[],{"_30":852,"_16":265,"_266":853,"_274":858,"_276":508},"d14efe9f79e3",[854],{"_30":855,"_16":270,"_271":856,"_146":857},"1be386a89492",[],"1. Collect and Process Your Data",[],{"_30":860,"_16":265,"_266":861,"_274":866,"_276":277},"f58a2424fb87",[862],{"_30":863,"_16":270,"_271":864,"_146":865},"f6d19573d3e0",[],"It’s a lot easier and efficient to train your model if your assembled training data is cleaned up and standardized. Below are steps and tools to get high-quality data ready (and build a custom dataset as needed):",[],{"_30":868,"_16":265,"_266":869,"_274":874,"_276":277},"a7be667fd782",[870],{"_30":871,"_16":270,"_271":872,"_146":873},"bcfc1277316e",[611],"Data Collection",[],{"_30":876,"_16":265,"_266":877,"_274":882,"_276":277},"b41d5175ce43",[878],{"_30":879,"_16":270,"_271":880,"_146":881},"4a633c39c71e",[],"Here are a few sources to either sample or start datasets:",[],{"_30":884,"_16":265,"_266":885,"_517":518,"_519":520,"_274":891,"_276":277},"b8faef395a35",[886],{"_30":887,"_16":270,"_271":888,"_146":890},"afb9f16478bd",[889],"cf5c693d8b8b","Hugging Face",[892],{"_30":889,"_16":42,"_386":893},"https://huggingface.co/datasets",{"_30":895,"_16":265,"_266":896,"_517":518,"_519":520,"_274":902,"_276":277},"d52c81ba1362",[897],{"_30":898,"_16":270,"_271":899,"_146":901},"5fe38fe8151d",[900],"6bfc539e4511","Kaggle",[903],{"_30":900,"_16":42,"_386":904},"https://www.kaggle.com/datasets",{"_30":906,"_16":265,"_266":907,"_517":518,"_519":520,"_274":913,"_276":277},"0020bcb8fa03",[908],{"_30":909,"_16":270,"_271":910,"_146":912},"032270edcf9f",[911],"6a0fc6040371","GitHub Repositories",[914],{"_30":911,"_16":42,"_386":915},"https://github.com/",{"_30":917,"_16":265,"_266":918,"_517":518,"_519":520,"_274":924,"_276":277},"87cf9b5e90ed",[919],{"_30":920,"_16":270,"_271":921,"_146":923},"a41bd5ab3e93",[922],"84dedaae3a5a","DataCamp",[925],{"_30":922,"_16":42,"_386":926},"https://www.datacamp.com/",{"_30":928,"_16":265,"_266":929,"_274":934,"_276":277},"0bc9363eb7b5",[930],{"_30":931,"_16":270,"_271":932,"_146":933},"f9015129e83e",[],"For the actual contents of your data, you’ll need to figure out what items make the most sense for your use case, such as:",[],{"_30":936,"_16":265,"_266":937,"_517":518,"_519":520,"_274":942,"_276":277},"b46620f2256d",[938],{"_30":939,"_16":270,"_271":940,"_146":941},"9258aa31c662",[],"Internal documentation",[],{"_30":944,"_16":265,"_266":945,"_517":518,"_519":520,"_274":950,"_276":277},"cba51ec4eb06",[946],{"_30":947,"_16":270,"_271":948,"_146":949},"f96929df15ae",[],"CRM or support tickets",[],{"_30":952,"_16":265,"_266":953,"_517":518,"_519":520,"_274":958,"_276":277},"381813992645",[954],{"_30":955,"_16":270,"_271":956,"_146":957},"5c86f748fbe4",[],"Chat logs or Slack messages",[],{"_30":960,"_16":265,"_266":961,"_274":975,"_276":277},"039644394654",[962,966,971],{"_30":963,"_16":270,"_271":964,"_146":965},"0d4af49a2851",[],"If you’re building a customer support LLM, for example, you can start with a dataset like",{"_30":967,"_16":270,"_271":968,"_146":970},"96562f5665bf",[969],"f278daf38b23"," MultiWOZ",{"_30":972,"_16":270,"_271":973,"_146":974},"b1bea17e7055",[],", which is a collection of multi-domain task-oriented dialogues.",[976],{"_30":969,"_16":42,"_386":977},"https://huggingface.co/datasets/pfb30/multi_woz_v22",{"_30":979,"_16":265,"_266":980,"_274":985,"_276":277},"5c27a11f692d",[981],{"_30":982,"_16":270,"_271":983,"_146":984},"02bce45544db",[611],"Preprocess Your Data",[],{"_30":987,"_16":265,"_266":988,"_274":993,"_276":277},"8e3d25feaee7",[989],{"_30":990,"_16":270,"_271":991,"_146":992},"326ace37baa5",[],"Once you’ve got your data, you’ll need to prepare it for training. Preprocessing your training data will result in greater efficiency and reduced latency. Here’s what you need to look for:",[],{"_30":995,"_16":265,"_266":996,"_517":518,"_519":520,"_274":1001,"_276":277},"7e6fea715c36",[997],{"_30":998,"_16":270,"_271":999,"_146":1000},"42564f134e7b",[],"Remove duplicates to avoid skewing your model",[],{"_30":1003,"_16":265,"_266":1004,"_517":518,"_519":520,"_274":1009,"_276":277},"36c003978211",[1005],{"_30":1006,"_16":270,"_271":1007,"_146":1008},"8fc4db96a9ca",[],"Watch for data gaps and remove rows with missing context",[],{"_30":1011,"_16":265,"_266":1012,"_517":518,"_519":520,"_274":1017,"_276":277},"b83a1da66d3b",[1013],{"_30":1014,"_16":270,"_271":1015,"_146":1016},"8a46aab4cf51",[],"Filter incomplete examples, such as dialogues without responses",[],{"_30":1019,"_16":265,"_266":1020,"_517":518,"_519":520,"_274":1025,"_276":277},"95a263361cd8",[1021],{"_30":1022,"_16":270,"_271":1023,"_146":1024},"5da8fe319dcc",[],"Label your data if needed for supervised fine-tuning",[],{"_30":1027,"_16":265,"_266":1028,"_517":518,"_519":520,"_274":1033,"_276":277},"f5c6e7caa203",[1029],{"_30":1030,"_16":270,"_271":1031,"_146":1032},"5a44eab38a8b",[],"Eliminate HTML tags and special characters",[],{"_30":1035,"_16":265,"_266":1036,"_517":518,"_519":520,"_274":1041,"_276":277},"35d65f06af5e",[1037],{"_30":1038,"_16":270,"_271":1039,"_146":1040},"4ceeccf4f939",[],"Normalize text (lowercasing, removing stopwords, etc.)",[],{"_30":1043,"_16":265,"_266":1044,"_274":1049,"_276":277},"7391d0d52865",[1045],{"_30":1046,"_16":270,"_271":1047,"_146":1048},"84a85bd49e14",[],"Finally, tokenize and convert your text to a numerical format using tools like Hugging Face's tokenizer:",[],{"_30":1051,"_16":265,"_266":1052,"_274":1056,"_276":277},"b1bf8c40d9f9",[1053],{"_30":1054,"_16":270,"_271":1055,"_146":445},"8bfe355d2893",[],[],{"_30":1058,"_16":265,"_266":1059,"_274":1064,"_276":277},"dfc9607744e1",[1060],{"_30":1061,"_16":270,"_271":1062,"_146":1063},"5de4855bdce9",[434,611],"python",[],{"_30":1066,"_16":265,"_266":1067,"_274":1072,"_276":277},"25dace9605dd",[1068],{"_30":1069,"_16":270,"_271":1070,"_146":1071},"6cfa6f96119e",[434,611],"from transformers import AutoTokenizer",[],{"_30":1074,"_16":265,"_266":1075,"_274":1080,"_276":277},"76fe1acc92bb",[1076],{"_30":1077,"_16":270,"_271":1078,"_146":1079},"6d9f44a5201e",[434,611],"sample_text = \"Hello, how can I help you today?\"",[],{"_30":1082,"_16":265,"_266":1083,"_274":1088,"_276":277},"4478003ae768",[1084],{"_30":1085,"_16":270,"_271":1086,"_146":1087},"691f6e5b3384",[434,611],"tokens = tokenizer(sample_text, padding=\"max_length\", truncation=True, return_tensors=\"pt\")",[],{"_30":1090,"_16":265,"_266":1091,"_274":1095,"_276":277},"9544c0bb6afb",[1092],{"_30":1093,"_16":270,"_271":1094,"_146":445},"faec8285f91d",[],[],{"_30":1097,"_16":265,"_266":1098,"_274":1103,"_276":277},"4dc3185cb77a",[1099],{"_30":1100,"_16":270,"_271":1101,"_146":1102},"ff2f0794ea0e",[],"You may also embed the tokens or assess attention layers later in the model pipeline, but at this stage, it’s a good idea to focus on ensuring your data is in a standardized, clean text form that will play nicely with any API you use to feed data to your model",[],{"_30":1105,"_16":265,"_266":1106,"_274":1111,"_276":508},"c560eec67861",[1107],{"_30":1108,"_16":270,"_271":1109,"_146":1110},"27d99ca7c7a5",[],"2. Build Your Model",[],{"_30":1113,"_16":265,"_266":1114,"_274":1155,"_276":277},"da614eefdf2d",[1115,1119,1124,1128,1133,1137,1142,1146,1151],{"_30":1116,"_16":270,"_271":1117,"_146":1118},"e9528b4d62d5",[],"At the heart of the majority of LLMs is the aforementioned Transformer architecture, but luckily you won’t need to build a full custom LLM from scratch. For example, you can use popular Python frameworks such as",{"_30":1120,"_16":270,"_271":1121,"_146":1123},"e0030fc44f07",[1122],"200df259a4c4"," PyTorch",{"_30":1125,"_16":270,"_271":1126,"_146":1127},"8a5b05dcd242",[]," and",{"_30":1129,"_16":270,"_271":1130,"_146":1132},"bb5fef39af3c",[1131],"fbf3c26796e8"," TensorFlow",{"_30":1134,"_16":270,"_271":1135,"_146":1136},"3253e9996071",[],", combined with libraries like",{"_30":1138,"_16":270,"_271":1139,"_146":1141},"6aff73665bb1",[1140],"f93601855d0c"," Hugging Face Transformers",{"_30":1143,"_16":270,"_271":1144,"_146":1145},"3092fbde2553",[]," or",{"_30":1147,"_16":270,"_271":1148,"_146":1150},"309c8187d5e2",[1149],"7ef6bde3c13c"," Langchain",{"_30":1152,"_16":270,"_271":1153,"_146":1154},"765dc39082a0",[]," to work with a prebuilt base model or customize your own.",[1156,1158,1160,1162],{"_30":1122,"_16":42,"_386":1157},"https://pytorch.org/",{"_30":1131,"_16":42,"_386":1159},"https://www.tensorflow.org/",{"_30":1140,"_16":42,"_386":1161},"https://huggingface.co/docs/transformers/en/index",{"_30":1149,"_16":42,"_386":1163},"https://python.langchain.com/docs/introduction/",{"_30":1165,"_16":265,"_266":1166,"_274":1171,"_276":277},"93ea0729a49c",[1167],{"_30":1168,"_16":270,"_271":1169,"_146":1170},"cafc718a6a32",[],"Here’s an illustrative example of loading a pretrained model using Hugging Face Transformers:",[],{"_30":1173,"_16":265,"_266":1174,"_274":1178,"_276":277},"63dede9551ce",[1175],{"_30":1176,"_16":270,"_271":1177,"_146":445},"ee67cbf2c70b",[],[],{"_30":1180,"_16":265,"_266":1181,"_274":1185,"_276":277},"0ba5c86cd007",[1182],{"_30":1183,"_16":270,"_271":1184,"_146":1063},"ac19618c6a0e",[434,611],[],{"_30":1187,"_16":265,"_266":1188,"_274":1193,"_276":277},"6f77aa6f1c9c",[1189],{"_30":1190,"_16":270,"_271":1191,"_146":1192},"3e50d9f0cfc0",[434,611],"from transformers import AutoModelForCausalLM",[],{"_30":1195,"_16":265,"_266":1196,"_274":1201,"_276":277},"8dcdf6695d87",[1197],{"_30":1198,"_16":270,"_271":1199,"_146":1200},"970ab77ab2de",[434,611],"model = AutoModelForCausalLM.from_pretrained(\"gpt2\")",[],{"_30":1203,"_16":265,"_266":1204,"_274":1208,"_276":277},"203978dc118f",[1205],{"_30":1206,"_16":270,"_271":1207,"_146":445},"04625271c687",[],[],{"_30":1210,"_16":265,"_266":1211,"_274":1225,"_276":277},"6f37d598e2f4",[1212,1216,1221],{"_30":1213,"_16":270,"_271":1214,"_146":1215},"9006358929e6",[],"The above makes sense if you're fine-tuning a full model. For efficient training, consider techniques like",{"_30":1217,"_16":270,"_271":1218,"_146":1220},"900df7a0682d",[1219],"4a8195ac5459"," LoRA",{"_30":1222,"_16":270,"_271":1223,"_146":1224},"ec21d03def74",[]," or adapters that let you train only a subset of the parameters.",[1226],{"_30":1219,"_16":42,"_386":1227},"https://huggingface.co/docs/diffusers/main/en/training/lora",{"_30":1229,"_16":265,"_266":1230,"_274":1235,"_276":508},"12c9b343c4c2",[1231],{"_30":1232,"_16":270,"_271":1233,"_146":1234},"545c044ae56c",[],"3. Set Up Your Environment",[],{"_30":1237,"_16":265,"_266":1238,"_274":1243,"_276":277},"da93f4b93555",[1239],{"_30":1240,"_16":270,"_271":1241,"_146":1242},"925494fdcaa1",[],"You’ll need a proper development environment to train and test an LLM, which requires high-performance capabilities. Here are some examples:",[],{"_30":1245,"_16":265,"_266":1246,"_517":518,"_519":520,"_274":1251,"_276":277},"652732d37431",[1247],{"_30":1248,"_16":270,"_271":1249,"_146":1250},"e144a9e43d78",[],"Python 3.8+",[],{"_30":1253,"_16":265,"_266":1254,"_517":518,"_519":520,"_274":1259,"_276":277},"b39d3df5d72a",[1255],{"_30":1256,"_16":270,"_271":1257,"_146":1258},"0d9404af6080",[],"GPU support (e.g., NVIDIA CUDA)",[],{"_30":1261,"_16":265,"_266":1262,"_517":518,"_519":520,"_274":1267,"_276":277},"eb1bfa1dec94",[1263],{"_30":1264,"_16":270,"_271":1265,"_146":1266},"1357b1b2b480",[],"Libraries: PyTorch, TensorFlow, Hugging Face Transformers, Tokenizers",[],{"_30":1269,"_16":265,"_266":1270,"_517":518,"_519":520,"_274":1275,"_276":277},"cdf50a9996cd",[1271],{"_30":1272,"_16":270,"_271":1273,"_146":1274},"55a2af22ad53",[],"Cloud options: Google Colab, AWS EC2 with GPU, Paperspace, or Lambda Labs",[],{"_30":1277,"_16":265,"_266":1278,"_517":518,"_519":520,"_274":1283,"_276":277},"dba0c7a1970c",[1279],{"_30":1280,"_16":270,"_271":1281,"_146":1282},"0aed82202f3d",[],"On-prem",[],{"_30":1285,"_16":265,"_266":1286,"_274":1291,"_276":277},"043f2d88de10",[1287],{"_30":1288,"_16":270,"_271":1289,"_146":1290},"4574ddb6ff17",[],"Example setup using Hugging Face:",[],{"_30":1293,"_16":265,"_266":1294,"_274":1298,"_276":277},"ee45ab69a38d",[1295],{"_30":1296,"_16":270,"_271":1297,"_146":445},"3ea8d209a3bb",[],[],{"_30":1300,"_16":265,"_266":1301,"_274":1306,"_276":277},"a1a752e2ff51",[1302],{"_30":1303,"_16":270,"_271":1304,"_146":1305},"616efc5200fb",[434,611],"bash",[],{"_30":1308,"_16":265,"_266":1309,"_274":1314,"_276":277},"a8e0c85069e6",[1310],{"_30":1311,"_16":270,"_271":1312,"_146":1313},"2fd4a6aee241",[434,611],"pip install torch transformers datasets",[],{"_30":1316,"_16":265,"_266":1317,"_274":1321,"_276":277},"b8f7284a3e1d",[1318],{"_30":1319,"_16":270,"_271":1320,"_146":445},"cc64e5555dbf",[],[],{"_30":1323,"_16":265,"_266":1324,"_274":1329,"_276":277},"029c37f93b2f",[1325],{"_30":1326,"_16":270,"_271":1327,"_146":1328},"5524562e32af",[],"To run on GPU:",[],{"_30":1331,"_16":265,"_266":1332,"_274":1336,"_276":277},"a2e2c40210cb",[1333],{"_30":1334,"_16":270,"_271":1335,"_146":445},"a176184797f0",[],[],{"_30":1338,"_16":265,"_266":1339,"_274":1343,"_276":277},"c8be8a626163",[1340],{"_30":1341,"_16":270,"_271":1342,"_146":1063},"1f66927ea090",[434,611],[],{"_30":1345,"_16":265,"_266":1346,"_274":1351,"_276":277},"bb4a8cecbdbd",[1347],{"_30":1348,"_16":270,"_271":1349,"_146":1350},"beba790a965a",[434,611],"import torch",[],{"_30":1353,"_16":265,"_266":1354,"_274":1359,"_276":277},"04928efdf136",[1355],{"_30":1356,"_16":270,"_271":1357,"_146":1358},"0d2dbc751a49",[434,611],"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")",[],{"_30":1361,"_16":265,"_266":1362,"_274":1366,"_276":277},"2080dc4f0953",[1363],{"_30":1364,"_16":270,"_271":1365,"_146":445},"84cad8e85e6e",[],[],{"_30":1368,"_16":265,"_266":1369,"_274":1374,"_276":508},"0b4a107b670b",[1370],{"_30":1371,"_16":270,"_271":1372,"_146":1373},"814651d9c22a",[],"Step 4: Train and Test the Model",[],{"_30":1376,"_16":265,"_266":1377,"_274":1382,"_276":277},"f6a86ece58be",[1378],{"_30":1379,"_16":270,"_271":1380,"_146":1381},"aa18e01f1de3",[],"Now comes the actual model training process.",[],{"_30":1384,"_16":265,"_266":1385,"_274":1390,"_276":277},"caf57683b559",[1386],{"_30":1387,"_16":270,"_271":1388,"_146":1389},"d28b83b169e0",[],"First, set the initial parameters:",[],{"_30":1392,"_16":265,"_266":1393,"_517":518,"_519":520,"_274":1403,"_276":277},"ae9eb76ba164",[1394,1399],{"_30":1395,"_16":270,"_271":1396,"_146":1398},"d5117d8883e3",[1397],"a4605b8db5b5","Epochs:",{"_30":1400,"_16":270,"_271":1401,"_146":1402},"ef8aa982610d",[]," Number of passes through the dataset (e.g., 3–5 for small datasets)",[1404],{"_30":1397,"_16":42,"_386":1405},"https://www.geeksforgeeks.org/machine-learning/epoch-in-machine-learning/",{"_30":1407,"_16":265,"_266":1408,"_517":518,"_519":520,"_274":1418,"_276":277},"4cc6a2236ed0",[1409,1414],{"_30":1410,"_16":270,"_271":1411,"_146":1413},"574e3f1618f1",[1412],"599c74643dc6","Batch size:",{"_30":1415,"_16":270,"_271":1416,"_146":1417},"b786ccb8fb1f",[]," Number of samples per batch (e.g., 16 or 32)",[1419],{"_30":1412,"_16":42,"_386":1420},"https://stats.stackexchange.com/questions/153531/what-is-batch-size-in-neural-network",{"_30":1422,"_16":265,"_266":1423,"_517":518,"_519":520,"_274":1433,"_276":277},"5ee4c55546eb",[1424,1429],{"_30":1425,"_16":270,"_271":1426,"_146":1428},"784514c05f31",[1427],"d173e1187ec5","Learning rate:",{"_30":1430,"_16":270,"_271":1431,"_146":1432},"aa700e16f1e5",[]," Controls how much the model adjusts per update",[1434],{"_30":1427,"_16":42,"_386":1435},"https://www.geeksforgeeks.org/machine-learning/impact-of-learning-rate-on-a-model/",{"_30":1437,"_16":265,"_266":1438,"_274":1443,"_276":277},"d49d671b6b09",[1439],{"_30":1440,"_16":270,"_271":1441,"_146":1442},"41fe10dcd2d9",[],"Example using Hugging Face's Trainer:",[],{"_30":1445,"_16":265,"_266":1446,"_274":1450,"_276":277},"1e6689be26d7",[1447],{"_30":1448,"_16":270,"_271":1449,"_146":445},"6cac4a834e4d",[],[],{"_30":1452,"_16":265,"_266":1453,"_274":1457,"_276":277},"897f84749d87",[1454],{"_30":1455,"_16":270,"_271":1456,"_146":1063},"1d34cf9a7f11",[434,611],[],{"_30":1459,"_16":265,"_266":1460,"_274":1465,"_276":277},"008ac0441014",[1461],{"_30":1462,"_16":270,"_271":1463,"_146":1464},"f4d3608e8d51",[434,611],"from transformers import Trainer, TrainingArguments",[],{"_30":1467,"_16":265,"_266":1468,"_274":1473,"_276":277},"d64145a71ac4",[1469],{"_30":1470,"_16":270,"_271":1471,"_146":1472},"b6e966968b85",[434,611],"training_args = TrainingArguments(",[],{"_30":1475,"_16":265,"_266":1476,"_274":1481,"_276":277},"5a3fade82aaf",[1477],{"_30":1478,"_16":270,"_271":1479,"_146":1480},"81863f552763",[434,611],"output_dir=\"./results\",",[],{"_30":1483,"_16":265,"_266":1484,"_274":1489,"_276":277},"155aeabd7b60",[1485],{"_30":1486,"_16":270,"_271":1487,"_146":1488},"b050aadef515",[434,611],"per_device_train_batch_size=16,",[],{"_30":1491,"_16":265,"_266":1492,"_274":1497,"_276":277},"b3734f734fda",[1493],{"_30":1494,"_16":270,"_271":1495,"_146":1496},"717d2e9e7f51",[434,611],"num_train_epochs=3,",[],{"_30":1499,"_16":265,"_266":1500,"_274":1505,"_276":277},"2635137240b7",[1501],{"_30":1502,"_16":270,"_271":1503,"_146":1504},"f745f0d4d80c",[434,611],"logging_dir=\"./logs\"",[],{"_30":1507,"_16":265,"_266":1508,"_274":1513,"_276":277},"e6a182750abb",[1509],{"_30":1510,"_16":270,"_271":1511,"_146":1512},"88af63eed6b4",[434,611],")",[],{"_30":1515,"_16":265,"_266":1516,"_274":1521,"_276":277},"cb944ec05015",[1517],{"_30":1518,"_16":270,"_271":1519,"_146":1520},"34a425306ab9",[434,611],"trainer = Trainer(",[],{"_30":1523,"_16":265,"_266":1524,"_274":1529,"_276":277},"93feeebf4b25",[1525],{"_30":1526,"_16":270,"_271":1527,"_146":1528},"b60afaca18a0",[434,611],"model=model,",[],{"_30":1531,"_16":265,"_266":1532,"_274":1537,"_276":277},"cdc9aafe831d",[1533],{"_30":1534,"_16":270,"_271":1535,"_146":1536},"5ac7aa6b8705",[434,611],"args=training_args,",[],{"_30":1539,"_16":265,"_266":1540,"_274":1545,"_276":277},"760bf0516762",[1541],{"_30":1542,"_16":270,"_271":1543,"_146":1544},"42a6941e2e4e",[434,611],"train_dataset=train_dataset,",[],{"_30":1547,"_16":265,"_266":1548,"_274":1553,"_276":277},"f197bcd8125b",[1549],{"_30":1550,"_16":270,"_271":1551,"_146":1552},"c0838d965b2e",[434,611],"eval_dataset=eval_dataset",[],{"_30":1555,"_16":265,"_266":1556,"_274":1560,"_276":277},"c839bad36ca8",[1557],{"_30":1558,"_16":270,"_271":1559,"_146":1512},"c662b5d7baf6",[434,611],[],{"_30":1562,"_16":265,"_266":1563,"_274":1568,"_276":277},"7c3a52353a55",[1564],{"_30":1565,"_16":270,"_271":1566,"_146":1567},"3472f55aa982",[434,611],"trainer.train()",[],{"_30":1570,"_16":265,"_266":1571,"_274":1575,"_276":277},"de5474cb5b93",[1572],{"_30":1573,"_16":270,"_271":1574,"_146":445},"942d3eff0959",[],[],{"_30":1577,"_16":265,"_266":1578,"_274":1583,"_276":277},"9f37638ac791",[1579],{"_30":1580,"_16":270,"_271":1581,"_146":1582},"cd2f66a89499",[611],"Optimization with Stochastic Gradient Descent (SGD)",[],{"_30":1585,"_16":265,"_266":1586,"_274":1600,"_276":277},"319aee8e1d4e",[1587,1591,1596],{"_30":1588,"_16":270,"_271":1589,"_146":1590},"8ab49e9d762d",[],"Modern LLMs often use",{"_30":1592,"_16":270,"_271":1593,"_146":1595},"9d99474cf45b",[1594],"733ef4f43a8d"," Adam or AdamW",{"_30":1597,"_16":270,"_271":1598,"_146":1599},"6eb0407b30b4",[],", which converge faster and handle sparse gradients better than SGD. It may make sense to use SGD only for experimentation or comparison, though if you prefer classic SGD, here’s an example:",[1601],{"_30":1594,"_16":42,"_386":1602},"https://yassin01.medium.com/adam-vs-adamw-understanding-weight-decay-and-its-impact-on-model-performance-b7414f0af8a1",{"_30":1604,"_16":265,"_266":1605,"_274":1609,"_276":277},"efe5b3847037",[1606],{"_30":1607,"_16":270,"_271":1608,"_146":445},"b6948e91d7f1",[],[],{"_30":1611,"_16":265,"_266":1612,"_274":1616,"_276":277},"55ffc54d07f6",[1613],{"_30":1614,"_16":270,"_271":1615,"_146":1063},"845edae625ef",[434,611],[],{"_30":1618,"_16":265,"_266":1619,"_274":1624,"_276":277},"23561d65d7f9",[1620],{"_30":1621,"_16":270,"_271":1622,"_146":1623},"4a399447798e",[434,611],"import torch.optim as optim",[],{"_30":1626,"_16":265,"_266":1627,"_274":1632,"_276":277},"b0d4eb23547a",[1628],{"_30":1629,"_16":270,"_271":1630,"_146":1631},"4aea7793b08c",[434,611],"optimizer = optim.SGD(model.parameters(), lr=0.01)",[],{"_30":1634,"_16":265,"_266":1635,"_274":1639,"_276":277},"803c4966d679",[1636],{"_30":1637,"_16":270,"_271":1638,"_146":445},"fc5ba022e8ad",[611],[],{"_30":1641,"_16":265,"_266":1642,"_274":1647,"_276":277},"36275908bc18",[1643],{"_30":1644,"_16":270,"_271":1645,"_146":1646},"9a4f7de5915b",[611],"Test and Interpret the Results:",[],{"_30":1649,"_16":265,"_266":1650,"_274":1655,"_276":277},"ccb1288ce6fe",[1651],{"_30":1652,"_16":270,"_271":1653,"_146":1654},"c11c9dcee08c",[],"Your next steps should include evaluating your model’s loss, perplexity, and task-specific metrics:",[],{"_30":1657,"_16":265,"_266":1658,"_274":1662,"_276":277},"1beb74d560ad",[1659],{"_30":1660,"_16":270,"_271":1661,"_146":445},"9577f6aef483",[],[],{"_30":1664,"_16":265,"_266":1665,"_274":1669,"_276":277},"31fa048f7728",[1666],{"_30":1667,"_16":270,"_271":1668,"_146":1063},"6476628fb0ac",[434,611],[],{"_30":1671,"_16":265,"_266":1672,"_274":1677,"_276":277},"00f123d0cdaa",[1673],{"_30":1674,"_16":270,"_271":1675,"_146":1676},"dc207d176a0b",[434,611],"trainer.evaluate()",[],{"_30":1679,"_16":265,"_266":1680,"_274":1684,"_276":277},"7d1c847fb434",[1681],{"_30":1682,"_16":270,"_271":1683,"_146":445},"c50b2a7597b9",[611],[],{"_30":1686,"_16":265,"_266":1687,"_274":1692,"_276":277},"5c65b9597a49",[1688],{"_30":1689,"_16":270,"_271":1690,"_146":1691},"51185e078604",[],"To further break down the evaluation criteria, let’s look at each category:",[],{"_30":1694,"_16":265,"_266":1695,"_517":518,"_519":520,"_274":1704,"_276":277},"c7b1f3da17ee",[1696,1700],{"_30":1697,"_16":270,"_271":1698,"_146":1699},"3be42d86e398",[611],"Perplexity:",{"_30":1701,"_16":270,"_271":1702,"_146":1703},"7c1011819446",[]," The exponentiated average loss. This is the go-to metric that tells you how “confused” the model is when predicting the next token. Lower perplexity means more confidence and accuracy in its predictions.",[],{"_30":1706,"_16":265,"_266":1707,"_517":518,"_519":520,"_274":1716,"_276":277},"43decf5bbefc",[1708,1712],{"_30":1709,"_16":270,"_271":1710,"_146":1711},"5ae741664ee0",[611],"Loss:",{"_30":1713,"_16":270,"_271":1714,"_146":1715},"c86f3f642510",[]," How far off the model’s predictions are from the correct answers (ground truth). It's computed during training using a function, which is usually cross-entropy loss for language models.",[],{"_30":1718,"_16":265,"_266":1719,"_517":518,"_519":520,"_274":1762,"_276":277},"ee32d65601af",[1720,1724,1728,1733,1736,1741,1745,1750,1753,1758],{"_30":1721,"_16":270,"_271":1722,"_146":1723},"f716ad6339bf",[611],"Task-Specific Metrics:",{"_30":1725,"_16":270,"_271":1726,"_146":1727},"e5b50c99fa5d",[]," Measure the model’s performance on the actual job it’s intended to do, rather than just general language ability. For example, If your LLM is summarizing text, you might use",{"_30":1729,"_16":270,"_271":1730,"_146":1732},"eb487ba6a725",[1731],"ad16f5ad82ba"," ROUGE",{"_30":1734,"_16":270,"_271":1735,"_146":1145},"7dfbde7a427c",[],{"_30":1737,"_16":270,"_271":1738,"_146":1740},"a4652cab3750",[1739],"476e60e9c057"," BLEU",{"_30":1742,"_16":270,"_271":1743,"_146":1744},"25ca19ba4ef0",[]," scores. If it’s answering questions,",{"_30":1746,"_16":270,"_271":1747,"_146":1749},"f8dd778eb897",[1748],"e85726a61feb"," Exact Match (EM)",{"_30":1751,"_16":270,"_271":1752,"_146":1145},"7c94df5209f7",[],{"_30":1754,"_16":270,"_271":1755,"_146":1757},"2120813ad565",[1756],"1d0861d72c3f"," F1 score",{"_30":1759,"_16":270,"_271":1760,"_146":1761},"a04d9651ccbc",[]," might be better. Each metric captures success in a way that generic benchmarks cannot.",[1763,1765,1767,1769],{"_30":1731,"_16":42,"_386":1764},"https://en.wikipedia.org/wiki/ROUGE_(metric)#:~:text=ROUGE%2C%20or%20Recall%2DOriented%20Understudy,produced%20summary%20and%20the%20reference.",{"_30":1739,"_16":42,"_386":1766},"https://cloud.google.com/translate/docs/advanced/automl-evaluate#:~:text=BLEU%20(BiLingual%20Evaluation%20Understudy)%20is,of%20high%20quality%20reference%20translations.",{"_30":1748,"_16":42,"_386":1768},"https://huggingface.co/spaces/evaluate-metric/exact_match",{"_30":1756,"_16":42,"_386":1770},"https://arize.com/blog-course/f1-score/",{"_30":1772,"_16":265,"_266":1773,"_274":1778,"_276":277},"97559d290eaf",[1774],{"_30":1775,"_16":270,"_271":1776,"_146":1777},"233142bc42cf",[],"As a general rule, during training, you’ll want to measure log loss and perplexity after every epoch/batch. At the validation stage, you’ll calculate all your metrics on a held-out dataset.",[],{"_30":1780,"_16":265,"_266":1781,"_274":1786,"_276":508},"df1ea9b1376b",[1782],{"_30":1783,"_16":270,"_271":1784,"_146":1785},"045b2164cbeb",[],"Step 5: Modify Parameters (Hyperparameter Tuning)",[],{"_30":1788,"_16":265,"_266":1789,"_274":1794,"_276":277},"5fae69f58a3d",[1790],{"_30":1791,"_16":270,"_271":1792,"_146":1793},"78601768863c",[],"With proper tuning, your LLM will train faster, generalize better, and perform more accurately on its assigned custom tasks.",[],{"_30":1796,"_16":265,"_266":1797,"_274":1807,"_276":277},"97aca39f519d",[1798,1803],{"_30":1799,"_16":270,"_271":1800,"_146":1802},"6b8121d8a492",[1801],"42fb16328577","Hyperparameter tuning",{"_30":1804,"_16":270,"_271":1805,"_146":1806},"aeff30708c93",[]," is the process of systematically adjusting the configuration settings that control how a machine learning model learns during training. Unlike model parameters (like weights and biases, which are learned from data), hyperparameters are set manually before training begins.",[1808],{"_30":1801,"_16":42,"_386":1809},"https://www.geeksforgeeks.org/machine-learning/hyperparameter-tuning/",{"_30":1811,"_16":265,"_266":1812,"_274":1817,"_276":277},"6c72e6c7994a",[1813],{"_30":1814,"_16":270,"_271":1815,"_146":1816},"4cd88078b2f6",[],"When training LLMs, tuning hyperparameters is crucial to getting better performance, especially on custom or domain-specific datasets.",[],{"_30":1819,"_16":265,"_266":1820,"_274":1825,"_276":277},"f49d754c7a9d",[1821],{"_30":1822,"_16":270,"_271":1823,"_146":1824},"7c61fa736ffc",[],"Since LLMs are incredibly sensitive to how they’re trained, even small changes in hyperparameters can lead to noticeable differences in output quality or model accuracy. Once you determine the correct hyperparameters, you can:",[],{"_30":1827,"_16":265,"_266":1828,"_517":518,"_519":520,"_274":1833,"_276":277},"3f19f70694f4",[1829],{"_30":1830,"_16":270,"_271":1831,"_146":1832},"842428f1567d",[],"Speed up convergence",[],{"_30":1835,"_16":265,"_266":1836,"_517":518,"_519":520,"_274":1841,"_276":277},"554c7b4f5f72",[1837],{"_30":1838,"_16":270,"_271":1839,"_146":1840},"05fdf320eb25",[],"Prevent overfitting or underfitting",[],{"_30":1843,"_16":265,"_266":1844,"_517":518,"_519":520,"_274":1849,"_276":277},"24392989b78d",[1845],{"_30":1846,"_16":270,"_271":1847,"_146":1848},"4a4dce7ccb3e",[],"Improve generalization to unseen text",[],{"_30":1851,"_16":265,"_266":1852,"_517":518,"_519":520,"_274":1857,"_276":277},"40f5c7c7f77a",[1853],{"_30":1854,"_16":270,"_271":1855,"_146":1856},"f4c9553ee0d1",[],"Reduce training time and cost",[],{"_30":1859,"_16":265,"_266":1860,"_274":1865,"_276":277},"c529c93a665e",[1861],{"_30":1862,"_16":270,"_271":1863,"_146":1864},"06cabb907ee9",[],"Hyperparameters you can tweak include:",[],{"_30":1867,"_16":265,"_266":1868,"_517":518,"_519":520,"_274":1877,"_276":277},"974389a6076e",[1869,1873],{"_30":1870,"_16":270,"_271":1871,"_146":1872},"887a2f183482",[611],"Learning Rate:",{"_30":1874,"_16":270,"_271":1875,"_146":1876},"3dc75f96c597",[]," Controls how much model weights are adjusted during backpropagation.",[],{"_30":1879,"_16":265,"_266":1880,"_517":518,"_519":520,"_274":1889,"_276":277},"cec6fa306772",[1881,1885],{"_30":1882,"_16":270,"_271":1883,"_146":1884},"20588ef0694c",[611],"Batch Size:",{"_30":1886,"_16":270,"_271":1887,"_146":1888},"d229a4d0ef54",[]," Number of samples processed before the model updates weights.",[],{"_30":1891,"_16":265,"_266":1892,"_517":518,"_519":520,"_274":1901,"_276":277},"c713036aff40",[1893,1897],{"_30":1894,"_16":270,"_271":1895,"_146":1896},"2f7898094d9e",[611],"Number of Epochs:",{"_30":1898,"_16":270,"_271":1899,"_146":1900},"34c43e6442ee",[]," How many times the model sees the entire training dataset.",[],{"_30":1903,"_16":265,"_266":1904,"_517":518,"_519":520,"_274":1913,"_276":277},"71623e76255e",[1905,1909],{"_30":1906,"_16":270,"_271":1907,"_146":1908},"5e69272ebe6b",[611],"Weight Decay:",{"_30":1910,"_16":270,"_271":1911,"_146":1912},"23ec9cab36ae",[]," L2 regularization that penalizes large weights.",[],{"_30":1915,"_16":265,"_266":1916,"_517":518,"_519":520,"_274":1925,"_276":277},"1ea478e3de2a",[1917,1921],{"_30":1918,"_16":270,"_271":1919,"_146":1920},"79dced7bec7d",[611],"Dropout Rate:",{"_30":1922,"_16":270,"_271":1923,"_146":1924},"782ec0f63087",[]," The probability of dropping a unit during training to help prevent overfitting.",[],{"_30":1927,"_16":265,"_266":1928,"_517":518,"_519":520,"_274":1937,"_276":277},"a0645a6fde8e",[1929,1933],{"_30":1930,"_16":270,"_271":1931,"_146":1932},"a9ed516204f4",[611],"Warmup Steps:",{"_30":1934,"_16":270,"_271":1935,"_146":1936},"de30c74005db",[]," Gradually increases the learning rate from zero to its target value over these steps.",[],{"_30":1939,"_16":265,"_266":1940,"_517":518,"_519":520,"_274":1949,"_276":277},"eee5d08334f2",[1941,1945],{"_30":1942,"_16":270,"_271":1943,"_146":1944},"9c21862343ee",[611],"Gradient Clipping:",{"_30":1946,"_16":270,"_271":1947,"_146":1948},"c9778bc2f030",[]," Prevents exploding gradients by capping the norm of the gradient vector.",[],{"_30":1951,"_16":265,"_266":1952,"_517":518,"_519":520,"_274":1961,"_276":277},"4fc53cba0135",[1953,1957],{"_30":1954,"_16":270,"_271":1955,"_146":1956},"dde3c9107361",[611],"Sequence Length:",{"_30":1958,"_16":270,"_271":1959,"_146":1960},"96f8989694a5",[]," Defines how many tokens the model can handle at once.",[],{"_30":1963,"_16":265,"_266":1964,"_274":1996,"_276":277},"f2b558b3a426",[1965,1969,1974,1978,1983,1987,1992],{"_30":1966,"_16":270,"_271":1967,"_146":1968},"ce7f358346f6",[],"You can use tools like",{"_30":1970,"_16":270,"_271":1971,"_146":1973},"e9bb94e613d3",[1972],"5b3b6897ecb3"," Weights & Biases",{"_30":1975,"_16":270,"_271":1976,"_146":1977},"e709e132aeef",[],",",{"_30":1979,"_16":270,"_271":1980,"_146":1982},"700709d7efd0",[1981],"d1feea0d0b06"," Optuna",{"_30":1984,"_16":270,"_271":1985,"_146":1986},"7d24157de1f5",[],", or",{"_30":1988,"_16":270,"_271":1989,"_146":1991},"c7739b8777ed",[1990],"3fe99a3e8b12"," Ray Tune",{"_30":1993,"_16":270,"_271":1994,"_146":1995},"a1b380b98145",[]," to automate tuning. It’s a good idea to not try to tune everything at once, which can muddle your results. Instead, it can be better to prioritize by learning rate, batch size, and epochs first, then work your way down.",[1997,1999,2001],{"_30":1972,"_16":42,"_386":1998},"https://wandb.ai/site/",{"_30":1981,"_16":42,"_386":2000},"https://optuna.org/",{"_30":1990,"_16":42,"_386":2002},"https://docs.ray.io/en/latest/tune/index.html",{"_30":2004,"_16":265,"_266":2005,"_274":2010,"_276":277},"f389ac8f5cea",[2006],{"_30":2007,"_16":270,"_271":2008,"_146":2009},"d4b87daac937",[],"It’s also helpful to evaluate your model using metrics like perplexity, BLEU score, or custom task-based metrics. It’s also a good idea to use a validation dataset to avoid overfitting:",[],{"_30":2012,"_16":265,"_266":2013,"_274":2017,"_276":277},"0763d6965500",[2014],{"_30":2015,"_16":270,"_271":2016,"_146":445},"1d263e8f784d",[],[],{"_30":2019,"_16":265,"_266":2020,"_274":2024,"_276":277},"6fd651c4eedf",[2021],{"_30":2022,"_16":270,"_271":2023,"_146":1063},"1843b399e242",[611,434],[],{"_30":2026,"_16":265,"_266":2027,"_274":2032,"_276":277},"aefb24996daa",[2028],{"_30":2029,"_16":270,"_271":2030,"_146":2031},"962ff56cf38f",[434,611],"# Example of evaluating perplexity",[],{"_30":2034,"_16":265,"_266":2035,"_274":2039,"_276":277},"18f4b1e815e6",[2036],{"_30":2037,"_16":270,"_271":2038,"_146":1350},"218e28f0d136",[434,611],[],{"_30":2041,"_16":265,"_266":2042,"_274":2047,"_276":277},"bd8c86184b5e",[2043],{"_30":2044,"_16":270,"_271":2045,"_146":2046},"ca571658f114",[434,611],"import math",[],{"_30":2049,"_16":265,"_266":2050,"_274":2054,"_276":277},"3b926cb8a1e7",[2051],{"_30":2052,"_16":270,"_271":2053,"_146":445},"adb9bd6c811d",[434,611],[],{"_30":2056,"_16":265,"_266":2057,"_274":2062,"_276":277},"585346723dd6",[2058],{"_30":2059,"_16":270,"_271":2060,"_146":2061},"ea484efe11a9",[434,611],"loss = trainer.evaluate()[\"eval_loss\"]",[],{"_30":2064,"_16":265,"_266":2065,"_274":2070,"_276":277},"3b32039df124",[2066],{"_30":2067,"_16":270,"_271":2068,"_146":2069},"39a88b4f515e",[434,611],"perplexity = math.exp(loss)",[],{"_30":2072,"_16":265,"_266":2073,"_274":2077,"_276":277},"912a266897af",[2074],{"_30":2075,"_16":270,"_271":2076,"_146":445},"8d3ee1acf062",[434,611],[],{"_30":2079,"_16":265,"_266":2080,"_274":2085,"_276":349},"0b5bc24b5cd8",[2081],{"_30":2082,"_16":270,"_271":2083,"_146":2084},"64ddc42c1151",[],"Final Thoughts",[],{"_30":2087,"_16":265,"_266":2088,"_274":2093,"_276":277},"4abed40b0b9e",[2089],{"_30":2090,"_16":270,"_271":2091,"_146":2092},"c0790557994b",[],"Training an LLM on your own data is no longer the exclusive domain of large research teams. With open-source tools, pre-trained models, and cloud-based compute, individual developers and small teams can create powerful, domain-specific LLMs.",[],{"_30":2095,"_16":265,"_266":2096,"_274":2101,"_276":277},"175af515bc15",[2097],{"_30":2098,"_16":270,"_271":2099,"_146":2100},"1a5618536edd",[],"If you’re exploring how to use LLMs to enhance your product or automate domain-specific tasks, this process can be your foundation.",[],16,"https://cdn.sanity.io/images/50q6fr1p/production/8e8929c5f8dd5dc2ce680a42da0d34ae82eb1429-1200x630.png","9470de57-6303-4af4-b60d-1d1db51c2f96","2025-09-04T05:25:00.000Z","train-LLM-on-own-data","How to Train an LLM on Your Own Data: Beginner’s Guide","blogArticle",{"_284":2110,"_294":2704,"_122":2705,"_297":2706,"_299":2707,"_301":2708,"_25":2709,"_304":2108},[2111,2141,2167,2175,2244,2247,2255,2266,2276,2287,2298,2309,2312,2331,2339,2347,2355,2363,2371,2395,2411,2419,2427,2435,2451,2467,2475,2483,2491,2499,2507,2515,2523,2531,2539,2563,2593,2601,2617,2625,2633,2648,2672,2680,2688,2696],{"_30":2112,"_16":265,"_266":2113,"_274":2136,"_276":277},"43659b6391dd",[2114,2118,2123,2127,2132],{"_30":2115,"_16":270,"_271":2116,"_146":2117},"927b289006d6",[],"There’s a great deal of interest in whether AI can (or can’t) provide value by increasing the productivity of engineering teams. This interview with developer productivity expert ",{"_30":2119,"_16":270,"_271":2120,"_146":2122},"4cf5db46e155",[2121],"95e1fbdbf193","Laura Tacho",{"_30":2124,"_16":270,"_271":2125,"_146":2126},"40df936ad7f6",[],", CTO at ",{"_30":2128,"_16":270,"_271":2129,"_146":2131},"b802c9019cfb",[2130],"8ff8cc5bc838","DX",{"_30":2133,"_16":270,"_271":2134,"_146":2135},"846d79533c84",[]," and 10+ year veteran of software development, covers opportunities for engineering orgs to take advantage of AI tooling and actionable suggestions for AI startup founders to build stickier, more-successful products for engineering end-customers.",[2137,2139],{"_30":2121,"_16":42,"_386":2138},"https://www.linkedin.com/in/lauratacho/",{"_30":2130,"_16":42,"_386":2140},"https://getdx.com",{"_30":2142,"_16":110,"_413":2143,"_111":2144,"_417":2146,"_419":2147,"_420":2148},"05a2a798afb4","Heavybit: Laura Tacho on Developer Productivity",{"_113":2145,"_16":115},"image-6f4a116db24ba9b19bc0e228af8dbec9c5ced77e-1600x686-png","LeadDev","Laura Tacho on Developer Productivity",[2149],{"_30":2150,"_16":265,"_266":2151,"_274":2164,"_276":277},"4768f3449404",[2152,2156,2160],{"_30":2153,"_16":270,"_271":2154,"_146":2155},"62d291476462",[434],"DX CTO Laura Tacho discusses developer productivity at ",{"_30":2157,"_16":270,"_271":2158,"_146":2146},"436029a6c9e5",[2159,434,433,611],"23125357c064",{"_30":2161,"_16":270,"_271":2162,"_146":2163},"7435e2a5b07e",[434],". Image courtesy LeadDev.",[2165],{"_30":2159,"_16":42,"_386":2166},"https://www.youtube.com/watch?v=b2BZi9xpYUU",{"_30":2168,"_16":265,"_266":2169,"_274":2174,"_276":349},"211070079858",[2170],{"_30":2171,"_16":270,"_271":2172,"_146":2173},"31766a44c486",[],"How Are Engineering Teams Using AI?",[],{"_30":2176,"_16":265,"_266":2177,"_274":2231,"_276":277},"cae011a3a85c",[2178,2183,2187,2192,2196,2201,2205,2210,2213,2218,2222,2227],{"_30":2179,"_16":270,"_271":2180,"_146":2182},"279382289e8e",[2181],"3d8d095daee5","78%",{"_30":2184,"_16":270,"_271":2185,"_146":2186},"fe46d0de453d",[]," of surveyed companies use AI, though only ",{"_30":2188,"_16":270,"_271":2189,"_146":2191},"c368492e4095",[2190],"33cc7615edd1","3%",{"_30":2193,"_16":270,"_271":2194,"_146":2195},"428a143cc876",[]," of engineering leaders report achieving “very high” productivity gains. The most widely-reported use case for AI is code assistants, including high-profile products like ",{"_30":2197,"_16":270,"_271":2198,"_146":2200},"9f40c590a76e",[2199],"5864e3a8ad63","GitHub Copilot",{"_30":2202,"_16":270,"_271":2203,"_146":2204},"7eeec9ce4df5",[],", ",{"_30":2206,"_16":270,"_271":2207,"_146":2209},"a6d44e3c7077",[2208],"9e4782f74bfe","Claude Code",{"_30":2211,"_16":270,"_271":2212,"_146":2204},"e60b3bfde376",[],{"_30":2214,"_16":270,"_271":2215,"_146":2217},"db5cdf3e9e06",[2216],"05c6998d6e09","Cursor",{"_30":2219,"_16":270,"_271":2220,"_146":2221},"4f08f86ca1a3",[],", and ",{"_30":2223,"_16":270,"_271":2224,"_146":2226},"76270a624ed7",[2225],"be8ef595a7fd","Continue",{"_30":2228,"_16":270,"_271":2229,"_146":2230},"5e8f3dbd7e7d",[],", but there are still adoption challenges that might not have anything to do with productivity.",[2232,2234,2236,2238,2240,2242],{"_30":2181,"_16":42,"_386":2233},"https://hai.stanford.edu/ai-index/2025-ai-index-report",{"_30":2190,"_16":42,"_386":2235},"https://www.itpro.com/business/business-strategy/engineering-firms-see-little-productivity-benefit-from-use-of-ai",{"_30":2199,"_16":42,"_386":2237},"https://github.com/features/copilot",{"_30":2208,"_16":42,"_386":2239},"https://www.anthropic.com/claude-code",{"_30":2216,"_16":42,"_386":2241},"https://cursor.com/",{"_30":2225,"_16":42,"_386":2243},"https://www.continue.dev/",{"_30":2245,"_16":122,"_111":2246},"c9f62ac9e519",{"_113":499,"_16":115},{"_30":2248,"_16":265,"_266":2249,"_274":2254,"_276":508},"549ece4c149a",[2250],{"_30":2251,"_16":270,"_271":2252,"_146":2253},"c663a0619ca0",[],"More Resources on Dev Productivity & AI Adoption",[],{"_30":2256,"_16":265,"_266":2257,"_517":518,"_519":520,"_274":2263,"_276":277},"1d712341d1d8",[2258],{"_30":2259,"_16":270,"_271":2260,"_146":2262},"c3d5945a6357",[2261],"f7c6aea34970","Article: Productivity and Mental Health Resources for Remote Teams",[2264],{"_30":2261,"_16":42,"_386":2265},"https://www.heavybit.com/library/article/productivity-and-mental-health-resources-for-remote-teams",{"_30":2267,"_16":265,"_266":2268,"_517":518,"_519":520,"_274":2274,"_276":277},"26ad1de48ddf",[2269],{"_30":2270,"_16":270,"_271":2271,"_146":2273},"56d6a101d23b",[2272],"08f01e88b385","Article: Enterprise AI Infrastructure: Compliance, Risks, Adoption",[2275],{"_30":2272,"_16":42,"_386":387},{"_30":2277,"_16":265,"_266":2278,"_517":518,"_519":520,"_274":2284,"_276":277},"3361aa1844ed",[2279],{"_30":2280,"_16":270,"_271":2281,"_146":2283},"ea3c8829cb3c",[2282],"b8a655acb6c0","Article: How AI Is Reshaping Enterprise Infrastructure",[2285],{"_30":2282,"_16":42,"_386":2286},"https://www.heavybit.com/library/article/how-ai-is-reshaping-enterprise-infrastructure",{"_30":2288,"_16":265,"_266":2289,"_517":518,"_519":520,"_274":2295,"_276":277},"035d81f9dbda",[2290],{"_30":2291,"_16":270,"_271":2292,"_146":2294},"00b4f4a9f4c8",[2293],"e4a185d7155f","Article: The Future of AI Code Generation",[2296],{"_30":2293,"_16":42,"_386":2297},"https://www.heavybit.com/library/article/future-of-ai-code-generation",{"_30":2299,"_16":265,"_266":2300,"_517":518,"_519":520,"_274":2306,"_276":277},"1c9676f6c904",[2301],{"_30":2302,"_16":270,"_271":2303,"_146":2305},"4c377764a00f",[2304],"d68bb634ab4a","Article: What to Know About Pricing Developer Tools",[2307],{"_30":2304,"_16":42,"_386":2308},"https://www.heavybit.com/library/article/pricing-developer-tools",{"_30":2310,"_16":122,"_111":2311},"47727a0a412b",{"_113":499,"_16":115},{"_30":2313,"_16":265,"_266":2314,"_274":2328,"_276":277},"579f45ebc6b4",[2315,2319,2324],{"_30":2316,"_16":270,"_271":2317,"_146":2318},"53b723db8b75",[],"While coding may seem to make sense for AI use cases because of how familiar it is to devs, it may not be the best opportunity. Says Tacho, “We have to look across the whole software development lifecycle to see productivity gains. We did a study of 180+ companies, and we found that for developers using AI, the number one time-saving use case was ",{"_30":2320,"_16":270,"_271":2321,"_146":2323},"079f140309be",[2322],"57e204ada84b","stack trace",{"_30":2325,"_16":270,"_271":2326,"_146":2327},"7bb573c02a82",[]," analysis, not code generation.”",[2329],{"_30":2322,"_16":42,"_386":2330},"https://en.wikipedia.org/wiki/Stack_trace",{"_30":2332,"_16":265,"_266":2333,"_274":2338,"_276":349},"3046b26f6937",[2334],{"_30":2335,"_16":270,"_271":2336,"_146":2337},"c473f9c9622d",[],"Why Don’t Engineers Adopt AI?",[],{"_30":2340,"_16":265,"_266":2341,"_274":2346,"_276":277},"9769fdf968f2",[2342],{"_30":2343,"_16":270,"_271":2344,"_146":2345},"895c44f07acf",[],"The CTO suggests that while the difference between inflated expectations and the disappointing reality of cleaning up after messy, nondeterministic AI is an adoption barrier, there are practical obstacles as well. “I would say training and enablement are probably the largest barrier.”",[],{"_30":2348,"_16":265,"_266":2349,"_274":2354,"_276":277},"48dea0e75c3f",[2350],{"_30":2351,"_16":270,"_271":2352,"_146":2353},"bfe00b45ed97",[],"“It’s not intuitively obvious, even for teams with lots of enthusiasm and optimism, where the highest-value use cases are hiding for a particular organization. Some patterns need to be taught and coached, just like with any other tool. Companies that skip that step (and try to rely on the optimism and enthusiasm of an individual developer) may see some initial success, but then they're going to see that success taper off.”",[],{"_30":2356,"_16":265,"_266":2357,"_274":2362,"_276":349},"469487ec8b53",[2358],{"_30":2359,"_16":270,"_271":2360,"_146":2361},"a16b4178d3be",[],"What’s Really Missing: Targeted Goals & Outcomes",[],{"_30":2364,"_16":265,"_266":2365,"_274":2370,"_276":277},"dccd3220193f",[2366],{"_30":2367,"_16":270,"_271":2368,"_146":2369},"c580b1ba4ef8",[],"So yes, individual devs may not use the company Copilot account more frequently because they haven’t been drilled on prompting best practices. However, entire engineering organizations may not be getting as much as they could from AI tools because there’s a lack of strategy.",[],{"_30":2372,"_16":265,"_266":2373,"_274":2394,"_276":277},"ce1c6cc5fc06",[2374,2378,2382,2386,2390],{"_30":2375,"_16":270,"_271":2376,"_146":2377},"6adc5ab9486d",[],"“I think where the rubber isn't meeting the road is figuring out: What are the business problems in the company that have the most to gain from AI usage?” The CTO clarifies: “There ",{"_30":2379,"_16":270,"_271":2380,"_146":2381},"c5ba9c4f68ef",[434],"should",{"_30":2383,"_16":270,"_271":2384,"_146":2385},"f07293fa6cb1",[]," be room for experimentation. There should be time to try something new and figure out: OK, what can we do with this tool? But when it's only ",{"_30":2387,"_16":270,"_271":2388,"_146":2389},"5f9df2d9d4c6",[434],"that",{"_30":2391,"_16":270,"_271":2392,"_146":2393},"e379adfcef8b",[],", and we don't have targeted goals and outcomes, then things can go off the rails.”",[],{"_30":2396,"_16":265,"_266":2397,"_274":2410,"_276":277},"2806b926c74f",[2398,2402,2406],{"_30":2399,"_16":270,"_271":2400,"_146":2401},"20ca35d47b12",[],"“The companies where I see the most adoption and the most durable benefit from AI are ones that came to AI ",{"_30":2403,"_16":270,"_271":2404,"_146":2405},"616c0fc5205d",[434],"with a specific problem",{"_30":2407,"_16":270,"_271":2408,"_146":2409},"388b7f348ed9",[],". For example: Companies that needed to modernize legacy infrastructure, and hypothesized that AI would help them modernize legacy components much faster than they could have previously.”",[],{"_30":2412,"_16":265,"_266":2413,"_274":2418,"_276":277},"5b354084e16b",[2414],{"_30":2415,"_16":270,"_271":2416,"_146":2417},"12e82b130519",[],"“Organizations that want to see organizational benefit need to think about this on an organizational level, not just rely on individuals figuring out what to do.”",[],{"_30":2420,"_16":265,"_266":2421,"_274":2426,"_276":349},"67040ffdada8",[2422],{"_30":2423,"_16":270,"_271":2424,"_146":2425},"3192c3835391",[],"How Can Startups Make the Most of AI?",[],{"_30":2428,"_16":265,"_266":2429,"_274":2434,"_276":277},"c31c1f6f739b",[2430],{"_30":2431,"_16":270,"_271":2432,"_146":2433},"e85f2ae6eb02",[],"Tacho concedes that different organizations with different resources (and different amounts of runway) will need to take different approaches. “I think you need to look at it as a spectrum of targeted experiments vs. targeted use cases. That proportion needs to be adjusted based on your company’s size and stage.”",[],{"_30":2436,"_16":265,"_266":2437,"_274":2450,"_276":277},"13620c1b08a7",[2438,2442,2446],{"_30":2439,"_16":270,"_271":2440,"_146":2441},"685874eb3c2b",[],"“For startups, those ",{"_30":2443,"_16":270,"_271":2444,"_146":2445},"4c1536f1ca42",[434],"are",{"_30":2447,"_16":270,"_271":2448,"_146":2449},"d0220dd97143",[]," the environments where the individual can have a huge impact. So it would make sense for people to be experimenting, to be rapidly figuring out where they can exploit the capabilities of AI for your business. But you should document that as you go and make sure it's not just one individual deciding to do one-off things. Build a playbook of use cases that aren’t ephemeral.”",[],{"_30":2452,"_16":265,"_266":2453,"_274":2465,"_276":2466},"c228156c21bf",[2454,2458,2461],{"_30":2455,"_16":270,"_271":2456,"_146":2457},"9d6d3ffd9258",[],"The companies where I see the most adoption and the most durable benefit from AI are ones that came to AI ",{"_30":2459,"_16":270,"_271":2460,"_146":2405},"21b77ac6de87",[434],{"_30":2462,"_16":270,"_271":2463,"_146":2464},"a0dd1112265f",[],".\" -Laura Tacho / CTO, DX",[],"blockquote",{"_30":2468,"_16":265,"_266":2469,"_274":2474,"_276":277},"9cbbe1b594fb",[2470],{"_30":2471,"_16":270,"_271":2472,"_146":2473},"dfbb3abbb75b",[],"“Depending on the market pressures on your business, that's going to dictate how you think about a rollout. Obviously, for startups that rely heavily on individual rigor and urgency in order to get work done, the AI strategy is going to fit better with focus on individuals or small teams. Larger organizations that are dealing with decades’ worth of legacy software need to take a different approach.”",[],{"_30":2476,"_16":265,"_266":2477,"_274":2482,"_276":349},"b6fd364a6a61",[2478],{"_30":2479,"_16":270,"_271":2480,"_146":2481},"6383886779c0",[],"How Should Engineering Teams Adopt AI?",[],{"_30":2484,"_16":265,"_266":2485,"_274":2490,"_276":277},"14628ec7a3ab",[2486],{"_30":2487,"_16":270,"_271":2488,"_146":2489},"0644eb3d81e9",[],"The CTO empathizes with AI startup founders, who face mounting pressure from a market that moves incredibly fast. “I feel like AI tooling is being commoditized at a rate I have never seen before. A few months ago, when people evaluated tools, they were looking at which model they could use. It was a differentiating feature. And now you can use whichever model you want. Sometimes it feels like things change hourly.”",[],{"_30":2492,"_16":265,"_266":2493,"_274":2498,"_276":277},"8ef01b409a4f",[2494],{"_30":2495,"_16":270,"_271":2496,"_146":2497},"d51e49bfa956",[],"However, the CTO notes the increasingly competitive nature of some AI product lines is at odds with the organizational benefit of keeping a unified stack, despite individual developers continuing to crave autonomy. “This is a tough one. The reality is that engineering leads feel that business pressure for vendor consolidation. No company wants to be forking over more money for an arbitrary number of vendors that all more or less solve the same problems.”",[],{"_30":2500,"_16":265,"_266":2501,"_274":2506,"_276":277},"f7b36798e60a",[2502],{"_30":2503,"_16":270,"_271":2504,"_146":2505},"66f553aae4c5",[],"Tacho suggests that a viable alternative for startups is to offer developer stipends to allow them freedom of choice. “It’s definitely a reasonable alternative, especially in a startup environment that may be a bit more open with regards to governance, risk, and security. It’ll also depend on whether you work in consumer vs. another industry.”",[],{"_30":2508,"_16":265,"_266":2509,"_274":2514,"_276":277},"5b7ef7b4a362",[2510],{"_30":2511,"_16":270,"_271":2512,"_146":2513},"3aa561d546e0",[],"The CTO notes that other companies do attempt to consolidate around single vendors, using targeted proof-of-concept campaigns and running bake-off comparisons between competing vendors when needed. “Some companies decide that since the market's so volatile that they’re going to trial something for six months and then figure out if things stabilize.”",[],{"_30":2516,"_16":265,"_266":2517,"_274":2522,"_276":277},"0b4d6c188f7f",[2518],{"_30":2519,"_16":270,"_271":2520,"_146":2521},"27a8a1a77ff3",[],"“I think no matter what approach an organization takes, there are going to be shadow AI tools out there because these are tools that right now are not prohibitively expensive for developers to pay for by themselves, and that is the reality, and it happens everywhere.",[],{"_30":2524,"_16":265,"_266":2525,"_274":2530,"_276":349},"e0a9a743391f",[2526],{"_30":2527,"_16":270,"_271":2528,"_146":2529},"9db58c07f28f",[],"How Can AI Startups Target Developers as Customers?",[],{"_30":2532,"_16":265,"_266":2533,"_274":2538,"_276":277},"4a634dd45258",[2534],{"_30":2535,"_16":270,"_271":2536,"_146":2537},"331f1c91166e",[],"The CTO notes that while certain market factors appear to be changing, the top drivers for purchasing AI tools, or any tools, remain the same. “I just spoke to a bunch of other CTOs, and while cost was not as much of a factor in purchasing decisions last year, it definitely is this year.”",[],{"_30":2540,"_16":265,"_266":2541,"_274":2562,"_276":277},"aa2b664ed034",[2542,2546,2550,2554,2558],{"_30":2543,"_16":270,"_271":2544,"_146":2545},"09c47ebcee3c",[],"“But ",{"_30":2547,"_16":270,"_271":2548,"_146":2549},"f774fb057e93",[434],"accuracy",{"_30":2551,"_16":270,"_271":2552,"_146":2553},"3bcaf1a11cff",[]," and ",{"_30":2555,"_16":270,"_271":2556,"_146":2557},"e45a0323c78c",[434],"security",{"_30":2559,"_16":270,"_271":2560,"_146":2561},"9112abe3800d",[]," are still top drivers for purchasing decisions. I think there's some table-stakes aspects for any AI tool that engineering teams have come to expect. Beyond the table stakes, the more that tools can illustrate better ROI and costs, the better.” Tools that can make usage metrics and data readily available will play nicer with developer productivity tooling to paint a clearer picture of ROI.",[],{"_30":2564,"_16":265,"_266":2565,"_274":2592,"_276":277},"9aff8549c806",[2566,2570,2574,2577,2581,2584,2588],{"_30":2567,"_16":270,"_271":2568,"_146":2569},"f789e6fe71d6",[],"“For a long time, the primary metrics we had were ",{"_30":2571,"_16":270,"_271":2572,"_146":2573},"019ea183d6b0",[434],"acceptance rate",{"_30":2575,"_16":270,"_271":2576,"_146":2204},"59845b7e0654",[],{"_30":2578,"_16":270,"_271":2579,"_146":2580},"79541bcec86f",[434],"number of licenses",{"_30":2582,"_16":270,"_271":2583,"_146":2221},"8d36458ffd19",[],{"_30":2585,"_16":270,"_271":2586,"_146":2587},"0fd4f5313dd7",[434],"how much was being paid for them",{"_30":2589,"_16":270,"_271":2590,"_146":2591},"aad05ba988f9",[],". But there wasn’t much transparency when it came to how tools were actually being used. So focusing on a really robust admin API is a great way to ensure that you can build a defensible ROI case and get people to keep their wallets open.”",[],{"_30":2594,"_16":265,"_266":2595,"_274":2600,"_276":349},"5bb4ab577ce6",[2596],{"_30":2597,"_16":270,"_271":2598,"_146":2599},"477de88b4589",[],"Opportunities for AI Products Targeting Engineering Teams",[],{"_30":2602,"_16":265,"_266":2603,"_274":2616,"_276":277},"93865a87df07",[2604,2608,2612],{"_30":2605,"_16":270,"_271":2606,"_146":2607},"1273a0a32665",[],"“As mentioned, areas like general-purpose code authoring tools may be reaching a level of saturation. But beyond code generation, there are opportunities to target other use cases across the software development lifecycle: Requirements authoring, code review, or changelogs. For AI products targeting engineering teams, the most valuable use cases are the ones that are ",{"_30":2609,"_16":270,"_271":2610,"_146":2611},"c2bd9ce32bc2",[434],"under-solved",{"_30":2613,"_16":270,"_271":2614,"_146":2615},"a4a864a900e9",[]," right now.”",[],{"_30":2618,"_16":265,"_266":2619,"_274":2624,"_276":277},"6e64cdf2d46f",[2620],{"_30":2621,"_16":270,"_271":2622,"_146":2623},"91ee8f9c0a9f",[],"Of course, many teams are still concerned about the non-deterministic nature of GenAI and the havoc that a hallucinating LLM might have on their codebase. However, the CTO notes that organizations that have built-in organizational resiliency tend to be the ones that make the most of AI.",[],{"_30":2626,"_16":265,"_266":2627,"_274":2632,"_276":277},"bba9cbbaf459",[2628],{"_30":2629,"_16":270,"_271":2630,"_146":2631},"5cd8ab410955",[],"“When you introduce AI, in all its non-deterministic glory [to resilient orgs], they're not falling over. They're not bottlenecked. Whereas companies who don't [have organizational resilience] and think that AI will be a magic solution to all the problems they've been avoiding? Those companies are getting a reality check. The fundamentals don't go away just because you have AI.”",[],{"_30":2634,"_16":265,"_266":2635,"_274":2647,"_276":2466},"36375c00e3d1",[2636,2640,2643],{"_30":2637,"_16":270,"_271":2638,"_146":2639},"3df04aba26d5",[],"Beyond code generation, there are opportunities to target other use cases across the software development lifecycle. For AI products targeting engineering teams, the most valuable use cases are the ones that are ",{"_30":2641,"_16":270,"_271":2642,"_146":2611},"ba209d80d01d",[434],{"_30":2644,"_16":270,"_271":2645,"_146":2646},"3f23dea25456",[]," right now.\"",[],{"_30":2649,"_16":265,"_266":2650,"_274":2671,"_276":277},"1569292b9daf",[2651,2655,2659,2663,2667],{"_30":2652,"_16":270,"_271":2653,"_146":2654},"2875d555a615",[],"“I think developer experience is not an outcome of using AI tools really well. It's a ",{"_30":2656,"_16":270,"_271":2657,"_146":2658},"2df82cb79eb4",[434],"prerequisite",{"_30":2660,"_16":270,"_271":2661,"_146":2662},"5eb089fcacc6",[]," to use them well. What's good for an individual human developer is also good for an agent. There's extra stuff that you might want to do for agents, but the physics of what makes software or codebases easy to contribute to, easy to change...that's important for AI agents ",{"_30":2664,"_16":270,"_271":2665,"_146":2666},"eaf4db67aa68",[434],"and",{"_30":2668,"_16":270,"_271":2669,"_146":2670},"661fd03f5d8c",[]," for humans.",[],{"_30":2673,"_16":265,"_266":2674,"_274":2679,"_276":277},"96d624d478ea",[2675],{"_30":2676,"_16":270,"_271":2677,"_146":2678},"98e49bf3fdb0",[],"“AI is an amplifier. If you have parts of your system that are a bit garbage, they're going to be amplified garbage now. And if you have really solid engineering practices that have stayed ahead of the industry curve, then that's going to be amplified and you're going to get even better results.”",[],{"_30":2681,"_16":265,"_266":2682,"_274":2687,"_276":277},"0b66778a3150",[2683],{"_30":2684,"_16":270,"_271":2685,"_146":2686},"22dcc00060db",[],"For AI startup founders specifically targeting engineering orgs, the slickest user interfaces and best new user experiences will only go so far. If those engineering orgs that AI themselves are struggling to maintain stable, resilient systems, they may struggle to succeed with any AI product.",[],{"_30":2689,"_16":265,"_266":2690,"_274":2695,"_276":277},"b5599fcb63e6",[2691],{"_30":2692,"_16":270,"_271":2693,"_146":2694},"8791ada4d0c1",[],"“To those AI product builders who are concerned about whether their product will be sticky with customers, I might say: Maybe think about building tools that fix your customers’ rickety systems. Probably a less exciting problem to solve, but people have their wallets open for it right now.”",[],{"_30":2697,"_16":265,"_266":2698,"_274":2703,"_276":277},"7b92f345cc9a",[2699],{"_30":2700,"_16":270,"_271":2701,"_146":2702},"4ab8a0ba2d26",[],"\n",[],12,"https://cdn.sanity.io/images/50q6fr1p/production/64845befec4c1deaffc6207521e99246fb71edef-1200x630.png","831ab261-12d5-4714-89b7-6b565a0b8e28","2025-09-04T05:24:00.000Z","ai-productivity-for-engineering-teams","How to Drive AI Productivity and Value for Engineering Teams",{"_284":2711,"_294":3405,"_122":3406,"_297":3407,"_299":3408,"_301":3409,"_25":3410,"_304":2108},[2712,2742,2745,2753,2764,2775,2786,2797,2800,2808,2826,2834,2838,2855,2863,2871,2898,2917,2940,2948,2964,2972,2996,3004,3020,3049,3080,3088,3120,3144,3152,3160,3168,3176,3184,3192,3200,3216,3242,3250,3258,3266,3274,3282,3301,3317,3325,3341,3345,3363,3371,3389],{"_30":2713,"_16":265,"_266":2714,"_274":2737,"_276":277},"4a3ccc6e0b02",[2715,2719,2724,2728,2733],{"_30":2716,"_16":270,"_271":2717,"_146":2718},"50af355933eb",[],"As AI becomes more of a reality in software development, teams are experimenting with a variety of use cases, including documentation and technical writing. In this article, veteran technical writer (and co-author of ",{"_30":2720,"_16":270,"_271":2721,"_146":2723},"2a561750dd51",[2722,434],"1f6b823e0e55","Docs for Developers: An Engineer’s Field Guide to Technical Writing",{"_30":2725,"_16":270,"_271":2726,"_146":2727},"fa5267ed1611",[],") ",{"_30":2729,"_16":270,"_271":2730,"_146":2732},"3ce6f0076da4",[2731],"d75e2d3d5574","Heidi Waterhouse",{"_30":2734,"_16":270,"_271":2735,"_146":2736},"b898656ddc5a",[]," discusses some of the biggest mistakes teams make using AI for docs, as well as the best opportunities to use AI productively to support user journeys.",[2738,2740],{"_30":2722,"_16":42,"_386":2739},"https://www.oreilly.com/library/view/docs-for-developers/9781484272176/",{"_30":2731,"_16":42,"_386":2741},"https://www.linkedin.com/in/heidiwaterhouse/",{"_30":2743,"_16":122,"_111":2744},"477cb546ed1b",{"_113":499,"_16":115},{"_30":2746,"_16":265,"_266":2747,"_274":2752,"_276":277},"01d336e93aa4",[2748],{"_30":2749,"_16":270,"_271":2750,"_146":2751},"da16f6fc20c9",[434],"More Articles on How Documentation and Software Dev Are Evolving:",[],{"_30":2754,"_16":265,"_266":2755,"_517":518,"_519":520,"_274":2761,"_276":277},"d88cc0b8b563",[2756],{"_30":2757,"_16":270,"_271":2758,"_146":2760},"49f9e16ceba8",[2759],"a8aafb07e661","Article: Who Owns the Documentation and Why It Should Be the Robots",[2762],{"_30":2759,"_16":42,"_386":2763},"https://www.heavybit.com/library/article/who-owns-the-documentation-and-why-it-should-be-the-robots",{"_30":2765,"_16":265,"_266":2766,"_517":518,"_519":520,"_274":2772,"_276":277},"7c905d126058",[2767],{"_30":2768,"_16":270,"_271":2769,"_146":2771},"d160fc2af3ad",[2770],"5f3984a9440a","Podcast: Documentation Deep Dive with Megan Sullivan of Gatsby",[2773],{"_30":2770,"_16":42,"_386":2774},"https://www.heavybit.com/library/podcasts/jamstack-radio/ep-72-documentation-deep-dive-with-megan-sullivan-of-gatsby",{"_30":2776,"_16":265,"_266":2777,"_517":518,"_519":520,"_274":2783,"_276":277},"0445a011f1ea",[2778],{"_30":2779,"_16":270,"_271":2780,"_146":2782},"6cd54a27c3f0",[2781],"18bdfcac1850","Video: Building Great API Docs",[2784],{"_30":2781,"_16":42,"_386":2785},"https://www.heavybit.com/library/video/building-great-api-docs",{"_30":2787,"_16":265,"_266":2788,"_517":518,"_519":520,"_274":2794,"_276":277},"1f28c510ae73",[2789],{"_30":2790,"_16":270,"_271":2791,"_146":2793},"3d69fe2f2d3c",[2792],"d1a13de0db2c","Video: How Great Documentation Drives Developer Adoption",[2795],{"_30":2792,"_16":42,"_386":2796},"https://www.heavybit.com/library/video/how-great-documentation-drives-developer-adoption",{"_30":2798,"_16":122,"_111":2799},"e60010441415",{"_113":499,"_16":115},{"_30":2801,"_16":265,"_266":2802,"_274":2807,"_276":349},"7f164fc1bfaf",[2803],{"_30":2804,"_16":270,"_271":2805,"_146":2806},"650778ce26f1",[],"AI as an Assistant, not a Solution",[],{"_30":2809,"_16":265,"_266":2810,"_274":2823,"_276":277},"843e9d33c4ec",[2811,2815,2819],{"_30":2812,"_16":270,"_271":2813,"_146":2814},"afdf910f0891",[],"Waterhouse elaborates on her recent session from the ",{"_30":2816,"_16":270,"_271":2817,"_146":2146},"8cc715084bec",[2818],"5148e717c058",{"_30":2820,"_16":270,"_271":2821,"_146":2822},"2fbedd1eef1f",[]," conference, at which she spoke on a similar topic. “The focus of my talk was discussing things we know AI can do well, and how to make that fit with our understanding of what people need from documentation.”",[2824],{"_30":2818,"_16":42,"_386":2825},"https://leaddev.com/technical-direction/documentation-and-ai-how-to-write-right-now",{"_30":2827,"_16":265,"_266":2828,"_274":2833,"_276":277},"e892cd8ba5ca",[2829],{"_30":2830,"_16":270,"_271":2831,"_146":2832},"be321766a4cb",[],"The author suggests that the key to getting value out of large language models (LLMs) today is thinking about AI “as an assistant, and not a solution. Because thinking about it as a solution is what leads us down some unfortunate paths that are really hard to dig ourselves out of.”",[],{"_30":2835,"_16":122,"_111":2836},"ec283b1972ae",{"_113":2837,"_16":115},"image-c05b350e923e3a160a092e5d09583b1ae2fc4a29-1200x675-jpg",{"_30":2839,"_16":265,"_266":2840,"_274":2853,"_276":277},"311937c0e261",[2841,2845,2849],{"_30":2842,"_16":270,"_271":2843,"_146":2844},"03f24312b420",[434],"Heidi Waterhouse discusses AI in documentation at LeadDev. Image courtesy ",{"_30":2846,"_16":270,"_271":2847,"_146":2146},"26f5c63ef375",[2848,434],"5dd2636cd8e7",{"_30":2850,"_16":270,"_271":2851,"_146":2852},"71411bc4fe97",[434],".",[2854],{"_30":2848,"_16":42,"_386":2825},{"_30":2856,"_16":265,"_266":2857,"_274":2862,"_276":349},"8eaadeefbd22",[2858],{"_30":2859,"_16":270,"_271":2860,"_146":2861},"0643f9a25ded",[],"Are LLMs the Best Tool for Software Docs and Technical Writing?",[],{"_30":2864,"_16":265,"_266":2865,"_274":2870,"_276":508},"387e0f972cd4",[2866],{"_30":2867,"_16":270,"_271":2868,"_146":2869},"fe6ce8cb3294",[],"Freshness: Using RAG to Ensure Docs Stay Up-to-Date Instead",[],{"_30":2872,"_16":265,"_266":2873,"_274":2895,"_276":277},"a057742a9e19",[2874,2878,2883,2887,2891],{"_30":2875,"_16":270,"_271":2876,"_146":2877},"c95dc890564f",[],"Aside from hiring a good writer a year or so before launching a product to document it properly, Waterhouse suggests that product teams identify what can and can’t be easily documented, such as standardized ",{"_30":2879,"_16":270,"_271":2880,"_146":2882},"4f85c0f18a06",[2881],"51d366865ec1","OpenAPI",{"_30":2884,"_16":270,"_271":2885,"_146":2886},"752ba0071a4d",[]," documentation. “There's not a lot of distinction between writing a good test case and writing a good prompt. The things you do to test are the things that you need to document: ",{"_30":2888,"_16":270,"_271":2889,"_146":2890},"bcb099607988",[434],"How do I do this thing?",{"_30":2892,"_16":270,"_271":2893,"_146":2894},"1e3f1caf20a5",[]," And so we go into test cases and extract user stories from those, because we know that those work.”",[2896],{"_30":2881,"_16":42,"_386":2897},"https://www.openapis.org/what-is-openapi",{"_30":2899,"_16":265,"_266":2900,"_274":2914,"_276":277},"632bf8098288",[2901,2905,2910],{"_30":2902,"_16":270,"_271":2903,"_146":2904},"33016087df4c",[],"The author points out that alternatives like ",{"_30":2906,"_16":270,"_271":2907,"_146":2909},"8c7f680b902a",[2908],"e4c22daf3f46","retrieval-augmented generation (RAG)",{"_30":2911,"_16":270,"_271":2912,"_146":2913},"159dd2beb7ae",[]," may be better suited to maintain docs freshness than using a LLM as a catch-all solution, even if it was trained on a large dataset. “Although we get better answers from very large [data] corpuses, I think it is dangerous because for nearly every LLM out there...their training corpus is at least a year old, and we don’t know exactly what it contains.”",[2915],{"_30":2908,"_16":42,"_386":2916},"https://en.wikipedia.org/wiki/Retrieval-augmented_generation",{"_30":2918,"_16":265,"_266":2919,"_274":2939,"_276":277},"2b4ab2c43456",[2920,2924,2928,2932,2935],{"_30":2921,"_16":270,"_271":2922,"_146":2923},"f8712eb98b17",[],"“So if you have made software [changes] in that time, the LLM doesn't know about it and it's not retrieving fresh data. So I think you should have a small, specific, internal [focus to] look at all ",{"_30":2925,"_16":270,"_271":2926,"_146":2927},"eda2ea7b5fab",[434],"this",{"_30":2929,"_16":270,"_271":2930,"_146":2931},"e2f25d68ceae",[]," stuff and use ",{"_30":2933,"_16":270,"_271":2934,"_146":2389},"cd82c4cea259",[434],{"_30":2936,"_16":270,"_271":2937,"_146":2938},"228911e19f6a",[]," as your context window, instead of a giant context window. That makes it both cheaper to run and more relevant.”",[],{"_30":2941,"_16":265,"_266":2942,"_274":2947,"_276":508},"2d5c67f21878",[2943],{"_30":2944,"_16":270,"_271":2945,"_146":2946},"98387814003a",[],"Indexing and Librarians: Better Docs Through Better Organization of Info?",[],{"_30":2949,"_16":265,"_266":2950,"_274":2963,"_276":277},"f26f1409f052",[2951,2955,2959],{"_30":2952,"_16":270,"_271":2953,"_146":2954},"8b000df2081b",[],"The author notes that for documentation, the real use case that most users want isn’t longer, prettier notes to read, but the ability to ask a specific question in natural language and receive a contextually correct, helpful answer. “That's the ‘killer app.’ That requires some indexing. I haven't built this yet myself, because nobody has paid me to do it. But being able to ask any question and get an authentic answer: ",{"_30":2956,"_16":270,"_271":2957,"_146":2958},"0d3825a99a41",[434],"That’s",{"_30":2960,"_16":270,"_271":2961,"_146":2962},"3474bd2e195d",[]," what you want.”",[],{"_30":2965,"_16":265,"_266":2966,"_274":2971,"_276":277},"8755ffb07e63",[2967],{"_30":2968,"_16":270,"_271":2969,"_146":2970},"6f17fbd0861a",[],"To get to such a state requires proper indexing of product information, so that your system can return a specific, accurate response. In other words, a highly deterministic result that’s at odds with GenAI’s inherently non-deterministic nature. “For example, if a person asks about installation, here's the set of documents they want, and you’d refine it with the set of documents you’d return. So you’d need a natural-language sorting on pre-indexed stuff.”",[],{"_30":2973,"_16":265,"_266":2974,"_274":2995,"_276":277},"2e2b0316c6ba",[2975,2979,2983,2987,2991],{"_30":2976,"_16":270,"_271":2977,"_146":2978},"1e272d57fef2",[],"“But that means that you have to pay someone to do the indexing with computer assistance. Indexing a whole set of things is difficult, but you can't just leave it to machines because machines don't understand anything. ",{"_30":2980,"_16":270,"_271":2981,"_146":2982},"1c9b4110a8b4",[434],"They do not ‘understand’",{"_30":2984,"_16":270,"_271":2985,"_146":2986},"c8d9bf9389b5",[]," as they are not capable of ‘understanding’ as a concept. So I think that there's going to be an uptick in the need for ",{"_30":2988,"_16":270,"_271":2989,"_146":2990},"1ffdee21044a",[434],"librarians",{"_30":2992,"_16":270,"_271":2993,"_146":2994},"c62130ca2ab3",[],": People who understand semantic information and semantic relationships.”",[],{"_30":2997,"_16":265,"_266":2998,"_274":3003,"_276":349},"56d9657f96d1",[2999],{"_30":3000,"_16":270,"_271":3001,"_146":3002},"64253605e0e3",[],"The State of Docs vs. What Has Been Lost to Time",[],{"_30":3005,"_16":265,"_266":3006,"_274":3019,"_276":277},"bf4a0690e218",[3007,3011,3015],{"_30":3008,"_16":270,"_271":3009,"_146":3010},"4ff89e358c46",[],"The author points out that the status quo of documentation isn’t reams of static docs that someone wrote years ago and no one has looked at since. “It hasn’t been like that for 15 years! But one of the interesting problems is that we've democratized technical writing and made it all markdown files, which means ",{"_30":3012,"_16":270,"_271":3013,"_146":3014},"9b740be808a6",[434],"we've taken the expertise out of it.",{"_30":3016,"_16":270,"_271":3017,"_146":3018},"09fb4283ef62",[],"”",[],{"_30":3021,"_16":265,"_266":3022,"_274":3044,"_276":277},"7c84fdb213bf",[3023,3027,3032,3035,3040],{"_30":3024,"_16":270,"_271":3025,"_146":3026},"c34a76a71b5b",[],"“I think back to the priesthood of ",{"_30":3028,"_16":270,"_271":3029,"_146":3031},"46dc06bc1392",[3030],"d2fdd7aa4bae","Framemaker",{"_30":3033,"_16":270,"_271":3034,"_146":2204},"79958f4e9183",[],{"_30":3036,"_16":270,"_271":3037,"_146":3039},"f56438e4f2f6",[3038],"38d0df6c95a1","RoboHelp",{"_30":3041,"_16":270,"_271":3042,"_146":3043},"165dc38160c1",[],", and other things we used to have. Extremely specific, context-sensitive help, where every page you were on in an application had an identifier, and if you clicked the little ‘help’ button, you would get help specific to that page. It would know where you were. There's no reason we can't have that with web applications. But nobody has been doing it. And I think just returning to something like that would be immensely useful.”",[3045,3047],{"_30":3030,"_16":42,"_386":3046},"https://en.wikipedia.org/wiki/Adobe_FrameMaker",{"_30":3038,"_16":42,"_386":3048},"https://en.wikipedia.org/wiki/Adobe_RoboHelp",{"_30":3050,"_16":265,"_266":3051,"_274":3077,"_276":277},"4e24d1c7496c",[3052,3056,3061,3065,3069,3073],{"_30":3053,"_16":270,"_271":3054,"_146":3055},"d281d014ae66",[],"“Today, what we're writing is: Here's how to get started with what you're doing. Here's the ",{"_30":3057,"_16":270,"_271":3058,"_146":3060},"9ac699936fa5",[3059,434],"c2e134fcf395","reference documentation",{"_30":3062,"_16":270,"_271":3063,"_146":3064},"552ef4be7312",[],". There's a whole framework about what a reference doc is and what an instruction doc is and what an example doc is. And all that knowledge is difficult to transmit to founders because it's ",{"_30":3066,"_16":270,"_271":3067,"_146":3068},"a46d8f94200d",[434],"expertise",{"_30":3070,"_16":270,"_271":3071,"_146":3072},"1eabbaf70c04",[],". It's like asking someone: ",{"_30":3074,"_16":270,"_271":3075,"_146":3076},"5dbbe3171d80",[434],"‘Please be an expert in user interfaces.’",[3078],{"_30":3059,"_16":42,"_386":3079},"https://learn.microsoft.com/en-us/style-guide/developer-content/reference-documentation",{"_30":3081,"_16":265,"_266":3082,"_274":3087,"_276":349},"62355c95d0d3",[3083],{"_30":3084,"_16":270,"_271":3085,"_146":3086},"c3a2fe19c3d9",[],"A Gentle Reminder: Docs Aren’t for Happy People",[],{"_30":3089,"_16":265,"_266":3090,"_274":3119,"_276":277},"bb2fd930e4e1",[3091,3095,3099,3103,3107,3111,3115],{"_30":3092,"_16":270,"_271":3093,"_146":3094},"aa12de6d8e9b",[],"“I think that what we need to be doing is not just writing ",{"_30":3096,"_16":270,"_271":3097,"_146":3098},"67d4c7b18156",[434],"more",{"_30":3100,"_16":270,"_271":3101,"_146":3102},"1c94f1b57024",[]," documentation, which doesn't actually help people. I think what we need to be doing is ",{"_30":3104,"_16":270,"_271":3105,"_146":3106},"ef637e5c40e2",[434],"making it easier to find answers",{"_30":3108,"_16":270,"_271":3109,"_146":3110},"f9e1a8dde5dd",[]," to a problem that you're having. This is always the goal of documentation. Because I always say that anyone reading documentation ",{"_30":3112,"_16":270,"_271":3113,"_146":3114},"7a717cf0a5a2",[434],"is already pissed off",{"_30":3116,"_16":270,"_271":3117,"_146":3118},"833d1cf90092",[],".”",[],{"_30":3121,"_16":265,"_266":3122,"_274":3143,"_276":277},"3947c9c0be6b",[3123,3127,3131,3135,3139],{"_30":3124,"_16":270,"_271":3125,"_146":3126},"55db9fbe8feb",[],"The author clarifies that people reading documentation ",{"_30":3128,"_16":270,"_271":3129,"_146":3130},"86cebac7bdc7",[434],"don’t want to read documentation",{"_30":3132,"_16":270,"_271":3133,"_146":3134},"cfae719ef91d",[],". They want to already be completing their tasks. “We need to do as much as we can so people ",{"_30":3136,"_16":270,"_271":3137,"_146":3138},"ad6d26a47655",[434],"don't have to read documentation",{"_30":3140,"_16":270,"_271":3141,"_146":3142},"4ce624bf222f",[],". What if machines could read documentation and answer questions. That'd be great! But we need to design better software.”",[],{"_30":3145,"_16":265,"_266":3146,"_274":3151,"_276":277},"6de1e3937293",[3147],{"_30":3148,"_16":270,"_271":3149,"_146":3150},"83269819f243",[],"In an ideal world, with better software, users wouldn’t need to fight a 20-page config guide to get it to work. “I tell my clients: ‘Have you made a Docker container for [your product]? Could you just give people a runtime environment that works for them so that they don't have to configure it?’”",[],{"_30":3153,"_16":265,"_266":3154,"_274":3159,"_276":277},"c397b4f562d5",[3155],{"_30":3156,"_16":270,"_271":3157,"_146":3158},"f9f9494f8182",[],"“I think we're moving in that direction a lot for the things that we used to use documentation for. But in order to do that, you still have to know how things work.”",[],{"_30":3161,"_16":265,"_266":3162,"_274":3167,"_276":2466},"86a216334632",[3163],{"_30":3164,"_16":270,"_271":3165,"_146":3166},"478bb6d780a5",[434],"What we need to be doing is making it easier to find answers to a problem that you're having. Anyone reading documentation is already pissed off.",[],{"_30":3169,"_16":265,"_266":3170,"_274":3175,"_276":349},"71dd80524e93",[3171],{"_30":3172,"_16":270,"_271":3173,"_146":3174},"29e520a25db0",[],"Are Smaller Models the Key to Better Docs?",[],{"_30":3177,"_16":265,"_266":3178,"_274":3183,"_276":277},"a0cd24558dc4",[3179],{"_30":3180,"_16":270,"_271":3181,"_146":3182},"724f410b037c",[],"Waterhouse notes that while GenAI is exhibiting potential for summarization use cases, and might be theoretically useful to gobble up reams of documentation and spit out answers, the solution isn’t quite that simple. “The thing that gobbles up data also has its own biases. It doesn't have a good semantic understanding of exactly what you're doing.”",[],{"_30":3185,"_16":265,"_266":3186,"_274":3191,"_276":277},"e5c06c8acfc6",[3187],{"_30":3188,"_16":270,"_271":3189,"_146":3190},"ace851eb267b",[],"The author suggests that an alternative might be smaller, lighter-weight models and retrieval tools trained on a highly-specific, frequently-updated dataset from which it answers questions. “But it's much clearer to think of it as a chat-based interface for an index rather than thinking of it as an answer machine.”",[],{"_30":3193,"_16":265,"_266":3194,"_274":3199,"_276":349},"f3177be1c521",[3195],{"_30":3196,"_16":270,"_271":3197,"_146":3198},"3063b3f40a09",[],"Weaponizing Skepticism",[],{"_30":3201,"_16":265,"_266":3202,"_274":3215,"_276":277},"7efb924aec0f",[3203,3207,3211],{"_30":3204,"_16":270,"_271":3205,"_146":3206},"66d674d2ed22",[],"The author notes the problem of software documentation seeming to have all the answers...put into the hands of LLMs that still confidently give ",{"_30":3208,"_16":270,"_271":3209,"_146":3210},"5fcf8ff83f2c",[434],"wrong",{"_30":3212,"_16":270,"_271":3213,"_146":3214},"bd300efe9070",[]," answers. “Recently, there was a thread online where someone recommended feeding some Perl string into an LLM to ‘see what it parses as.’ It turns out the string parsed as a UNIX command to wipe your hard drive.”",[],{"_30":3217,"_16":265,"_266":3218,"_274":3239,"_276":277},"570b27432821",[3219,3223,3227,3231,3236],{"_30":3220,"_16":270,"_271":3221,"_146":3222},"f053624b212a",[],"“Now, that was obfuscated. And a human who has been on the Internet for a while knows not to run random unchecked strings. But [AI] doesn’t know that. It doesn't have any sense of danger. And I honestly think that we are going to have to ",{"_30":3224,"_16":270,"_271":3225,"_146":3226},"ff1e6ce171a8",[434],"code in",{"_30":3228,"_16":270,"_271":3229,"_146":3230},"808cee9768d2",[]," a sense of danger. For example, how a prominent startup founder’s vibe-coding tool accidentally ",{"_30":3232,"_16":270,"_271":3233,"_146":3235},"3560c7cdd1a5",[3234],"ff4c8e75f31f","wiped their production database",{"_30":3237,"_16":270,"_271":3238,"_146":3118},"c6e792576ee7",[],[3240],{"_30":3234,"_16":42,"_386":3241},"https://fortune.com/2025/07/23/ai-coding-tool-replit-wiped-database-called-it-a-catastrophic-failure/",{"_30":3243,"_16":265,"_266":3244,"_274":3249,"_276":277},"8ac1778bad56",[3245],{"_30":3246,"_16":270,"_271":3247,"_146":3248},"4c1f15b1b821",[],"“I think what we need to build into our tools is a sense of skepticism. (We can call it ‘testing,’ but testing is just weaponized skepticism.) We need to teach AI tools a sense of danger and a sense of skepticism before we let them loose in production.”",[],{"_30":3251,"_16":265,"_266":3252,"_274":3257,"_276":277},"8c235fc46113",[3253],{"_30":3254,"_16":270,"_271":3255,"_146":3256},"62fefd3c38e2",[],"“Separately, we need to think about how AI will consume what we write in a different way than a human being might. How do we design documentation not just to be consumed by humans, but to be consumed by machine learning [algorithms]?” the author muses. “Honestly, the OpenAPI project seems like the start of that: This is a semantically sound, well-formatted thing that you can consume whether or not you're a machine. It’s predictable and you can count on it to have the same structure so that it's not going to give you weird answers.”",[],{"_30":3259,"_16":265,"_266":3260,"_274":3265,"_276":277},"116e4281a02a",[3261],{"_30":3262,"_16":270,"_271":3263,"_146":3264},"001c4aa79ba8",[],"“Making our documentation accessible to machines is going to be valuable, because then we can use machines to ask [the documentation itself] questions.",[],{"_30":3267,"_16":265,"_266":3268,"_274":3273,"_276":349},"ee73bbdf42a1",[3269],{"_30":3270,"_16":270,"_271":3271,"_146":3272},"4ce560cf1861",[],"Making Builders Future-Proof in an AI Future",[],{"_30":3275,"_16":265,"_266":3276,"_274":3281,"_276":277},"ae1bb7d436bb",[3277],{"_30":3278,"_16":270,"_271":3279,"_146":3280},"3acabfbe8ddf",[],"Waterhouse also considers the impact of AI on engineering teams, including for junior developers. “Every time we make an advance in technology, we feel like we're going to lose those skills, and it turns out that somebody cares about preserving them. Also, even mechanization doesn't reduce the amount of work that needs to happen.”",[],{"_30":3283,"_16":265,"_266":3284,"_274":3298,"_276":277},"1240c8ff52c6",[3285,3289,3294],{"_30":3286,"_16":270,"_271":3287,"_146":3288},"52ef98d47b28",[],"The author cites ",{"_30":3290,"_16":270,"_271":3291,"_146":3293},"b6ec122caf91",[3292,434],"2efc9997f119","More Work for Mother",{"_30":3295,"_16":270,"_271":3296,"_146":3297},"29e68a15f109",[]," by Ruth Schwartz Cowan, a book about how laundry used to be an infrequent, time-consuming chore that required hiring washerwomen servants. After the advent of the washing machine, the position of washerwoman vanished from society, but because washing machines made doing laundry more accessible, families would do laundry more frequently, causing housewives to eventually invest the same number of hours per year doing laundry as they did previously.",[3299],{"_30":3292,"_16":42,"_386":3300},"https://hss.sas.upenn.edu/content/more-work-mother-ironies-household-technology-open-hearth-microwave",{"_30":3302,"_16":265,"_266":3303,"_274":3316,"_276":277},"9ff79b8c9fa2",[3304,3308,3312],{"_30":3305,"_16":270,"_271":3306,"_146":3307},"00cb0d99b24a",[],"“We're going to have junior testers who can run exponentially more tests, but somebody still has to ",{"_30":3309,"_16":270,"_271":3310,"_146":3311},"9749ea1359ca",[434],"write",{"_30":3313,"_16":270,"_271":3314,"_146":3315},"0773b9d1c319",[]," them. Those skills will be sustained, but they're not going to be ‘a whole job.’ The problem is somebody has to understand the fundamentals.”",[],{"_30":3318,"_16":265,"_266":3319,"_274":3324,"_276":277},"27888d340ed0",[3320],{"_30":3321,"_16":270,"_271":3322,"_146":3323},"4de966a87265",[],"“We watched this happen with the abstraction of languages up the stack. There are not a lot of people coding in C anymore because our abstraction layer above C is good enough. But there's a reason people are still using COBOL. It's because everything but COBOL has floating point errors for extreme edge cases of math that actually matter quite a lot. So we still need to have at least a few experts from somewhere who care about the deep internals.”",[],{"_30":3326,"_16":265,"_266":3327,"_274":3340,"_276":277},"2c2818bede39",[3328,3332,3336],{"_30":3329,"_16":270,"_271":3330,"_146":3331},"f218e4e6851b",[],"The author notes that older programming experts are retiring, leaving a vacuum for experts on computing fundamentals. “For a lot of these really sophisticated abstractions that LLMs rely on, we need someone who understands how to program the thing",{"_30":3333,"_16":270,"_271":3334,"_146":3335},"661081550a9f",[434]," that programs the thing",{"_30":3337,"_16":270,"_271":3338,"_146":3339},"3647325ef424",[]," that makes the chips.” The current state of play in AI is marked by inflated expectations and budgets, but hopefully, market forces will iron out the issues.",[],{"_30":3342,"_16":122,"_111":3343},"a25da1bae406",{"_113":3344,"_16":115},"image-76eca1fb8989a4b23ea771a6fe7df42544f23d54-385x489-png",{"_30":3346,"_16":265,"_266":3347,"_274":3360,"_276":277},"63ebf93fefe8",[3348,3352,3357],{"_30":3349,"_16":270,"_271":3350,"_146":3351},"8ba6fcf4d6ef",[434],"Without experts in the fundamentals, software will have increasingly precarious dependencies. Image courtesy ",{"_30":3353,"_16":270,"_271":3354,"_146":3356},"fa5a87958d01",[3355,434],"2e5e13988414","XKCD",{"_30":3358,"_16":270,"_271":3359,"_146":2852},"e12172b35539",[434],[3361],{"_30":3355,"_16":42,"_386":3362},"https://xkcd.com/2347/",{"_30":3364,"_16":265,"_266":3365,"_274":3370,"_276":349},"491f84300d63",[3366],{"_30":3367,"_16":270,"_271":3368,"_146":3369},"b6f78677a708",[],"What Will Never Change: Who and What Docs Are For",[],{"_30":3372,"_16":265,"_266":3373,"_274":3386,"_276":277},"bf299bca401b",[3374,3377,3382],{"_30":3375,"_16":270,"_271":3376,"_146":3288},"e25741bc0804",[],{"_30":3378,"_16":270,"_271":3379,"_146":3381},"5f75260ec026",[3380,434],"6601d8a80d0a","Making Users Awesome",{"_30":3383,"_16":270,"_271":3384,"_146":3385},"e0dcd1b4f63d",[]," by Kathy Sierra’s fundamental point about software: No one uses it because they ‘love using software.’ “People use software because they’re trying to accomplish a task or get something done.”",[3387],{"_30":3380,"_16":42,"_386":3388},"https://www.oreilly.com/library/view/badass-making-users/9781491919057/",{"_30":3390,"_16":265,"_266":3391,"_274":3404,"_276":277},"5e5ab1180ce9",[3392,3396,3400],{"_30":3393,"_16":270,"_271":3394,"_146":3395},"dcc1ffdab42f",[],"“And when we're writing documentation, every piece of documentation that somebody has to read ",{"_30":3397,"_16":270,"_271":3398,"_146":3399},"105ee7f9f43e",[434],"slows them down on the way to getting something done",{"_30":3401,"_16":270,"_271":3402,"_146":3403},"00126c3c43f0",[],". It may speed them up in the long run, they may understand something better, but on the whole we are diverting them from their task.”",[],13,"https://cdn.sanity.io/images/50q6fr1p/production/47f933319f9e2a3ca8db58e8b18ff02be37f700d-1200x630.png","5a4e7241-65b4-4ede-8b19-efe37601f59f","2025-08-28T16:24:00.000Z","software-documentation-in-the-age-of-ai","The Future of Software Documentation in the Age of AI","modules","moreFromLibrary",[3414,3416,3418],{"_30":3415,"_113":298,"_16":115},"sXH8xxQcmsm9",{"_30":3417,"_113":318,"_16":115},"bVFNeqUa2yVv",{"_30":3419,"_113":334,"_16":115},"54krRhgUDXp6","outlines",[3422],{"_30":3423,"_16":265,"_266":3424,"_274":3429,"_276":277},"dfec96be23c0",[3425],{"_30":3426,"_16":270,"_271":3427,"_146":3428},"3a6d2fd725d3",[],"

AI-First Organization Design feat. Raza Habib\n
- Why Building LLM Apps Is Different
- People, Ideas, Machines: In That Order
- The Anatomy of an LLM App
- Case Study: GitHub Copilot & Other Examples
- The Right Team for AI Product Development
- Centralized Versus Bottom-Up AI Strategy
- Evaluation as the Product Spec
- Tooling & Operational Best Practices
- Key Pitfalls & Practical Recommendations
\n

",[],"posterFrame",{"_16":122,"_111":3432},{"_113":3433,"_16":115},"image-1362e768e11cd626bebe2e9f7903e489d225b1df-1280x720-png","seo",{"_16":3434,"_21":273,"_3436":170,"_3437":170,"_3438":273,"_120":3431},"nofollow","noindex","OGdescription","slideDeck",{"_16":3441,"_111":3442,"_3444":3445},"file",{"_113":3443,"_16":115},"file-125582ae0cba82603cf3145c0e6e18e98cfead81-html","fileLink","https://cdn.sanity.io/files/50q6fr1p/production/125582ae0cba82603cf3145c0e6e18e98cfead81.html",{"_16":301,"_3447":3448},"current","ai-first-organization-design-feat-raza-habib","speakers","speakersWithOrg",[3452],{"_30":3453,"_16":3454,"_3455":-5,"_3456":3457},"e0d340b43c8b","speaker","organization","person",{"_11":3458,"_13":3459,"_14":3460,"_127":3461,"_16":3456,"_17":3464,"_3465":170,"_3466":3467,"_3455":3468,"_3476":170,"_3477":3478,"_3481":3482,"_3434":3483,"_3484":170},"2025-09-22T13:33:52Z","9a0e69c8-b0b5-4cbf-8ce5-4cfecb21d525","fzhfcc1AiP5Q7XSAygVlak",{"_129":3462},{"_131":3459,"_132":3463},"Ka78wZKaDLupg97Te2Garp","2025-09-22T18:18:06Z","generate","name","Raza Habib",{"_11":3469,"_13":3470,"_14":3471,"_16":3455,"_17":3472,"_3465":170,"_3466":3473,"_3474":170,"_3434":3475},"2025-09-22T13:37:08Z","52a76ebd-26fd-4b7b-8787-ba4ee28a22f6","ouh5UxJ2ABFaVjvI3C6H7z","2025-09-22T13:37:10Z","Anthropic","portfolioCompany",{"_16":3434,"_3436":170,"_3437":170},"partner","photo",{"_16":110,"_111":3479},{"_113":3480,"_16":115},"image-98e64746637da191821cb0d2210d3e55db578cab-1743x1743-jpg","position","Member of Technical Staff",{"_16":3434,"_3436":170,"_3437":170},"teamMember","startDate","tags",[3488,3497,3505,3513,3521],{"_11":3489,"_13":3490,"_14":3491,"_16":3492,"_17":3489,"_21":3493,"_3466":3494,"_301":3495},"2024-09-05T17:31:35Z","017fa9be-32e2-46ee-bddf-f13275fbdf00","aeN0todGvIjkWyTywFzV6F","tag","Pertaining to the technology underpinning AI and machine learning operations.","AI Infrastructure",{"_16":301,"_3447":3496},"ai-infrastructure",{"_11":3498,"_13":3499,"_14":3500,"_16":3492,"_17":3498,"_21":3501,"_3466":3502,"_301":3503},"2023-07-17T23:27:35Z","5893abdf-ea1d-4456-8114-7493984631a6","NgvOL1OJzLJ5agOL2f1ypV","Large-language models are algorithms that, when trained on datasets, power AI chatbots to return results when prompted.","LLM",{"_16":301,"_3447":3504},"llm",{"_11":3506,"_13":3507,"_14":3508,"_16":3492,"_17":3506,"_21":3509,"_3466":3510,"_301":3511},"2023-08-28T22:38:33Z","ce8ede15-ce64-431d-9cbd-d81338d1be6a","NgvOL1OJzLJ5agOL2f1pgf","Generative AI is a form of machine learning that produces content (such as text, images, video, or audio) in response to being prompted.","Generative AI",{"_16":301,"_3447":3512},"generative-ai",{"_11":3514,"_13":3515,"_14":3516,"_16":3492,"_17":3514,"_21":3517,"_3466":3518,"_301":3519},"2023-07-17T23:23:49Z","eec559a9-10e7-4405-82e4-e0736495ec36","TfMkf5gVJ7UWv1vMi5Qxor","Artificial intelligence, including AI-powered coding assistants built on large language models.","AI",{"_16":301,"_3447":3520},"ai",{"_11":3522,"_13":3523,"_14":3524,"_16":3492,"_17":3522,"_21":445,"_3466":3525,"_301":3526},"2022-09-09T00:15:40Z","kFpYkfzWCiYI8xC60oV4mM","TYN2nJHhdg1RiOg5ta1HWG","Operations",{"_16":301,"_3447":3527},"operations","AI-First Organization Design feat. Raza Habib","transcript",[3531,3574,3624,3665,3695,3725,3755,3816,3836,3896,3946,3986,4006,4027,4068,4098,4105,4113,4120,4141,4181,4231,4271,4321,4341,4381,4412,4442,4507,4538,4559,4589,4640,4701,4731,4738,4746,4753,4773,4814,4875,4916,4956,4996,5047,5078,5118,5158,5188,5208,5228,5269,5289,5319,5326,5334,5341,5352,5403,5424,5445,5486,5537,5558,5588,5648,5669,5709,5740,5770,5800,5821,5872,5913,5920,5928,5935,5966,5996,6037,6067,6107,6147,6167,6188,6238,6268,6298,6329,6379,6389,6420,6440,6470,6490,6530,6561,6568,6576,6583,6614,6664,6715,6756,6797,6847,6867,6898,6918,6949,6980,7031,7051,7082,7089,7097,7104,7135,7155,7186,7216,7256,7277,7317,7338,7368,7388,7409,7440,7471,7555,7562,7570,7577,7588,7609,7700,7710,7751,7772,7802,7823,7844,7878,7909,7949,7979,8009,8040,8071,8121,8151,8158,8166,8173,8204,8235,8256,8296,8327,8358,8388,8418,8449,8480,8547,8577,8617,8657,8697,8704,8712,8719,8760,8801,8822,8842,8863,8893,8933,8973,9003,9024,9075,9115,9178,9218,9258,9279,9309,9350,9380],{"_30":3532,"_16":265,"_266":3533,"_274":3564,"_276":277},"380a8da6c9a8",[3534,3539,3543,3548,3551,3556,3559],{"_30":3535,"_16":270,"_271":3536,"_146":3538},"08fd34986893",[3537],"d65e7cb","All right, Good evening.",{"_30":3540,"_16":270,"_271":3541,"_146":3542},"482ebe06efbc",[]," ",{"_30":3544,"_16":270,"_271":3545,"_146":3547},"1a1e2b8b3141",[3546],"37fb87c","Well, thank you guys so much for having me.",{"_30":3549,"_16":270,"_271":3550,"_146":3542},"1e4cb036d618",[],{"_30":3552,"_16":270,"_271":3553,"_146":3555},"34eb8cf4769e",[3554],"9e22135","My name is Raza. I'm one of the co-founders",{"_30":3557,"_16":270,"_271":3558,"_146":3542},"b1a4026ef7fe",[],{"_30":3560,"_16":270,"_271":3561,"_146":3563},"4e1629dcd9cd",[3562],"82aa4f3","and CEO of Humanloop.",[3565,3569,3571,3572],{"_30":3537,"_16":3566,"_3567":3568},"timestamp","time","00:00:02.560",{"_30":3546,"_16":3566,"_3567":3570},"00:00:06.640",{"_30":3554,"_16":3566,"_3567":3570},{"_30":3562,"_16":3566,"_3567":3573},"00:00:09.600",{"_30":3575,"_16":265,"_266":3576,"_274":3614,"_276":277},"642f6aab4daf",[3577,3582,3585,3590,3593,3598,3601,3606,3609],{"_30":3578,"_16":270,"_271":3579,"_146":3581},"cfdabeee3faf",[3580],"80a8914","I'm going to be sharing",{"_30":3583,"_16":270,"_271":3584,"_146":3542},"8f6f25f1ed11",[],{"_30":3586,"_16":270,"_271":3587,"_146":3589},"65d47ee62ba6",[3588],"3a20594","tonight just some of the lessons we've learned, best practices and",{"_30":3591,"_16":270,"_271":3592,"_146":3542},"335a0bbda14c",[],{"_30":3594,"_16":270,"_271":3595,"_146":3597},"0ba694e6dece",[3596],"19ad7b7","pitfalls, from working with a lot of teams who've been",{"_30":3599,"_16":270,"_271":3600,"_146":3542},"2c78c35c9b7f",[],{"_30":3602,"_16":270,"_271":3603,"_146":3605},"512ced7ce95a",[3604],"b683246","deploying AI applications and productions built with",{"_30":3607,"_16":270,"_271":3608,"_146":3542},"c6399c524ec8",[],{"_30":3610,"_16":270,"_271":3611,"_146":3613},"b4f8e73730be",[3612],"2b6231d","LLMs, or in productions built with LLMs.",[3615,3616,3618,3620,3622],{"_30":3580,"_16":3566,"_3567":3573},{"_30":3588,"_16":3566,"_3567":3617},"00:00:12.440",{"_30":3596,"_16":3566,"_3567":3619},"00:00:15.000",{"_30":3604,"_16":3566,"_3567":3621},"00:00:17.640",{"_30":3612,"_16":3566,"_3567":3623},"00:00:20.440",{"_30":3625,"_16":265,"_266":3626,"_274":3656,"_276":277},"e0c504c12b36",[3627,3632,3635,3640,3643,3648,3651],{"_30":3628,"_16":270,"_271":3629,"_146":3631},"888391731eae",[3630],"d022248","So maybe just a little bit about who I am and",{"_30":3633,"_16":270,"_271":3634,"_146":3542},"67af69b86fda",[],{"_30":3636,"_16":270,"_271":3637,"_146":3639},"302d70058e36",[3638],"a9e83f6","why I have any right to tell you anything about",{"_30":3641,"_16":270,"_271":3642,"_146":3542},"e5af1826970c",[],{"_30":3644,"_16":270,"_271":3645,"_146":3647},"b0ac235d7638",[3646],"ee005a1","this at all, and you can decide how",{"_30":3649,"_16":270,"_271":3650,"_146":3542},"d5577e1e7c5b",[],{"_30":3652,"_16":270,"_271":3653,"_146":3655},"5136b4274fb6",[3654],"9d20260","much to pay attention to that.",[3657,3659,3661,3663],{"_30":3630,"_16":3566,"_3567":3658},"00:00:23.280",{"_30":3638,"_16":3566,"_3567":3660},"00:00:26.000",{"_30":3646,"_16":3566,"_3567":3662},"00:00:28.820",{"_30":3654,"_16":3566,"_3567":3664},"00:00:31.780",{"_30":3666,"_16":265,"_266":3667,"_274":3689,"_276":277},"ab3495a2342e",[3668,3673,3676,3681,3684],{"_30":3669,"_16":270,"_271":3670,"_146":3672},"293bed25a466",[3671],"4abe0b7","So for the last few years I've been the CEO of",{"_30":3674,"_16":270,"_271":3675,"_146":3542},"f814c3218a72",[],{"_30":3677,"_16":270,"_271":3678,"_146":3680},"54d35c468629",[3679],"8ad345c","Humanloop, as I said, and I'll talk about what",{"_30":3682,"_16":270,"_271":3683,"_146":3542},"df07d0187b16",[],{"_30":3685,"_16":270,"_271":3686,"_146":3688},"066e37cb6b24",[3687],"51db298","Humanloop does in a moment.",[3690,3691,3693],{"_30":3671,"_16":3566,"_3567":3664},{"_30":3679,"_16":3566,"_3567":3692},"00:00:34.340",{"_30":3687,"_16":3566,"_3567":3694},"00:00:37.060",{"_30":3696,"_16":265,"_266":3697,"_274":3719,"_276":277},"2af9d2cc2d1d",[3698,3703,3706,3711,3714],{"_30":3699,"_16":270,"_271":3700,"_146":3702},"16c132f3ab11",[3701],"ffd4a95","And then before that I did a PhD",{"_30":3704,"_16":270,"_271":3705,"_146":3542},"6a8b56b9dd0b",[],{"_30":3707,"_16":270,"_271":3708,"_146":3710},"1f65c6087b09",[3709],"00e9076","in deep learning at UCL in the UK and I worked at",{"_30":3712,"_16":270,"_271":3713,"_146":3542},"858916dc7128",[],{"_30":3715,"_16":270,"_271":3716,"_146":3718},"a0aa4ecac848",[3717],"43aa2db","Google for a little bit.",[3720,3721,3723],{"_30":3701,"_16":3566,"_3567":3694},{"_30":3709,"_16":3566,"_3567":3722},"00:00:40.060",{"_30":3717,"_16":3566,"_3567":3724},"00:00:43.059",{"_30":3726,"_16":265,"_266":3727,"_274":3749,"_276":277},"bbc8db95ae16",[3728,3733,3736,3741,3744],{"_30":3729,"_16":270,"_271":3730,"_146":3732},"6350b4137c99",[3731],"ea7f69f","So I've been both a researcher and",{"_30":3734,"_16":270,"_271":3735,"_146":3542},"483613aa1ab6",[],{"_30":3737,"_16":270,"_271":3738,"_146":3740},"f24fc18f81d4",[3739],"db699db","practitioner of machine learning and then also",{"_30":3742,"_16":270,"_271":3743,"_146":3542},"765b5e03fe30",[],{"_30":3745,"_16":270,"_271":3746,"_146":3748},"8c1fa2f00409",[3747],"c3d5f4d","helping people build products.",[3750,3751,3753],{"_30":3731,"_16":3566,"_3567":3724},{"_30":3739,"_16":3566,"_3567":3752},"00:00:45.820",{"_30":3747,"_16":3566,"_3567":3754},"00:00:48.600",{"_30":3756,"_16":265,"_266":3757,"_274":3803,"_276":277},"5c6b9e9d7429",[3758,3763,3766,3771,3774,3779,3782,3787,3790,3795,3798],{"_30":3759,"_16":270,"_271":3760,"_146":3762},"f6330546fc4d",[3761],"0d0d610","We were really early to this space, so we were comfortably the first",{"_30":3764,"_16":270,"_271":3765,"_146":3542},"ed8980038d9d",[],{"_30":3767,"_16":270,"_271":3768,"_146":3770},"bdee9df01558",[3769],"0f88f41","LLM Ops platform and, as a result we",{"_30":3772,"_16":270,"_271":3773,"_146":3542},"b466ebdf08b5",[],{"_30":3775,"_16":270,"_271":3776,"_146":3778},"eab1001c6087",[3777],"cfd1115","had the privilege of one working with some of the earliest adopters of the technology,",{"_30":3780,"_16":270,"_271":3781,"_146":3542},"7328f22cc665",[],{"_30":3783,"_16":270,"_271":3784,"_146":3786},"a9ace18e41c1",[3785],"eebb025","but also sort of seeing the evolution over the last couple of",{"_30":3788,"_16":270,"_271":3789,"_146":3542},"08c9c9f63872",[],{"_30":3791,"_16":270,"_271":3792,"_146":3794},"c37639f8185a",[3793],"f36af31","years and seeing both things that have worked and",{"_30":3796,"_16":270,"_271":3797,"_146":3542},"50ae5658b02a",[],{"_30":3799,"_16":270,"_271":3800,"_146":3802},"78928d30f555",[3801],"2b34f4a","haven't worked.",[3804,3806,3808,3810,3812,3814],{"_30":3761,"_16":3566,"_3567":3805},"00:00:51.540",{"_30":3769,"_16":3566,"_3567":3807},"00:00:54.420",{"_30":3777,"_16":3566,"_3567":3809},"00:00:57.310",{"_30":3785,"_16":3566,"_3567":3811},"00:01:00.670",{"_30":3793,"_16":3566,"_3567":3813},"00:01:03.550",{"_30":3801,"_16":3566,"_3567":3815},"00:01:06.470",{"_30":3817,"_16":265,"_266":3818,"_274":3832,"_276":277},"242fd04f1ef9",[3819,3824,3827],{"_30":3820,"_16":270,"_271":3821,"_146":3823},"9277981c5f35",[3822],"127dd8c","Some of the companies we worked with are on this",{"_30":3825,"_16":270,"_271":3826,"_146":3542},"434f95a4a04a",[],{"_30":3828,"_16":270,"_271":3829,"_146":3831},"84d91ba77a85",[3830],"eee49c8","slide, but I think we can move past quickly.",[3833,3834],{"_30":3822,"_16":3566,"_3567":3815},{"_30":3830,"_16":3566,"_3567":3835},"00:01:09.470",{"_30":3837,"_16":265,"_266":3838,"_274":3884,"_276":2466},"54ac2de85a4c",[3839,3844,3847,3852,3855,3860,3863,3868,3871,3876,3879],{"_30":3840,"_16":270,"_271":3841,"_146":3843},"ab7a449a072b",[3842],"91e2c7e","The first big thing that I",{"_30":3845,"_16":270,"_271":3846,"_146":3542},"25534b321fac",[],{"_30":3848,"_16":270,"_271":3849,"_146":3851},"f3fe984cea9f",[3850],"f19afcf","want to communicate though, and I think this doesn't necessarily",{"_30":3853,"_16":270,"_271":3854,"_146":3542},"14427b6e8baa",[],{"_30":3856,"_16":270,"_271":3857,"_146":3859},"32a81e691457",[3858],"5971e01","need to be said anymore, but",{"_30":3861,"_16":270,"_271":3862,"_146":3542},"94ef3b5d7a60",[],{"_30":3864,"_16":270,"_271":3865,"_146":3867},"e6bec2138020",[3866],"a900d75","we're well into the stage now that generative",{"_30":3869,"_16":270,"_271":3870,"_146":3542},"2c32bc602494",[],{"_30":3872,"_16":270,"_271":3873,"_146":3875},"d17af7ee7baa",[3874],"e5840e5","AI is actually producing real",{"_30":3877,"_16":270,"_271":3878,"_146":3542},"19b41b64e081",[],{"_30":3880,"_16":270,"_271":3881,"_146":3883},"0192240de675",[3882],"585443d","revenue even outside of coding.",[3885,3887,3889,3891,3892,3894],{"_30":3842,"_16":3566,"_3567":3886},"00:01:12.269",{"_30":3850,"_16":3566,"_3567":3888},"00:01:15.110",{"_30":3858,"_16":3566,"_3567":3890},"00:01:18.070",{"_30":3866,"_16":3566,"_3567":3890},{"_30":3874,"_16":3566,"_3567":3893},"00:01:20.830",{"_30":3882,"_16":3566,"_3567":3895},"00:01:23.390",{"_30":3897,"_16":265,"_266":3898,"_274":3936,"_276":277},"fc99cf81c88e",[3899,3904,3907,3912,3915,3920,3923,3928,3931],{"_30":3900,"_16":270,"_271":3901,"_146":3903},"58f9ae4e1981",[3902],"2764ec4","So there's a small",{"_30":3905,"_16":270,"_271":3906,"_146":3542},"31b4cb0f7515",[],{"_30":3908,"_16":270,"_271":3909,"_146":3911},"62eaa47793da",[3910],"ca9e234","number of these coding startups that have done extremely",{"_30":3913,"_16":270,"_271":3914,"_146":3542},"3b5c1f15868f",[],{"_30":3916,"_16":270,"_271":3917,"_146":3919},"2c00f970c027",[3918],"321b7a7","well and I think there was some skepticism for a long",{"_30":3921,"_16":270,"_271":3922,"_146":3542},"6bb61b1fdd9f",[],{"_30":3924,"_16":270,"_271":3925,"_146":3927},"b021b7b76184",[3926],"7b879fc","time as to whether or not people",{"_30":3929,"_16":270,"_271":3930,"_146":3542},"1f0224ad3efd",[],{"_30":3932,"_16":270,"_271":3933,"_146":3935},"5c2261ec63ba",[3934],"2f33d4d","were actually making money with this.",[3937,3938,3940,3942,3944],{"_30":3902,"_16":3566,"_3567":3895},{"_30":3910,"_16":3566,"_3567":3939},"00:01:26.360",{"_30":3918,"_16":3566,"_3567":3941},"00:01:29.200",{"_30":3926,"_16":3566,"_3567":3943},"00:01:32.040",{"_30":3934,"_16":3566,"_3567":3945},"00:01:34.880",{"_30":3947,"_16":265,"_266":3948,"_274":3978,"_276":277},"8be9fab96bb6",[3949,3954,3957,3962,3965,3970,3973],{"_30":3950,"_16":270,"_271":3951,"_146":3953},"0a2b8e6d5dcb",[3952],"15da022","And so we definitely",{"_30":3955,"_16":270,"_271":3956,"_146":3542},"4c1954bb5da2",[],{"_30":3958,"_16":270,"_271":3959,"_146":3961},"c9376da295b5",[3960],"89b4405","see that within our customers, this is true. And so I've got a quote",{"_30":3963,"_16":270,"_271":3964,"_146":3542},"257bf06a8537",[],{"_30":3966,"_16":270,"_271":3967,"_146":3969},"cd7003f391c7",[3968],"aefdef6","from one of our customers, FileVine, who are in the legal tech",{"_30":3971,"_16":270,"_271":3972,"_146":3542},"907525680a24",[],{"_30":3974,"_16":270,"_271":3975,"_146":3977},"78607f0a4b91",[3976],"611e8a4","space and interesting.",[3979,3980,3982,3984],{"_30":3952,"_16":3566,"_3567":3945},{"_30":3960,"_16":3566,"_3567":3981},"00:01:37.840",{"_30":3968,"_16":3566,"_3567":3983},"00:01:40.800",{"_30":3976,"_16":3566,"_3567":3985},"00:01:43.760",{"_30":3987,"_16":265,"_266":3988,"_274":4002,"_276":277},"b73006a56c2b",[3989,3994,3997],{"_30":3990,"_16":270,"_271":3991,"_146":3993},"8d6b33ebcf5a",[3992],"1abf5d3","They were a fairly mature company when",{"_30":3995,"_16":270,"_271":3996,"_146":3542},"6253cdb9887d",[],{"_30":3998,"_16":270,"_271":3999,"_146":4001},"227e1b88c4aa",[4000],"73e18ab","they started shipping AI features and products.",[4003,4004],{"_30":3992,"_16":3566,"_3567":3985},{"_30":4000,"_16":3566,"_3567":4005},"00:01:46.760",{"_30":4007,"_16":265,"_266":4008,"_274":4022,"_276":277},"6c7c0ee0edb9",[4009,4014,4017],{"_30":4010,"_16":270,"_271":4011,"_146":4013},"7e2240056cd9",[4012],"5bb6d4c","And since they started, their AI revenue now",{"_30":4015,"_16":270,"_271":4016,"_146":3542},"945232b9da23",[],{"_30":4018,"_16":270,"_271":4019,"_146":4021},"35ec0215020d",[4020],"730eb1e","outstrips their traditional SaaS revenue.",[4023,4025],{"_30":4012,"_16":3566,"_3567":4024},"00:01:49.680",{"_30":4020,"_16":3566,"_3567":4026},"00:01:52.110",{"_30":4028,"_16":265,"_266":4029,"_274":4059,"_276":277},"0a14c37296cb",[4030,4035,4038,4043,4046,4051,4054],{"_30":4031,"_16":270,"_271":4032,"_146":4034},"007c8460faf1",[4033],"752f644","Even though they were a Series C-ish or D-ish",{"_30":4036,"_16":270,"_271":4037,"_146":3542},"733340314a02",[],{"_30":4039,"_16":270,"_271":4040,"_146":4042},"c1bffe3e0864",[4041],"de7c241","company at this time that they started doing that, and they're",{"_30":4044,"_16":270,"_271":4045,"_146":3542},"8012837383e1",[],{"_30":4047,"_16":270,"_271":4048,"_146":4050},"99e7b52636a6",[4049],"f2a66de","not at all exceptional",{"_30":4052,"_16":270,"_271":4053,"_146":3542},"d6ffe8b6ee76",[],{"_30":4055,"_16":270,"_271":4056,"_146":4058},"a6cf71754f7c",[4057],"d65c54a","amongst our customers.",[4060,4062,4064,4066],{"_30":4033,"_16":3566,"_3567":4061},"00:01:55.070",{"_30":4041,"_16":3566,"_3567":4063},"00:01:58.070",{"_30":4049,"_16":3566,"_3567":4065},"00:02:01.030",{"_30":4057,"_16":3566,"_3567":4067},"00:02:03.790",{"_30":4069,"_16":265,"_266":4070,"_274":4092,"_276":277},"b446b45bf1b4",[4071,4076,4079,4084,4087],{"_30":4072,"_16":270,"_271":4073,"_146":4075},"487490a1cfcb",[4074],"32bd059","So we are well past the stage of people just",{"_30":4077,"_16":270,"_271":4078,"_146":3542},"1a67427de5f3",[],{"_30":4080,"_16":270,"_271":4081,"_146":4083},"2a56cbcabd35",[4082],"d2024ab","tinkering and experimenting and people are actually",{"_30":4085,"_16":270,"_271":4086,"_146":3542},"3acd7e58665f",[],{"_30":4088,"_16":270,"_271":4089,"_146":4091},"35d4820e63e9",[4090],"af93dbe","really making some money with this now.",[4093,4094,4096],{"_30":4074,"_16":3566,"_3567":4067},{"_30":4082,"_16":3566,"_3567":4095},"00:02:06.710",{"_30":4090,"_16":3566,"_3567":4097},"00:02:09.630",{"_30":4099,"_16":265,"_266":4100,"_274":4104,"_276":277},"6214c854d7a5",[4101],{"_30":4102,"_16":270,"_271":4103,"_146":445},"6a494384345a",[],[],{"_30":4106,"_16":265,"_266":4107,"_274":4112,"_276":349},"bc216e264758",[4108],{"_30":4109,"_16":270,"_271":4110,"_146":4111},"f8d33f0c53a1",[],"Why Building LLM Apps Is Different",[],{"_30":4114,"_16":265,"_266":4115,"_274":4119,"_276":277},"3e3d827051b9",[4116],{"_30":4117,"_16":270,"_271":4118,"_146":445},"dd44ad7cc9f2",[],[],{"_30":4121,"_16":265,"_266":4122,"_274":4136,"_276":277},"a5c9ab09d9a0",[4123,4128,4131],{"_30":4124,"_16":270,"_271":4125,"_146":4127},"a1e64e8217e5",[4126],"1b601d2","So I want to talk through best practices, tips for building teams and",{"_30":4129,"_16":270,"_271":4130,"_146":3542},"3b53b4b11b5b",[],{"_30":4132,"_16":270,"_271":4133,"_146":4135},"27e713970f6f",[4134],"1623329","things like that.",[4137,4139],{"_30":4126,"_16":3566,"_3567":4138},"00:02:12.630",{"_30":4134,"_16":3566,"_3567":4140},"00:02:15.590",{"_30":4142,"_16":265,"_266":4143,"_274":4173,"_276":277},"0440c03d6c22",[4144,4149,4152,4157,4160,4165,4168],{"_30":4145,"_16":270,"_271":4146,"_146":4148},"52d003fb9ea2",[4147],"410cd23","But before I do that I just want to get ourselves grounded",{"_30":4150,"_16":270,"_271":4151,"_146":3542},"732974956acb",[],{"_30":4153,"_16":270,"_271":4154,"_146":4156},"80b5c5173850",[4155],"30add1b","in what I mean by an LLM app, maybe talk about one",{"_30":4158,"_16":270,"_271":4159,"_146":3542},"c606c86ad5a4",[],{"_30":4161,"_16":270,"_271":4162,"_146":4164},"b8216c937297",[4163],"cc1c3c8","example, get ourselves on the same page and then",{"_30":4166,"_16":270,"_271":4167,"_146":3542},"e343074a3bb3",[],{"_30":4169,"_16":270,"_271":4170,"_146":4172},"87f46d4c82b3",[4171],"b7296ae","jump into that.",[4174,4175,4177,4179],{"_30":4147,"_16":3566,"_3567":4140},{"_30":4155,"_16":3566,"_3567":4176},"00:02:18.550",{"_30":4163,"_16":3566,"_3567":4178},"00:02:21.260",{"_30":4171,"_16":3566,"_3567":4180},"00:02:24.140",{"_30":4182,"_16":265,"_266":4183,"_274":4221,"_276":277},"bd97e02472c9",[4184,4189,4192,4197,4200,4205,4208,4213,4216],{"_30":4185,"_16":270,"_271":4186,"_146":4188},"1de416d83eee",[4187],"452ac50","And so when we think of",{"_30":4190,"_16":270,"_271":4191,"_146":3542},"459b5f423d0b",[],{"_30":4193,"_16":270,"_271":4194,"_146":4196},"b2b4a07e76de",[4195],"f64e631","LLM applications, I sort of think of them as composed",{"_30":4198,"_16":270,"_271":4199,"_146":3542},"3ec33f3c702a",[],{"_30":4201,"_16":270,"_271":4202,"_146":4204},"ec14647e7b9b",[4203],"25a5d91","of the core AI block and then",{"_30":4206,"_16":270,"_271":4207,"_146":3542},"2114f361666f",[],{"_30":4209,"_16":270,"_271":4210,"_146":4212},"ed2d01a5a73b",[4211],"4ac53ce","traditional software logic around it that we're all",{"_30":4214,"_16":270,"_271":4215,"_146":3542},"afe780bc6fb8",[],{"_30":4217,"_16":270,"_271":4218,"_146":4220},"6a588aeb3795",[4219],"f79a39e","kind of familiar with.",[4222,4223,4225,4227,4229],{"_30":4187,"_16":3566,"_3567":4180},{"_30":4195,"_16":3566,"_3567":4224},"00:02:26.740",{"_30":4203,"_16":3566,"_3567":4226},"00:02:29.620",{"_30":4211,"_16":3566,"_3567":4228},"00:02:32.180",{"_30":4219,"_16":3566,"_3567":4230},"00:02:35.060",{"_30":4232,"_16":265,"_266":4233,"_274":4263,"_276":277},"416177adc326",[4234,4239,4242,4247,4250,4255,4258],{"_30":4235,"_16":270,"_271":4236,"_146":4238},"7f37c0f9dc0c",[4237],"0d87cd7","And the core AI block has a base",{"_30":4240,"_16":270,"_271":4241,"_146":3542},"0e87df7a6f32",[],{"_30":4243,"_16":270,"_271":4244,"_146":4246},"654be6a4ac25",[4245],"24074ca","model, so that's OpenAI or",{"_30":4248,"_16":270,"_271":4249,"_146":3542},"e21018fc21e9",[],{"_30":4251,"_16":270,"_271":4252,"_146":4254},"99f6678e0217",[4253],"476803d","Anthropic, some sort of open source,",{"_30":4256,"_16":270,"_271":4257,"_146":3542},"ad5d17af37e2",[],{"_30":4259,"_16":270,"_271":4260,"_146":4262},"853a578827cd",[4261],"23efc1f","fine tuned model.",[4264,4265,4267,4269],{"_30":4237,"_16":3566,"_3567":4230},{"_30":4245,"_16":3566,"_3567":4266},"00:02:37.900",{"_30":4253,"_16":3566,"_3567":4268},"00:02:40.900",{"_30":4261,"_16":3566,"_3567":4270},"00:02:43.700",{"_30":4272,"_16":265,"_266":4273,"_274":4311,"_276":277},"6d0483f39eaf",[4274,4279,4282,4287,4290,4295,4298,4303,4306],{"_30":4275,"_16":270,"_271":4276,"_146":4278},"2683c3dfd8f4",[4277],"f4f10f8","It then has some set of natural",{"_30":4280,"_16":270,"_271":4281,"_146":3542},"b24731479c79",[],{"_30":4283,"_16":270,"_271":4284,"_146":4286},"356f3289f98c",[4285],"47b23b0","language instructions that are usually some fixed template",{"_30":4288,"_16":270,"_271":4289,"_146":3542},"615ba9251e11",[],{"_30":4291,"_16":270,"_271":4292,"_146":4294},"d61278b07f9e",[4293],"aef9c0f","that, that has places that are going to be populated in production",{"_30":4296,"_16":270,"_271":4297,"_146":3542},"a4bb48c3816e",[],{"_30":4299,"_16":270,"_271":4300,"_146":4302},"685767b6fe08",[4301],"702363d","or at inference time—so a templated",{"_30":4304,"_16":270,"_271":4305,"_146":3542},"cca7e49eb039",[],{"_30":4307,"_16":270,"_271":4308,"_146":4310},"a77e3530f7d6",[4309],"14be710","structure",[4312,4313,4315,4317,4319],{"_30":4277,"_16":3566,"_3567":4270},{"_30":4285,"_16":3566,"_3567":4314},"00:02:46.540",{"_30":4293,"_16":3566,"_3567":4316},"00:02:49.540",{"_30":4301,"_16":3566,"_3567":4318},"00:02:52.450",{"_30":4309,"_16":3566,"_3567":4320},"00:02:55.010",{"_30":4322,"_16":265,"_266":4323,"_274":4337,"_276":277},"1e4c7e7e538b",[4324,4329,4332],{"_30":4325,"_16":270,"_271":4326,"_146":4328},"b34ea946d3fe",[4327],"fbe0630","And that templated structure has a huge impact on the",{"_30":4330,"_16":270,"_271":4331,"_146":3542},"339c3095f689",[],{"_30":4333,"_16":270,"_271":4334,"_146":4336},"a10a71c76398",[4335],"cab7d3c","performance of the system.",[4338,4339],{"_30":4327,"_16":3566,"_3567":4320},{"_30":4335,"_16":3566,"_3567":4340},"00:02:57.930",{"_30":4342,"_16":265,"_266":4343,"_274":4373,"_276":277},"96c8e4dd1a84",[4344,4349,4352,4357,4360,4365,4368],{"_30":4345,"_16":270,"_271":4346,"_146":4348},"64d1eddb4489",[4347],"bcbe74b","And then some data selection",{"_30":4350,"_16":270,"_271":4351,"_146":3542},"85ae9be3eac0",[],{"_30":4353,"_16":270,"_271":4354,"_146":4356},"e636f09bbc25",[4355],"940df5e","strategy for how you're going to populate that. And ultimately then",{"_30":4358,"_16":270,"_271":4359,"_146":3542},"166296289022",[],{"_30":4361,"_16":270,"_271":4362,"_146":4364},"992ace722a13",[4363],"525e517","also, the models are augmented with the ability to",{"_30":4366,"_16":270,"_271":4367,"_146":3542},"894865a2eafd",[],{"_30":4369,"_16":270,"_271":4370,"_146":4372},"b0fe87fb8495",[4371],"bd7261b","call APIs or even other LLM blocks.",[4374,4375,4377,4379],{"_30":4347,"_16":3566,"_3567":4340},{"_30":4355,"_16":3566,"_3567":4376},"00:03:00.730",{"_30":4363,"_16":3566,"_3567":4378},"00:03:03.730",{"_30":4371,"_16":3566,"_3567":4380},"00:03:06.730",{"_30":4382,"_16":265,"_266":4383,"_274":4405,"_276":277},"0e6a111a82bf",[4384,4389,4392,4397,4400],{"_30":4385,"_16":270,"_271":4386,"_146":4388},"25af0f23dacf",[4387],"5a2179b","So tools, and the way it works is just",{"_30":4390,"_16":270,"_271":4391,"_146":3542},"19b9743b0808",[],{"_30":4393,"_16":270,"_271":4394,"_146":4396},"1c6b19aa51a8",[4395],"cbcc698","at some DSL, so the LLM spits out a string, you parse",{"_30":4398,"_16":270,"_271":4399,"_146":3542},"fe21a2c4c678",[],{"_30":4401,"_16":270,"_271":4402,"_146":4404},"74bbb9873670",[4403],"19bb986","the string, you call the appropriate function, you return the result.",[4406,4408,4410],{"_30":4387,"_16":3566,"_3567":4407},"00:03:09.650",{"_30":4395,"_16":3566,"_3567":4409},"00:03:12.610",{"_30":4403,"_16":3566,"_3567":4411},"00:03:15.490",{"_30":4413,"_16":265,"_266":4414,"_274":4436,"_276":277},"07581583340b",[4415,4420,4423,4428,4431],{"_30":4416,"_16":270,"_271":4417,"_146":4419},"c77dc6e010be",[4418],"4b88ba2","And then",{"_30":4421,"_16":270,"_271":4422,"_146":3542},"57019c3b39a8",[],{"_30":4424,"_16":270,"_271":4425,"_146":4427},"85cfa0f51ff3",[4426],"d5f3ab2","these blocks are composed, they can be in a",{"_30":4429,"_16":270,"_271":4430,"_146":3542},"8475783e339a",[],{"_30":4432,"_16":270,"_271":4433,"_146":4435},"3cc6cb786e8b",[4434],"9c47b8d","loop and then you call it an agent. They can be changed together.",[4437,4438,4440],{"_30":4418,"_16":3566,"_3567":4411},{"_30":4426,"_16":3566,"_3567":4439},"00:03:18.450",{"_30":4434,"_16":3566,"_3567":4441},"00:03:21.360",{"_30":4443,"_16":265,"_266":4444,"_274":4494,"_276":277},"d4b806ab817e",[4445,4449,4454,4457,4462,4465,4470,4473,4478,4481,4486,4489],{"_30":4446,"_16":270,"_271":4447,"_146":4448},"293299c57051",[],"A",{"_30":4450,"_16":270,"_271":4451,"_146":4453},"e300541a1aca",[4452],"9ae6e34","nd increasingly actually it really is",{"_30":4455,"_16":270,"_271":4456,"_146":3542},"bd33175be828",[],{"_30":4458,"_16":270,"_271":4459,"_146":4461},"0bdafdc98cd5",[4460],"ac86814","not that much more complicated than this block because as the models",{"_30":4463,"_16":270,"_271":4464,"_146":3542},"a725dc7f52c4",[],{"_30":4466,"_16":270,"_271":4467,"_146":4469},"c9a486b6307d",[4468],"f306d7f","get smarter and smarter, a lot of the scaffolding that we",{"_30":4471,"_16":270,"_271":4472,"_146":3542},"5b4d77517122",[],{"_30":4474,"_16":270,"_271":4475,"_146":4477},"3d399dd48a43",[4476],"47c2c59","have put around it, less and less of",{"_30":4479,"_16":270,"_271":4480,"_146":3542},"0a18fc27a849",[],{"_30":4482,"_16":270,"_271":4483,"_146":4485},"106bd6cf4d25",[4484],"749eeb2","it is necessary or it's more just to orchestrate the agents",{"_30":4487,"_16":270,"_271":4488,"_146":3542},"efa58aed8304",[],{"_30":4490,"_16":270,"_271":4491,"_146":4493},"3e60908f6d3b",[4492],"1079b01","themselves.",[4495,4497,4499,4501,4503,4505],{"_30":4452,"_16":3566,"_3567":4496},"00:03:24.660",{"_30":4460,"_16":3566,"_3567":4498},"00:03:27.600",{"_30":4468,"_16":3566,"_3567":4500},"00:03:30.520",{"_30":4476,"_16":3566,"_3567":4502},"00:03:33.680",{"_30":4484,"_16":3566,"_3567":4504},"00:03:36.640",{"_30":4492,"_16":3566,"_3567":4506},"00:03:39.640",{"_30":4508,"_16":265,"_266":4509,"_274":4531,"_276":277},"28a7523626b4",[4510,4515,4518,4523,4526],{"_30":4511,"_16":270,"_271":4512,"_146":4514},"97338a70ecb1",[4513],"c72afd2","And so, a concrete example of this, the OG LLM app, the",{"_30":4516,"_16":270,"_271":4517,"_146":3542},"8331ec19110a",[],{"_30":4519,"_16":270,"_271":4520,"_146":4522},"0c3fbeb7836c",[4521],"23a79d6","first one that I think sort of had real",{"_30":4524,"_16":270,"_271":4525,"_146":3542},"b10ca7e8f031",[],{"_30":4527,"_16":270,"_271":4528,"_146":4530},"c342884849ef",[4529],"09cfe92","revenue and real traction was GitHub Copilot.",[4532,4534,4536],{"_30":4513,"_16":3566,"_3567":4533},"00:03:42.280",{"_30":4521,"_16":3566,"_3567":4535},"00:03:45.240",{"_30":4529,"_16":3566,"_3567":4537},"00:03:48.230",{"_30":4539,"_16":265,"_266":4540,"_274":4554,"_276":277},"f13527ca185b",[4541,4546,4549],{"_30":4542,"_16":270,"_271":4543,"_146":4545},"a951242d1092",[4544],"d73cc00","And so GitHub Copilot perfectly fits this template.",{"_30":4547,"_16":270,"_271":4548,"_146":3542},"a4acad485a4d",[],{"_30":4550,"_16":270,"_271":4551,"_146":4553},"11babb1141e3",[4552],"74f963f","There's a small fine tuned model.",[4555,4557],{"_30":4544,"_16":3566,"_3567":4556},"00:03:51.230",{"_30":4552,"_16":3566,"_3567":4558},"00:03:53.950",{"_30":4560,"_16":265,"_266":4561,"_274":4583,"_276":277},"4d2dee459437",[4562,4567,4570,4575,4578],{"_30":4563,"_16":270,"_271":4564,"_146":4566},"24d11f04af79",[4565],"f2deeda","I don't know if this",{"_30":4568,"_16":270,"_271":4569,"_146":3542},"0fb1b8562236",[],{"_30":4571,"_16":270,"_271":4572,"_146":4574},"9bf0d1d3ce6b",[4573],"e068c42","is still true in terms of the size of the model, but it started off with a",{"_30":4576,"_16":270,"_271":4577,"_146":3542},"5282694fc501",[],{"_30":4579,"_16":270,"_271":4580,"_146":4582},"4e06c5261f88",[4581],"0075c00","12 billion parameter model.",[4584,4585,4587],{"_30":4565,"_16":3566,"_3567":4558},{"_30":4573,"_16":3566,"_3567":4586},"00:03:56.830",{"_30":4581,"_16":3566,"_3567":4588},"00:03:59.710",{"_30":4590,"_16":265,"_266":4591,"_274":4629,"_276":277},"3207040e9b44",[4592,4597,4600,4605,4608,4613,4616,4621,4624],{"_30":4593,"_16":270,"_271":4594,"_146":4596},"fc25648ea7a8",[4595],"b297cde","The way it would work is that they would take the text from behind your cursor,",{"_30":4598,"_16":270,"_271":4599,"_146":3542},"211c9ef94276",[],{"_30":4601,"_16":270,"_271":4602,"_146":4604},"12b331033ded",[4603],"1b507d8","they would look at the files you'd recently touched, look",{"_30":4606,"_16":270,"_271":4607,"_146":3542},"a8920b2b87c3",[],{"_30":4609,"_16":270,"_271":4610,"_146":4612},"8d613a045ac0",[4611],"9a313a3","at similarity between that, had some other heuristics",{"_30":4614,"_16":270,"_271":4615,"_146":3542},"10beae731483",[],{"_30":4617,"_16":270,"_271":4618,"_146":4620},"3c0c1e46f937",[4619],"233434c","for how they would populate the context, and then they would make a",{"_30":4622,"_16":270,"_271":4623,"_146":3542},"09504faad70b",[],{"_30":4625,"_16":270,"_271":4626,"_146":4628},"8295fce5cdc8",[4627],"ba5a34d","prediction as to what would come next.",[4630,4632,4634,4636,4638],{"_30":4595,"_16":3566,"_3567":4631},"00:04:02.630",{"_30":4603,"_16":3566,"_3567":4633},"00:04:05.590",{"_30":4611,"_16":3566,"_3567":4635},"00:04:08.350",{"_30":4619,"_16":3566,"_3567":4637},"00:04:11.310",{"_30":4627,"_16":3566,"_3567":4639},"00:04:14.150",{"_30":4641,"_16":265,"_266":4642,"_274":4688,"_276":277},"8af68332465a",[4643,4648,4651,4656,4659,4664,4667,4672,4675,4680,4683],{"_30":4644,"_16":270,"_271":4645,"_146":4647},"df22c7b3b8e4",[4646],"9c6e7a1","And in addition to having that structure that we just",{"_30":4649,"_16":270,"_271":4650,"_146":3542},"c12531035fc7",[],{"_30":4652,"_16":270,"_271":4653,"_146":4655},"9fc43fbe0233",[4654],"96360ac","described, the other thing that I think they did that",{"_30":4657,"_16":270,"_271":4658,"_146":3542},"0dc062dd1c44",[],{"_30":4660,"_16":270,"_271":4661,"_146":4663},"f2b73afce1a4",[4662],"59785e7","set the foundation for what everyone who",{"_30":4665,"_16":270,"_271":4666,"_146":3542},"74371c278b2c",[],{"_30":4668,"_16":270,"_271":4669,"_146":4671},"ae5aa809639f",[4670],"9a7beab","succeeded has done since then, is they made really good",{"_30":4673,"_16":270,"_271":4674,"_146":3542},"787686223b49",[],{"_30":4676,"_16":270,"_271":4677,"_146":4679},"5373d7404819",[4678],"421de29","evaluation a foundation to making this application",{"_30":4681,"_16":270,"_271":4682,"_146":3542},"1cfe9a4bcbf7",[],{"_30":4684,"_16":270,"_271":4685,"_146":4687},"82418eebbec9",[4686],"a3927a7","work well.",[4689,4691,4693,4695,4697,4699],{"_30":4646,"_16":3566,"_3567":4690},"00:04:17.040",{"_30":4654,"_16":3566,"_3567":4692},"00:04:19.840",{"_30":4662,"_16":3566,"_3567":4694},"00:04:22.640",{"_30":4670,"_16":3566,"_3567":4696},"00:04:25.160",{"_30":4678,"_16":3566,"_3567":4698},"00:04:28.080",{"_30":4686,"_16":3566,"_3567":4700},"00:04:31.000",{"_30":4702,"_16":265,"_266":4703,"_274":4725,"_276":277},"c6b4bf889536",[4704,4709,4712,4717,4720],{"_30":4705,"_16":270,"_271":4706,"_146":4708},"8de0122c8fc9",[4707],"e2b2727","And I'll talk about that more in a little bit. And",{"_30":4710,"_16":270,"_271":4711,"_146":3542},"da207359dbf4",[],{"_30":4713,"_16":270,"_271":4714,"_146":4716},"59b70b0e5071",[4715],"7df580d","then they kind of regularly fine tune on the data that they're getting",{"_30":4718,"_16":270,"_271":4719,"_146":3542},"fee9dfa57068",[],{"_30":4721,"_16":270,"_271":4722,"_146":4724},"31d4dbee03e2",[4723],"a2f1f07","from this evaluation.",[4726,4727,4729],{"_30":4707,"_16":3566,"_3567":4700},{"_30":4715,"_16":3566,"_3567":4728},"00:04:34.000",{"_30":4723,"_16":3566,"_3567":4730},"00:04:36.760",{"_30":4732,"_16":265,"_266":4733,"_274":4737,"_276":277},"6a0824cfff14",[4734],{"_30":4735,"_16":270,"_271":4736,"_146":445},"60ce60256a6a",[],[],{"_30":4739,"_16":265,"_266":4740,"_274":4745,"_276":349},"0885cae229e2",[4741],{"_30":4742,"_16":270,"_271":4743,"_146":4744},"2efbe4a5ae13",[],"People, Ideas, Machines: In That Order",[],{"_30":4747,"_16":265,"_266":4748,"_274":4752,"_276":277},"6274443fc561",[4749],{"_30":4750,"_16":270,"_271":4751,"_146":445},"0f1263d817fe",[],[],{"_30":4754,"_16":265,"_266":4755,"_274":4769,"_276":277},"89e5b4cd7542",[4756,4761,4764],{"_30":4757,"_16":270,"_271":4758,"_146":4760},"2f74e2dc774b",[4759],"895270d","But hopefully that gives us some",{"_30":4762,"_16":270,"_271":4763,"_146":3542},"e2402a299c7f",[],{"_30":4765,"_16":270,"_271":4766,"_146":4768},"d1a56878d384",[4767],"805e8fb","sense of like, okay, what is the anatomy of the thing we're talking about?",[4770,4771],{"_30":4759,"_16":3566,"_3567":4730},{"_30":4767,"_16":3566,"_3567":4772},"00:04:39.720",{"_30":4774,"_16":265,"_266":4775,"_274":4805,"_276":277},"ec8438570a0a",[4776,4781,4784,4789,4792,4797,4800],{"_30":4777,"_16":270,"_271":4778,"_146":4780},"8e4bb92b9a20",[4779],"d068f4a","And this rough structure—One of the things that's amazing",{"_30":4782,"_16":270,"_271":4783,"_146":3542},"5895b7b2fac5",[],{"_30":4785,"_16":270,"_271":4786,"_146":4788},"5b3fe91dfe7f",[4787],"6fedbc2","about these AI applications and LLMs is",{"_30":4790,"_16":270,"_271":4791,"_146":3542},"dd21db2ced2b",[],{"_30":4793,"_16":270,"_271":4794,"_146":4796},"569435d3f4af",[4795],"41555c8","that the rough structure and the components are the same",{"_30":4798,"_16":270,"_271":4799,"_146":3542},"9393ccc1fea1",[],{"_30":4801,"_16":270,"_271":4802,"_146":4804},"5a10851d23aa",[4803],"ff82f18","across many different applications and verticals.",[4806,4808,4810,4812],{"_30":4779,"_16":3566,"_3567":4807},"00:04:43.030",{"_30":4787,"_16":3566,"_3567":4809},"00:04:45.910",{"_30":4795,"_16":3566,"_3567":4811},"00:04:48.590",{"_30":4803,"_16":3566,"_3567":4813},"00:04:51.430",{"_30":4815,"_16":265,"_266":4816,"_274":4862,"_276":2466},"cf3d3a7daf1c",[4817,4822,4825,4830,4833,4838,4841,4846,4849,4854,4857],{"_30":4818,"_16":270,"_271":4819,"_146":4821},"30af0d495e5f",[4820],"6d6d881","This can",{"_30":4823,"_16":270,"_271":4824,"_146":3542},"e7151f8694fc",[],{"_30":4826,"_16":270,"_271":4827,"_146":4829},"e7355ec2048a",[4828],"019fe4f","be the legal coding assistant, it can be the",{"_30":4831,"_16":270,"_271":4832,"_146":3542},"c3aed0efd292",[],{"_30":4834,"_16":270,"_271":4835,"_146":4837},"504176144390",[4836],"04f307f","meeting summarizer, it can be your coding assistant. And",{"_30":4839,"_16":270,"_271":4840,"_146":3542},"6fd98d51a3fe",[],{"_30":4842,"_16":270,"_271":4843,"_146":4845},"bcb73199a149",[4844],"2eed888","really what's changing is the data selection strategy, the tools you're",{"_30":4847,"_16":270,"_271":4848,"_146":3542},"b46de435296a",[],{"_30":4850,"_16":270,"_271":4851,"_146":4853},"3f7b38a9be10",[4852],"ce4fe4a","exposing to the model and the prompts. But",{"_30":4855,"_16":270,"_271":4856,"_146":3542},"85dfbd8adf74",[],{"_30":4858,"_16":270,"_271":4859,"_146":4861},"e9a56e815d10",[4860],"e024d6a","the code is actually remarkably similar in lots of cases.",[4863,4865,4867,4869,4871,4873],{"_30":4820,"_16":3566,"_3567":4864},"00:04:52.430",{"_30":4828,"_16":3566,"_3567":4866},"00:04:54.270",{"_30":4836,"_16":3566,"_3567":4868},"00:04:57.150",{"_30":4844,"_16":3566,"_3567":4870},"00:05:00.150",{"_30":4852,"_16":3566,"_3567":4872},"00:05:03.030",{"_30":4860,"_16":3566,"_3567":4874},"00:05:05.870",{"_30":4876,"_16":265,"_266":4877,"_274":4907,"_276":277},"cd5b393f175e",[4878,4883,4886,4891,4894,4899,4902],{"_30":4879,"_16":270,"_271":4880,"_146":4882},"74a7a54faecd",[4881],"64da158","Okay, so what have we seen that really works",{"_30":4884,"_16":270,"_271":4885,"_146":3542},"9ae578618dcb",[],{"_30":4887,"_16":270,"_271":4888,"_146":4890},"83d6b38d1ccd",[4889],"4dc65cc","well for the teams that have succeeded and",{"_30":4892,"_16":270,"_271":4893,"_146":3542},"09e346230897",[],{"_30":4895,"_16":270,"_271":4896,"_146":4898},"520141bd05ea",[4897],"1370dd9","gone on to produce revenue generating successful loved",{"_30":4900,"_16":270,"_271":4901,"_146":3542},"40bcf54b542b",[],{"_30":4903,"_16":270,"_271":4904,"_146":4906},"efe973b55a04",[4905],"6f19a01","products?",[4908,4910,4912,4914],{"_30":4881,"_16":3566,"_3567":4909},"00:05:09.430",{"_30":4889,"_16":3566,"_3567":4911},"00:05:12.350",{"_30":4897,"_16":3566,"_3567":4913},"00:05:15.430",{"_30":4905,"_16":3566,"_3567":4915},"00:05:18.280",{"_30":4917,"_16":265,"_266":4918,"_274":4948,"_276":277},"39e4148e3084",[4919,4924,4927,4932,4935,4940,4943],{"_30":4920,"_16":270,"_271":4921,"_146":4923},"36a99d610715",[4922],"ccd3d73","So I really like this quote",{"_30":4925,"_16":270,"_271":4926,"_146":3542},"4e75073077c2",[],{"_30":4928,"_16":270,"_271":4929,"_146":4931},"36e83c8a3171",[4930],"e109540","from Colonel Boyd of OODA Loop fame. Apparently he used to shout this",{"_30":4933,"_16":270,"_271":4934,"_146":3542},"acd862b88334",[],{"_30":4936,"_16":270,"_271":4937,"_146":4939},"c1f004231832",[4938],"f64ea5a","at the Pentagon, which was, \"People, ideas, machines,",{"_30":4941,"_16":270,"_271":4942,"_146":3542},"364670075c54",[],{"_30":4944,"_16":270,"_271":4945,"_146":4947},"c598935fe838",[4946],"9caf532","in that order.\"",[4949,4950,4952,4954],{"_30":4922,"_16":3566,"_3567":4915},{"_30":4930,"_16":3566,"_3567":4951},"00:05:21.280",{"_30":4938,"_16":3566,"_3567":4953},"00:05:24.280",{"_30":4946,"_16":3566,"_3567":4955},"00:05:27.240",{"_30":4957,"_16":265,"_266":4958,"_274":4988,"_276":277},"0d01891db021",[4959,4964,4967,4972,4975,4980,4983],{"_30":4960,"_16":270,"_271":4961,"_146":4963},"20211127f6f1",[4962],"068c72a","And I roughly think if you're trying",{"_30":4965,"_16":270,"_271":4966,"_146":3542},"c642859bd1d7",[],{"_30":4968,"_16":270,"_271":4969,"_146":4971},"c3fcbd1b0387",[4970],"7430284","to build a new AI product and staff it right then I also",{"_30":4973,"_16":270,"_271":4974,"_146":3542},"16fafd860897",[],{"_30":4976,"_16":270,"_271":4977,"_146":4979},"80fad115ecbf",[4978],"e84b96e","think this is roughly the right order in which to think about things",{"_30":4981,"_16":270,"_271":4982,"_146":3542},"beca1e8dde47",[],{"_30":4984,"_16":270,"_271":4985,"_146":4987},"8a181ac045af",[4986],"8ed30ea","like what is the makeup of the team that I need?",[4989,4990,4992,4994],{"_30":4962,"_16":3566,"_3567":4955},{"_30":4970,"_16":3566,"_3567":4991},"00:05:30.120",{"_30":4978,"_16":3566,"_3567":4993},"00:05:33.120",{"_30":4986,"_16":3566,"_3567":4995},"00:05:35.960",{"_30":4997,"_16":265,"_266":4998,"_274":5036,"_276":277},"70ffedcdf29b",[4999,5004,5007,5012,5015,5020,5023,5028,5031],{"_30":5000,"_16":270,"_271":5001,"_146":5003},"e407aa90c189",[5002],"6b30e01","And I",{"_30":5005,"_16":270,"_271":5006,"_146":3542},"ded693069247",[],{"_30":5008,"_16":270,"_271":5009,"_146":5011},"92c6d928d44a",[5010],"acf60d7","think that's really different actually",{"_30":5013,"_16":270,"_271":5014,"_146":3542},"90dae7a156f0",[],{"_30":5016,"_16":270,"_271":5017,"_146":5019},"6eeef87bcb5b",[5018],"ebb5512","to traditional AI products. So like the machine learning products that came before",{"_30":5021,"_16":270,"_271":5022,"_146":3542},"d834234cc872",[],{"_30":5024,"_16":270,"_271":5025,"_146":5027},"579f15b3a284",[5026],"d9d91f7","and then sort of the ideas here I think is actually like getting",{"_30":5029,"_16":270,"_271":5030,"_146":3542},"4c02f63bc347",[],{"_30":5032,"_16":270,"_271":5033,"_146":5035},"c8e092a54017",[5034],"657e1bf","your evaluation criteria really clear.",[5037,5039,5041,5043,5045],{"_30":5002,"_16":3566,"_3567":5038},"00:05:36.960",{"_30":5010,"_16":3566,"_3567":5040},"00:05:38.920",{"_30":5018,"_16":3566,"_3567":5042},"00:05:42.410",{"_30":5026,"_16":3566,"_3567":5044},"00:05:46.060",{"_30":5034,"_16":3566,"_3567":5046},"00:05:48.970",{"_30":5048,"_16":265,"_266":5049,"_274":5071,"_276":277},"12110dc2f026",[5050,5055,5058,5063,5066],{"_30":5051,"_16":270,"_271":5052,"_146":5054},"35a523997723",[5053],"f61d4ec","So the evals for an LLM application kind",{"_30":5056,"_16":270,"_271":5057,"_146":3542},"c5c87ffdc6dc",[],{"_30":5059,"_16":270,"_271":5060,"_146":5062},"ff0b05ac6b05",[5061],"59ffda6","of are the spec and AI development is very",{"_30":5064,"_16":270,"_271":5065,"_146":3542},"434aa87d1eaf",[],{"_30":5067,"_16":270,"_271":5068,"_146":5070},"d180736a2692",[5069],"a0993fb","highly iterative.",[5072,5074,5076],{"_30":5053,"_16":3566,"_3567":5073},"00:05:52.020",{"_30":5061,"_16":3566,"_3567":5075},"00:05:54.930",{"_30":5069,"_16":3566,"_3567":5077},"00:05:57.810",{"_30":5079,"_16":265,"_266":5080,"_274":5110,"_276":277},"50ebcc75aca8",[5081,5086,5089,5094,5097,5102,5105],{"_30":5082,"_16":270,"_271":5083,"_146":5085},"e2cc8c357be0",[5084],"3621bb1","So it's not that you can sit down",{"_30":5087,"_16":270,"_271":5088,"_146":3542},"828ab436949d",[],{"_30":5090,"_16":270,"_271":5091,"_146":5093},"4da658150d3a",[5092],"2d0c19c","and say up front exactly what your app is",{"_30":5095,"_16":270,"_271":5096,"_146":3542},"1d239dcc177b",[],{"_30":5098,"_16":270,"_271":5099,"_146":5101},"572019486d61",[5100],"6c3d934","going to look like. As I said, the",{"_30":5103,"_16":270,"_271":5104,"_146":3542},"6628c8752ecf",[],{"_30":5106,"_16":270,"_271":5107,"_146":5109},"5c5abcacba7e",[5108],"e39e983","components are largely the same.",[5111,5112,5114,5116],{"_30":5084,"_16":3566,"_3567":5077},{"_30":5092,"_16":3566,"_3567":5113},"00:06:00.690",{"_30":5100,"_16":3566,"_3567":5115},"00:06:03.650",{"_30":5108,"_16":3566,"_3567":5117},"00:06:06.210",{"_30":5119,"_16":265,"_266":5120,"_274":5150,"_276":277},"02d43c7ead11",[5121,5126,5129,5134,5137,5142,5145],{"_30":5122,"_16":270,"_271":5123,"_146":5125},"210e19a2c618",[5124],"029ea9f","What you typically",{"_30":5127,"_16":270,"_271":5128,"_146":3542},"71d8dd0f26d7",[],{"_30":5130,"_16":270,"_271":5131,"_146":5133},"9bd8013211f3",[5132],"f82ba76","do is you build an end to end app as quickly as you can, you sort of get",{"_30":5135,"_16":270,"_271":5136,"_146":3542},"c49389362112",[],{"_30":5138,"_16":270,"_271":5139,"_146":5141},"3543ba6ac2d5",[5140],"cd0c8d6","some signs of life and then you iterate on all of the",{"_30":5143,"_16":270,"_271":5144,"_146":3542},"fa9a834b86b2",[],{"_30":5146,"_16":270,"_271":5147,"_146":5149},"69e0a7e91866",[5148],"b5b0cbf","components to make it good enough.",[5151,5152,5154,5156],{"_30":5124,"_16":3566,"_3567":5117},{"_30":5132,"_16":3566,"_3567":5153},"00:06:09.210",{"_30":5140,"_16":3566,"_3567":5155},"00:06:12.210",{"_30":5148,"_16":3566,"_3567":5157},"00:06:14.830",{"_30":5159,"_16":265,"_266":5160,"_274":5182,"_276":277},"fdc0d06970c1",[5161,5166,5169,5174,5177],{"_30":5162,"_16":270,"_271":5163,"_146":5165},"bbf3cc6d642a",[5164],"b6f291d","And so if you're iterating very",{"_30":5167,"_16":270,"_271":5168,"_146":3542},"c0a97fa85eb2",[],{"_30":5170,"_16":270,"_271":5171,"_146":5173},"f23588a3fb39",[5172],"7404128","quickly, you need to have some measure of how good the thing",{"_30":5175,"_16":270,"_271":5176,"_146":3542},"1226b5ff9466",[],{"_30":5178,"_16":270,"_271":5179,"_146":5181},"46ce1ba9b91e",[5180],"1d5f98e","that I'm working with is.",[5183,5184,5186],{"_30":5164,"_16":3566,"_3567":5157},{"_30":5172,"_16":3566,"_3567":5185},"00:06:17.750",{"_30":5180,"_16":3566,"_3567":5187},"00:06:20.670",{"_30":5189,"_16":265,"_266":5190,"_274":5204,"_276":277},"f6a170934997",[5191,5196,5199],{"_30":5192,"_16":270,"_271":5193,"_146":5195},"c4b841ac0f98",[5194],"79fc46e","Otherwise you're going to be making lots of",{"_30":5197,"_16":270,"_271":5198,"_146":3542},"ceccc4d60467",[],{"_30":5200,"_16":270,"_271":5201,"_146":5203},"b01434c81f73",[5202],"02eb8e1","changes and you don't know whether you're actually improving things.",[5205,5206],{"_30":5194,"_16":3566,"_3567":5187},{"_30":5202,"_16":3566,"_3567":5207},"00:06:23.670",{"_30":5209,"_16":265,"_266":5210,"_274":5224,"_276":277},"d0d44486cca0",[5211,5216,5219],{"_30":5212,"_16":270,"_271":5213,"_146":5215},"accbe606c9c5",[5214],"93dfa3c","So we'll go into a",{"_30":5217,"_16":270,"_271":5218,"_146":3542},"081315cb14b5",[],{"_30":5220,"_16":270,"_271":5221,"_146":5223},"6d02c23e8156",[5222],"c41cc4b","lot more detail on that. But I think evaluation is the second most important thing.",[5225,5226],{"_30":5214,"_16":3566,"_3567":5207},{"_30":5222,"_16":3566,"_3567":5227},"00:06:26.630",{"_30":5229,"_16":265,"_266":5230,"_274":5260,"_276":277},"2945b05f3439",[5231,5236,5239,5244,5247,5252,5255],{"_30":5232,"_16":270,"_271":5233,"_146":5235},"0e40b330af99",[5234],"003a58b","And then finally if you've got those things right, then the",{"_30":5237,"_16":270,"_271":5238,"_146":3542},"3737feb76bba",[],{"_30":5240,"_16":270,"_271":5241,"_146":5243},"41225302ce21",[5242],"68da6f6","tooling you have around it and the way you build it and obviously I run an",{"_30":5245,"_16":270,"_271":5246,"_146":3542},"e2c8240ed593",[],{"_30":5248,"_16":270,"_271":5249,"_146":5251},"6b41e404c520",[5250],"618d312","AI tooling company, so I would say that, but I do think",{"_30":5253,"_16":270,"_271":5254,"_146":3542},"28befab20080",[],{"_30":5256,"_16":270,"_271":5257,"_146":5259},"56821639c47f",[5258],"34b7698","it's true.",[5261,5263,5265,5267],{"_30":5234,"_16":3566,"_3567":5262},"00:06:29.350",{"_30":5242,"_16":3566,"_3567":5264},"00:06:32.150",{"_30":5250,"_16":3566,"_3567":5266},"00:06:35.070",{"_30":5258,"_16":3566,"_3567":5268},"00:06:38.030",{"_30":5270,"_16":265,"_266":5271,"_274":5285,"_276":277},"47b34ec20fea",[5272,5277,5280],{"_30":5273,"_16":270,"_271":5274,"_146":5276},"7a07cef46659",[5275],"6f7e868","But I think it's third after getting",{"_30":5278,"_16":270,"_271":5279,"_146":3542},"85d35b9a3f87",[],{"_30":5281,"_16":270,"_271":5282,"_146":5284},"c307d40eb28f",[5283],"d0374e5","these first two things right.",[5286,5287],{"_30":5275,"_16":3566,"_3567":5268},{"_30":5283,"_16":3566,"_3567":5288},"00:06:40.970",{"_30":5290,"_16":265,"_266":5291,"_274":5313,"_276":277},"99c33d91a2f6",[5292,5297,5300,5305,5308],{"_30":5293,"_16":270,"_271":5294,"_146":5296},"f558aeee129c",[5295],"4327b25","So maybe we can talk about the team",{"_30":5298,"_16":270,"_271":5299,"_146":3542},"dd02a5177e4f",[],{"_30":5301,"_16":270,"_271":5302,"_146":5304},"f4409c296a76",[5303],"803f813","a little bit and the makeups that we've seen succeed",{"_30":5306,"_16":270,"_271":5307,"_146":3542},"911275c1b241",[],{"_30":5309,"_16":270,"_271":5310,"_146":5312},"aab745742c0e",[5311],"c22864c","in some of the organizations.",[5314,5315,5317],{"_30":5295,"_16":3566,"_3567":5288},{"_30":5303,"_16":3566,"_3567":5316},"00:06:43.890",{"_30":5311,"_16":3566,"_3567":5318},"00:06:47.090",{"_30":5320,"_16":265,"_266":5321,"_274":5325,"_276":277},"a7152375eda1",[5322],{"_30":5323,"_16":270,"_271":5324,"_146":445},"56e5892e2f0e",[],[],{"_30":5327,"_16":265,"_266":5328,"_274":5333,"_276":349},"384eb972239b",[5329],{"_30":5330,"_16":270,"_271":5331,"_146":5332},"6f298696967b",[],"The Anatomy of an LLM App",[],{"_30":5335,"_16":265,"_266":5336,"_274":5340,"_276":277},"401e5865b36c",[5337],{"_30":5338,"_16":270,"_271":5339,"_146":445},"57eef76a4122",[],[],{"_30":5342,"_16":265,"_266":5343,"_274":5349,"_276":277},"6bbd246485ad",[5344],{"_30":5345,"_16":270,"_271":5346,"_146":5348},"c615fe7ce1ab",[5347],"c932f29","So the first thing I would say is that you don't need a lot of specialist AI expertise.",[5350],{"_30":5347,"_16":3566,"_3567":5351},"00:06:49.970",{"_30":5353,"_16":265,"_266":5354,"_274":5392,"_276":277},"351114a57380",[5355,5360,5363,5368,5371,5376,5379,5384,5387],{"_30":5356,"_16":270,"_271":5357,"_146":5359},"fef84a944c6a",[5358],"81f3401","So if you were building an AI",{"_30":5361,"_16":270,"_271":5362,"_146":3542},"a72b3366eeb8",[],{"_30":5364,"_16":270,"_271":5365,"_146":5367},"48394799a71e",[5366],"352299e","product four years ago, three years ago even,",{"_30":5369,"_16":270,"_271":5370,"_146":3542},"4e41979cb5e0",[],{"_30":5372,"_16":270,"_271":5373,"_146":5375},"65c1b791d336",[5374],"377eb39","then you probably would want a lot of people with backgrounds like me",{"_30":5377,"_16":270,"_271":5378,"_146":3542},"8cabf91f2a26",[],{"_30":5380,"_16":270,"_271":5381,"_146":5383},"05b491cc8264",[5382],"ef217e6","who are either data scientists who had studied machine learning, had done",{"_30":5385,"_16":270,"_271":5386,"_146":3542},"7ffe9f342b3d",[],{"_30":5388,"_16":270,"_271":5389,"_146":5391},"06d13200b5cd",[5390],"6f98a4c","research, understood how to do that.",[5393,5395,5397,5399,5401],{"_30":5358,"_16":3566,"_3567":5394},"00:06:52.810",{"_30":5366,"_16":3566,"_3567":5396},"00:06:55.810",{"_30":5374,"_16":3566,"_3567":5398},"00:06:58.770",{"_30":5382,"_16":3566,"_3567":5400},"00:07:01.650",{"_30":5390,"_16":3566,"_3567":5402},"00:07:04.570",{"_30":5404,"_16":265,"_266":5405,"_274":5419,"_276":277},"b9633059f9c3",[5406,5411,5414],{"_30":5407,"_16":270,"_271":5408,"_146":5410},"4b28e2a6298d",[5409],"a59ea10","You really need a lot less of that now because,",{"_30":5412,"_16":270,"_271":5413,"_146":3542},"87f09a13889d",[],{"_30":5415,"_16":270,"_271":5416,"_146":5418},"2f0eb994908f",[5417],"ea49798","in some senses, there's people like either side of the API line, right?",[5420,5422],{"_30":5409,"_16":3566,"_3567":5421},"00:07:07.360",{"_30":5417,"_16":3566,"_3567":5423},"00:07:10.320",{"_30":5425,"_16":265,"_266":5426,"_274":5440,"_276":277},"025534234f05",[5427,5432,5435],{"_30":5428,"_16":270,"_271":5429,"_146":5431},"f51b4a74fbd0",[5430],"cf6a7db","A lot of these applications are being built on top of third party",{"_30":5433,"_16":270,"_271":5434,"_146":3542},"c661cebe63ce",[],{"_30":5436,"_16":270,"_271":5437,"_146":5439},"96f28e1e100b",[5438],"df06df8","models that are served via API.",[5441,5443],{"_30":5430,"_16":3566,"_3567":5442},"00:07:13.240",{"_30":5438,"_16":3566,"_3567":5444},"00:07:16.200",{"_30":5446,"_16":265,"_266":5447,"_274":5477,"_276":277},"b1ba73f5ab25",[5448,5453,5456,5461,5464,5469,5472],{"_30":5449,"_16":270,"_271":5450,"_146":5452},"33545b324285",[5451],"11eac82","And the way that you're customizing them isn't by fine tuning or training the",{"_30":5454,"_16":270,"_271":5455,"_146":3542},"53fea06c7e52",[],{"_30":5457,"_16":270,"_271":5458,"_146":5460},"e9f216e61891",[5459],"d7f31ed","models that much, it's by prompting or orchestrating",{"_30":5462,"_16":270,"_271":5463,"_146":3542},"1376ff060d11",[],{"_30":5465,"_16":270,"_271":5466,"_146":5468},"93cb9f933ee5",[5467],"762e63b","agents, in which case actually having deep",{"_30":5470,"_16":270,"_271":5471,"_146":3542},"40db85d28593",[],{"_30":5473,"_16":270,"_271":5474,"_146":5476},"f621b0b7b35f",[5475],"3e5647d","machine learning expertise isn't that useful.",[5478,5480,5482,5484],{"_30":5451,"_16":3566,"_3567":5479},"00:07:19.120",{"_30":5459,"_16":3566,"_3567":5481},"00:07:22.120",{"_30":5467,"_16":3566,"_3567":5483},"00:07:24.840",{"_30":5475,"_16":3566,"_3567":5485},"00:07:27.600",{"_30":5487,"_16":265,"_266":5488,"_274":5526,"_276":2466},"434645c7876a",[5489,5494,5497,5502,5505,5510,5513,5518,5521],{"_30":5490,"_16":270,"_271":5491,"_146":5493},"d6c7ee456ae8",[5492],"b5f286f","The type of person that we've seen be particularly",{"_30":5495,"_16":270,"_271":5496,"_146":3542},"690cdbaa2b07",[],{"_30":5498,"_16":270,"_271":5499,"_146":5501},"a09432612de1",[5500],"758098c","successful, and the core person we see in these",{"_30":5503,"_16":270,"_271":5504,"_146":3542},"6948db20997b",[],{"_30":5506,"_16":270,"_271":5507,"_146":5509},"506ef1c2cf42",[5508],"ec19e00","teams, is typically someone who's closer to a full",{"_30":5511,"_16":270,"_271":5512,"_146":3542},"180da74f2196",[],{"_30":5514,"_16":270,"_271":5515,"_146":5517},"0994ed0515fc",[5516],"ebba23c","stack engineer and is very product-minded and also",{"_30":5519,"_16":270,"_271":5520,"_146":3542},"6d78140f1485",[],{"_30":5522,"_16":270,"_271":5523,"_146":5525},"d7879c26abac",[5524],"e472857","is using the technology themselves a lot.",[5527,5529,5531,5533,5535],{"_30":5492,"_16":3566,"_3567":5528},"00:07:30.480",{"_30":5500,"_16":3566,"_3567":5530},"00:07:33.320",{"_30":5508,"_16":3566,"_3567":5532},"00:07:36.280",{"_30":5516,"_16":3566,"_3567":5534},"00:07:39.160",{"_30":5524,"_16":3566,"_3567":5536},"00:07:42.040",{"_30":5538,"_16":265,"_266":5539,"_274":5553,"_276":277},"00d8dadca008",[5540,5545,5548],{"_30":5541,"_16":270,"_271":5542,"_146":5544},"0442878a4b29",[5543],"3c643b6","So one of the challenges with building with LLMs today is that the",{"_30":5546,"_16":270,"_271":5547,"_146":3542},"c0df51011f35",[],{"_30":5549,"_16":270,"_271":5550,"_146":5552},"d546cf8bcaab",[5551],"68422c3","frontier of capabilities is very spiky.",[5554,5556],{"_30":5543,"_16":3566,"_3567":5555},"00:07:45.000",{"_30":5551,"_16":3566,"_3567":5557},"00:07:47.680",{"_30":5559,"_16":265,"_266":5560,"_274":5582,"_276":277},"0e3c44290a3c",[5561,5566,5569,5574,5577],{"_30":5562,"_16":270,"_271":5563,"_146":5565},"9b81378c5163",[5564],"50b4e7f","There",{"_30":5567,"_16":270,"_271":5568,"_146":3542},"551526bad682",[],{"_30":5570,"_16":270,"_271":5571,"_146":5573},"e90e90fca9f7",[5572],"6af05ce","was this lovely quote from Nat Friedman where he described it as sort of",{"_30":5575,"_16":270,"_271":5576,"_146":3542},"ff3061f00c1c",[],{"_30":5578,"_16":270,"_271":5579,"_146":5581},"06ab722cc1dc",[5580],"b8ee897","alternating between spooky and kooky.",[5583,5584,5586],{"_30":5564,"_16":3566,"_3567":5557},{"_30":5572,"_16":3566,"_3567":5585},"00:07:50.599",{"_30":5580,"_16":3566,"_3567":5587},"00:07:53.520",{"_30":5589,"_16":265,"_266":5590,"_274":5636,"_276":2466},"ebeb31bea35f",[5591,5596,5599,5604,5607,5612,5615,5620,5623,5628,5631],{"_30":5592,"_16":270,"_271":5593,"_146":5595},"9db14010c946",[5594],"78ea93e","Sometimes the",{"_30":5597,"_16":270,"_271":5598,"_146":3542},"7628dca08600",[],{"_30":5600,"_16":270,"_271":5601,"_146":5603},"64fcf39c15c9",[5602],"12723b3","models will do things that you're surprised that they're capable",{"_30":5605,"_16":270,"_271":5606,"_146":3542},"85395b7cfd84",[],{"_30":5608,"_16":270,"_271":5609,"_146":5611},"cf48d25efc65",[5610],"af991c9","of and then just once in a while they'll fail in a way that is",{"_30":5613,"_16":270,"_271":5614,"_146":3542},"f0915a8807c7",[],{"_30":5616,"_16":270,"_271":5617,"_146":5619},"1799fc073c31",[5618],"66e3ac1","also surprising. Having an intuition for",{"_30":5621,"_16":270,"_271":5622,"_146":3542},"9133f6bc76a5",[],{"_30":5624,"_16":270,"_271":5625,"_146":5627},"b6f84b24e693",[5626],"3664699","where the current frontier is, because it's very fast moving,",{"_30":5629,"_16":270,"_271":5630,"_146":3542},"28d1d1834e86",[],{"_30":5632,"_16":270,"_271":5633,"_146":5635},"91c0da7e08a2",[5634],"6fedde7","is hard to do unless you're just actually playing with these things a lot.",[5637,5638,5640,5642,5644,5646],{"_30":5594,"_16":3566,"_3567":5587},{"_30":5602,"_16":3566,"_3567":5639},"00:07:56.480",{"_30":5610,"_16":3566,"_3567":5641},"00:07:59.360",{"_30":5618,"_16":3566,"_3567":5643},"00:08:02.350",{"_30":5626,"_16":3566,"_3567":5645},"00:08:05.190",{"_30":5634,"_16":3566,"_3567":5647},"00:08:08.500",{"_30":5649,"_16":265,"_266":5650,"_274":5664,"_276":277},"835033c34939",[5651,5656,5659],{"_30":5652,"_16":270,"_271":5653,"_146":5655},"0af0ee56623c",[5654],"1ac5c19","I think Shawn Wang, also known as Swyx,",{"_30":5657,"_16":270,"_271":5658,"_146":3542},"b315f714b74a",[],{"_30":5660,"_16":270,"_271":5661,"_146":5663},"9d55575951e5",[5662],"bba6d4c","is trying to coin the \"AI engineer\" term.",[5665,5667],{"_30":5654,"_16":3566,"_3567":5666},"00:08:14.350",{"_30":5662,"_16":3566,"_3567":5668},"00:08:17.350",{"_30":5670,"_16":265,"_266":5671,"_274":5701,"_276":277},"6a98e5bf8a78",[5672,5677,5680,5685,5688,5693,5696],{"_30":5673,"_16":270,"_271":5674,"_146":5676},"5f2ce89ddfea",[5675],"2d6ab85","I'm not sure whether it's sort",{"_30":5678,"_16":270,"_271":5679,"_146":3542},"1dc741ec37f7",[],{"_30":5681,"_16":270,"_271":5682,"_146":5684},"3bff7f2837a7",[5683],"03d497c","of clearly defined yet, but a generalist full stack",{"_30":5686,"_16":270,"_271":5687,"_146":3542},"3ebd11b2bdee",[],{"_30":5689,"_16":270,"_271":5690,"_146":5692},"96c7e159eb43",[5691],"c7b6da2","engineer is often better than a machine learning engineer",{"_30":5694,"_16":270,"_271":5695,"_146":3542},"e772df070f4c",[],{"_30":5697,"_16":270,"_271":5698,"_146":5700},"2efd0ac66c7f",[5699],"c7b2937","and probably makes up the majority of your team or a product engineer.",[5702,5703,5705,5707],{"_30":5675,"_16":3566,"_3567":5668},{"_30":5683,"_16":3566,"_3567":5704},"00:08:20.350",{"_30":5691,"_16":3566,"_3567":5706},"00:08:22.990",{"_30":5699,"_16":3566,"_3567":5708},"00:08:25.990",{"_30":5710,"_16":265,"_266":5711,"_274":5733,"_276":277},"eed245265698",[5712,5717,5720,5725,5728],{"_30":5713,"_16":270,"_271":5714,"_146":5716},"26555bc28c47",[5715],"e87ed71","And then the people who I think are most overlooked but are",{"_30":5718,"_16":270,"_271":5719,"_146":3542},"89af130d7b82",[],{"_30":5721,"_16":270,"_271":5722,"_146":5724},"b7033263f7a1",[5723],"d62bbca","actually genuinely very critical to success are",{"_30":5726,"_16":270,"_271":5727,"_146":3542},"32aa6c447264",[],{"_30":5729,"_16":270,"_271":5730,"_146":5732},"fb21417f8c56",[5731],"be487aa","the domain experts.",[5734,5736,5738],{"_30":5715,"_16":3566,"_3567":5735},"00:08:29.580",{"_30":5723,"_16":3566,"_3567":5737},"00:08:32.580",{"_30":5731,"_16":3566,"_3567":5739},"00:08:35.580",{"_30":5741,"_16":265,"_266":5742,"_274":5764,"_276":277},"73b0f8bebd86",[5743,5748,5751,5756,5759],{"_30":5744,"_16":270,"_271":5745,"_146":5747},"d38a95c92edc",[5746],"311e898","And in a moment I'll go through a few company",{"_30":5749,"_16":270,"_271":5750,"_146":3542},"ec79dde4c671",[],{"_30":5752,"_16":270,"_271":5753,"_146":5755},"4b33ea1c07be",[5754],"114ef35","examples and talk about what the role of the",{"_30":5757,"_16":270,"_271":5758,"_146":3542},"9e04c5016272",[],{"_30":5760,"_16":270,"_271":5761,"_146":5763},"cfb6f03d010f",[5762],"f08404b","domain expert was in each case.",[5765,5766,5768],{"_30":5746,"_16":3566,"_3567":5739},{"_30":5754,"_16":3566,"_3567":5767},"00:08:38.260",{"_30":5762,"_16":3566,"_3567":5769},"00:08:41.180",{"_30":5771,"_16":265,"_266":5772,"_274":5794,"_276":277},"c8c2521170d6",[5773,5778,5781,5786,5789],{"_30":5774,"_16":270,"_271":5775,"_146":5777},"430679f88b65",[5776],"acd8cef","But essentially,",{"_30":5779,"_16":270,"_271":5780,"_146":3542},"ca339488daf2",[],{"_30":5782,"_16":270,"_271":5783,"_146":5785},"d7783d1073ce",[5784],"dbbded8","the people who are actually writing the prompts and giving the",{"_30":5787,"_16":270,"_271":5788,"_146":3542},"e259df9b8712",[],{"_30":5790,"_16":270,"_271":5791,"_146":5793},"5b4b65541c4a",[5792],"558fee8","instructions to the model that end up in your code base have a very big impact.",[5795,5796,5798],{"_30":5776,"_16":3566,"_3567":5769},{"_30":5784,"_16":3566,"_3567":5797},"00:08:43.799",{"_30":5792,"_16":3566,"_3567":5799},"00:08:46.500",{"_30":5801,"_16":265,"_266":5802,"_274":5816,"_276":277},"d9d06f735c77",[5803,5808,5811],{"_30":5804,"_16":270,"_271":5805,"_146":5807},"b1deba1eba8c",[5806],"57de682","And they're also a big part of setting the",{"_30":5809,"_16":270,"_271":5810,"_146":3542},"a51df1e43774",[],{"_30":5812,"_16":270,"_271":5813,"_146":5815},"bd0cab78f180",[5814],"3c13990","evaluation criteria. I think that that's often overlooked.",[5817,5819],{"_30":5806,"_16":3566,"_3567":5818},"00:08:49.260",{"_30":5814,"_16":3566,"_3567":5820},"00:08:52.020",{"_30":5822,"_16":265,"_266":5823,"_274":5861,"_276":277},"f5fe9c00f43e",[5824,5829,5832,5837,5840,5845,5848,5853,5856],{"_30":5825,"_16":270,"_271":5826,"_146":5828},"cbbe255eedbd",[5827],"efe6b9a","Then finally you do want some people with",{"_30":5830,"_16":270,"_271":5831,"_146":3542},"0651de5cf05a",[],{"_30":5833,"_16":270,"_271":5834,"_146":5836},"2905e4abb080",[5835],"65faa1d","some machine learning or data science expertise, but mainly just to",{"_30":5838,"_16":270,"_271":5839,"_146":3542},"ec31c150e441",[],{"_30":5841,"_16":270,"_271":5842,"_146":5844},"d3f4ad309a65",[5843],"2a6c0f7","help you set up the evals and the test data sets",{"_30":5846,"_16":270,"_271":5847,"_146":3542},"78f6dc414b8c",[],{"_30":5849,"_16":270,"_271":5850,"_146":5852},"aee8761ff031",[5851],"51af6b8","correctly so that you are not shooting yourself",{"_30":5854,"_16":270,"_271":5855,"_146":3542},"9ac8308898d2",[],{"_30":5857,"_16":270,"_271":5858,"_146":5860},"586647431e66",[5859],"31a30dc","in the foot, you're not training on the test set and things like that.",[5862,5864,5866,5868,5870],{"_30":5827,"_16":3566,"_3567":5863},"00:08:54.580",{"_30":5835,"_16":3566,"_3567":5865},"00:08:57.580",{"_30":5843,"_16":3566,"_3567":5867},"00:09:00.540",{"_30":5851,"_16":3566,"_3567":5869},"00:09:03.200",{"_30":5859,"_16":3566,"_3567":5871},"00:09:06.160",{"_30":5873,"_16":265,"_266":5874,"_274":5904,"_276":277},"b05618d0921d",[5875,5880,5883,5888,5891,5896,5899],{"_30":5876,"_16":270,"_271":5877,"_146":5879},"c440c1f803fb",[5878],"6bb8638","So some",{"_30":5881,"_16":270,"_271":5882,"_146":3542},"b6f7bb6854b0",[],{"_30":5884,"_16":270,"_271":5885,"_146":5887},"96abbb376dc9",[5886],"6365eb4","statistical literacy is helpful, but it's",{"_30":5889,"_16":270,"_271":5890,"_146":3542},"6da764f5d77d",[],{"_30":5892,"_16":270,"_271":5893,"_146":5895},"647dfc54937d",[5894],"52efc3b","90% not 10%. One person on",{"_30":5897,"_16":270,"_271":5898,"_146":3542},"7912fd8af2e1",[],{"_30":5900,"_16":270,"_271":5901,"_146":5903},"c6276cb7f1fd",[5902],"6592a65","the team can easily do that.",[5905,5907,5909,5911],{"_30":5878,"_16":3566,"_3567":5906},"00:09:07.160",{"_30":5886,"_16":3566,"_3567":5908},"00:09:08.720",{"_30":5894,"_16":3566,"_3567":5910},"00:09:11.400",{"_30":5902,"_16":3566,"_3567":5912},"00:09:14.400",{"_30":5914,"_16":265,"_266":5915,"_274":5919,"_276":277},"5b461b6200ab",[5916],{"_30":5917,"_16":270,"_271":5918,"_146":445},"ed4daf233534",[],[],{"_30":5921,"_16":265,"_266":5922,"_274":5927,"_276":349},"63afc71bbe26",[5923],{"_30":5924,"_16":270,"_271":5925,"_146":5926},"0b06a208dcb7",[],"Case Study: GitHub Copilot & Other Examples",[],{"_30":5929,"_16":265,"_266":5930,"_274":5934,"_276":277},"f1651c9c8695",[5931],{"_30":5932,"_16":270,"_271":5933,"_146":445},"5611e4362482",[],[],{"_30":5936,"_16":265,"_266":5937,"_274":5959,"_276":277},"58296c88738f",[5938,5943,5946,5951,5954],{"_30":5939,"_16":270,"_271":5940,"_146":5942},"2d0c5bae232c",[5941],"84e5048","So maybe to give some concrete examples,",{"_30":5944,"_16":270,"_271":5945,"_146":3542},"126adf68ac8c",[],{"_30":5947,"_16":270,"_271":5948,"_146":5950},"c2df934a3156",[5949],"9cee18c","some of these are customers, some of them are just people I've spoken",{"_30":5952,"_16":270,"_271":5953,"_146":3542},"9f1252826d1a",[],{"_30":5955,"_16":270,"_271":5956,"_146":5958},"0ffd8e50b1a3",[5957],"c741146","to a lot about building their application.",[5960,5962,5964],{"_30":5941,"_16":3566,"_3567":5961},"00:09:17.160",{"_30":5949,"_16":3566,"_3567":5963},"00:09:20.160",{"_30":5957,"_16":3566,"_3567":5965},"00:09:23.120",{"_30":5967,"_16":265,"_266":5968,"_274":5990,"_276":277},"1ce45d6fe8aa",[5969,5974,5977,5982,5985],{"_30":5970,"_16":270,"_271":5971,"_146":5973},"7ce3e6931748",[5972],"432a516","But if you're",{"_30":5975,"_16":270,"_271":5976,"_146":3542},"a1dc8109fa73",[],{"_30":5978,"_16":270,"_271":5979,"_146":5981},"a378a059d644",[5980],"cad5b2b","Ironclad or FileVine, at FileVine, it's the lawyers",{"_30":5983,"_16":270,"_271":5984,"_146":3542},"d537b975761f",[],{"_30":5986,"_16":270,"_271":5987,"_146":5989},"e823cebe971f",[5988],"248ee65","or legal professionals who are writing a lot of the prompts.",[5991,5992,5994],{"_30":5972,"_16":3566,"_3567":5965},{"_30":5980,"_16":3566,"_3567":5993},"00:09:26.040",{"_30":5988,"_16":3566,"_3567":5995},"00:09:28.760",{"_30":5997,"_16":265,"_266":5998,"_274":6028,"_276":277},"0a3cf23ddd62",[5999,6004,6007,6012,6015,6020,6023],{"_30":6000,"_16":270,"_271":6001,"_146":6003},"31ef1c197e73",[6002],"ad3f844","And so they're sitting in a UI that does prompt editing",{"_30":6005,"_16":270,"_271":6006,"_146":3542},"8d6df44b7ccf",[],{"_30":6008,"_16":270,"_271":6009,"_146":6011},"6f728d5339ec",[6010],"43e1d65","in Humanloop in this case and spending",{"_30":6013,"_16":270,"_271":6014,"_146":3542},"4fa47314c0aa",[],{"_30":6016,"_16":270,"_271":6017,"_146":6019},"46615c0f883c",[6018],"51f6413","hours iterating on them, changing them, looking at evals, tweaking",{"_30":6021,"_16":270,"_271":6022,"_146":3542},"861c0c19c91d",[],{"_30":6024,"_16":270,"_271":6025,"_146":6027},"2a0a57ba815b",[6026],"f451136","the tool definitions.",[6029,6031,6033,6035],{"_30":6002,"_16":3566,"_3567":6030},"00:09:31.760",{"_30":6010,"_16":3566,"_3567":6032},"00:09:34.630",{"_30":6018,"_16":3566,"_3567":6034},"00:09:37.430",{"_30":6026,"_16":3566,"_3567":6036},"00:09:40.350",{"_30":6038,"_16":265,"_266":6039,"_274":6061,"_276":277},"58433d6ad381",[6040,6045,6048,6053,6056],{"_30":6041,"_16":270,"_271":6042,"_146":6044},"75bbd39e2315",[6043],"c3b72d8","It's not the engineers doing",{"_30":6046,"_16":270,"_271":6047,"_146":3542},"e878184e9f92",[],{"_30":6049,"_16":270,"_271":6050,"_146":6052},"336c882956a8",[6051],"03ad946","that because the engineers don't have the domain expertise to know",{"_30":6054,"_16":270,"_271":6055,"_146":3542},"b470cdc8f2c8",[],{"_30":6057,"_16":270,"_271":6058,"_146":6060},"4311182110a7",[6059],"0509d50","how to instruct the model.",[6062,6063,6065],{"_30":6043,"_16":3566,"_3567":6036},{"_30":6051,"_16":3566,"_3567":6064},"00:09:43.270",{"_30":6059,"_16":3566,"_3567":6066},"00:09:46.350",{"_30":6068,"_16":265,"_266":6069,"_274":6099,"_276":277},"a87728f6b1db",[6070,6075,6078,6083,6086,6091,6094],{"_30":6071,"_16":270,"_271":6072,"_146":6074},"c12439355be0",[6073],"a589c05","And without those legal experts",{"_30":6076,"_16":270,"_271":6077,"_146":3542},"e10923404c1a",[],{"_30":6079,"_16":270,"_271":6080,"_146":6082},"ad04ee4d0ed5",[6081],"a4a47b9","to pass that information in, not",{"_30":6084,"_16":270,"_271":6085,"_146":3542},"9b09ff2534c7",[],{"_30":6087,"_16":270,"_271":6088,"_146":6090},"eaa7f7872962",[6089],"98f4f92","about the outcome, but about what you want the model",{"_30":6092,"_16":270,"_271":6093,"_146":3542},"1afd742b1dae",[],{"_30":6095,"_16":270,"_271":6096,"_146":6098},"df8d165a754c",[6097],"5380c73","to do, like what does good even look like.",[6100,6101,6103,6105],{"_30":6073,"_16":3566,"_3567":6066},{"_30":6081,"_16":3566,"_3567":6102},"00:09:49.350",{"_30":6089,"_16":3566,"_3567":6104},"00:09:52.360",{"_30":6097,"_16":3566,"_3567":6106},"00:09:55.270",{"_30":6108,"_16":265,"_266":6109,"_274":6139,"_276":277},"c0fa8fb35a7c",[6110,6115,6118,6123,6126,6131,6134],{"_30":6111,"_16":270,"_271":6112,"_146":6114},"43d45b43a1da",[6113],"7e2d084","And they're also",{"_30":6116,"_16":270,"_271":6117,"_146":3542},"dcd5ce143f57",[],{"_30":6119,"_16":270,"_271":6120,"_146":6122},"7f5184cb4500",[6121],"d0507b0","involved in",{"_30":6124,"_16":270,"_271":6125,"_146":3542},"ea3f1df33d82",[],{"_30":6127,"_16":270,"_271":6128,"_146":6130},"608fb130f30e",[6129],"b5613fa","helping create the evaluation sets and in doing some of the",{"_30":6132,"_16":270,"_271":6133,"_146":3542},"0006b3cc3bf1",[],{"_30":6135,"_16":270,"_271":6136,"_146":6138},"f0461ab7f868",[6137],"643b599","evaluation as well.",[6140,6141,6143,6145],{"_30":6113,"_16":3566,"_3567":6106},{"_30":6121,"_16":3566,"_3567":6142},"00:09:57.950",{"_30":6129,"_16":3566,"_3567":6144},"00:10:01.030",{"_30":6137,"_16":3566,"_3567":6146},"00:10:03.870",{"_30":6148,"_16":265,"_266":6149,"_274":6163,"_276":277},"9425b72164dd",[6150,6155,6158],{"_30":6151,"_16":270,"_271":6152,"_146":6154},"07030a2173d3",[6153],"c6783eb","I'll talk about sort of how to do evaluation. At",{"_30":6156,"_16":270,"_271":6157,"_146":3542},"d1249d19b116",[],{"_30":6159,"_16":270,"_271":6160,"_146":6162},"aebdcd18e390",[6161],"d542c0d","Duolingo, it's language learning specialists.",[6164,6165],{"_30":6153,"_16":3566,"_3567":6146},{"_30":6161,"_16":3566,"_3567":6166},"00:10:06.550",{"_30":6168,"_16":265,"_266":6169,"_274":6183,"_276":277},"54306a86830f",[6170,6175,6178],{"_30":6171,"_16":270,"_271":6172,"_146":6174},"8fd34b3f7f94",[6173],"e078883","At Fathom, you know, you can imagine it's actually a PM who does a lot of their",{"_30":6176,"_16":270,"_271":6177,"_146":3542},"25db6bbf80fe",[],{"_30":6179,"_16":270,"_271":6180,"_146":6182},"e88fa5542c11",[6181],"24548b9","prompt engineering or it was when I spoke to them last.",[6184,6186],{"_30":6173,"_16":3566,"_3567":6185},"00:10:09.510",{"_30":6181,"_16":3566,"_3567":6187},"00:10:12.270",{"_30":6189,"_16":265,"_266":6190,"_274":6228,"_276":277},"f53c5903aa7d",[6191,6196,6199,6204,6207,6212,6215,6220,6223],{"_30":6192,"_16":270,"_271":6193,"_146":6195},"f6c0a3022757",[6194],"95b0a39","But if you think about",{"_30":6197,"_16":270,"_271":6198,"_146":3542},"7c03012fd643",[],{"_30":6200,"_16":270,"_271":6201,"_146":6203},"57b639493662",[6202],"336b6b1","a summary for a meeting, you want a different summary for",{"_30":6205,"_16":270,"_271":6206,"_146":3542},"b62356c7c722",[],{"_30":6208,"_16":270,"_271":6209,"_146":6211},"75f7eca87309",[6210],"debd40a","a one-on-one than you want for a sales meeting, than you want for a",{"_30":6213,"_16":270,"_271":6214,"_146":3542},"39155147684b",[],{"_30":6216,"_16":270,"_271":6217,"_146":6219},"f7e68a55b50b",[6218],"95b415a","planning session, and the engineer is not necessarily the person",{"_30":6221,"_16":270,"_271":6222,"_146":3542},"b4e2c7bd41bd",[],{"_30":6224,"_16":270,"_271":6225,"_146":6227},"54aa3f763fbc",[6226],"140b49f","who's best placed to do this.",[6229,6230,6232,6234,6236],{"_30":6194,"_16":3566,"_3567":6187},{"_30":6202,"_16":3566,"_3567":6231},"00:10:15.190",{"_30":6210,"_16":3566,"_3567":6233},"00:10:18.150",{"_30":6218,"_16":3566,"_3567":6235},"00:10:20.790",{"_30":6226,"_16":3566,"_3567":6237},"00:10:23.550",{"_30":6239,"_16":265,"_266":6240,"_274":6262,"_276":277},"60169d1b18df",[6241,6246,6249,6254,6257],{"_30":6242,"_16":270,"_271":6243,"_146":6245},"5c23ced64065",[6244],"c5d40cb","So there's one",{"_30":6247,"_16":270,"_271":6248,"_146":3542},"9620e9404933",[],{"_30":6250,"_16":270,"_271":6251,"_146":6253},"605f9a003e13",[6252],"0c8772b","category of exceptional products where the engineer is, which is if you're building",{"_30":6255,"_16":270,"_271":6256,"_146":3542},"b266577c08fb",[],{"_30":6258,"_16":270,"_271":6259,"_146":6261},"5083fde7931e",[6260],"655b556","coding assistants.",[6263,6264,6266],{"_30":6244,"_16":3566,"_3567":6237},{"_30":6252,"_16":3566,"_3567":6265},"00:10:26.350",{"_30":6260,"_16":3566,"_3567":6267},"00:10:29.230",{"_30":6269,"_16":265,"_266":6270,"_274":6292,"_276":277},"4c60701fb1eb",[6271,6276,6279,6284,6287],{"_30":6272,"_16":270,"_271":6273,"_146":6275},"e97c55009709",[6274],"58124f7","But for everybody else, you're",{"_30":6277,"_16":270,"_271":6278,"_146":3542},"f79d94dbf1b9",[],{"_30":6280,"_16":270,"_271":6281,"_146":6283},"97cd6853b05b",[6282],"9ce2558","going to need to really heavily rely on domain experts,",{"_30":6285,"_16":270,"_271":6286,"_146":3542},"143d0ef6115b",[],{"_30":6288,"_16":270,"_271":6289,"_146":6291},"9e8b3d265e12",[6290],"fcf48cc","and I think that's often underlooked how important they are.",[6293,6294,6296],{"_30":6274,"_16":3566,"_3567":6267},{"_30":6282,"_16":3566,"_3567":6295},"00:10:32.210",{"_30":6290,"_16":3566,"_3567":6297},"00:10:35.570",{"_30":6299,"_16":265,"_266":6300,"_274":6322,"_276":277},"d58e8f634b2b",[6301,6306,6309,6314,6317],{"_30":6302,"_16":270,"_271":6303,"_146":6305},"3dd8268c0f75",[6304],"9a22ef1","And actually whilst",{"_30":6307,"_16":270,"_271":6308,"_146":3542},"0abf33e5bff9",[],{"_30":6310,"_16":270,"_271":6311,"_146":6313},"ad71aaae4bb4",[6312],"ce35728","we're talking about teams as well, before I go on to talking about",{"_30":6315,"_16":270,"_271":6316,"_146":3542},"55e1dcf63b76",[],{"_30":6318,"_16":270,"_271":6319,"_146":6321},"c5b1d9907858",[6320],"33f10d6","evaluation—",[6323,6325,6327],{"_30":6304,"_16":3566,"_3567":6324},"00:10:38.410",{"_30":6312,"_16":3566,"_3567":6326},"00:10:41.370",{"_30":6320,"_16":3566,"_3567":6328},"00:10:44.090",{"_30":6330,"_16":265,"_266":6331,"_274":6369,"_276":2466},"251242cefc0d",[6332,6337,6340,6345,6348,6353,6356,6361,6364],{"_30":6333,"_16":270,"_271":6334,"_146":6336},"174837dc7715",[6335],"f05b70c","Another interesting thing that we've seen is",{"_30":6338,"_16":270,"_271":6339,"_146":3542},"f0b1a6d3e6d5",[],{"_30":6341,"_16":270,"_271":6342,"_146":6344},"ac9e9eb9510f",[6343],"f5e9eba","early on, one thing larger companies are often trying to figure",{"_30":6346,"_16":270,"_271":6347,"_146":3542},"b451bdefa9bc",[],{"_30":6349,"_16":270,"_271":6350,"_146":6352},"a76a17ed8b7b",[6351],"8fa5d26","out is how do I get my company to adopt",{"_30":6354,"_16":270,"_271":6355,"_146":3542},"e7041a7ba404",[],{"_30":6357,"_16":270,"_271":6358,"_146":6360},"6116e40e3d69",[6359],"7333c6d","AI or figure out how to build things—how do I transform it and get",{"_30":6362,"_16":270,"_271":6363,"_146":3542},"232849f56a40",[],{"_30":6365,"_16":270,"_271":6366,"_146":6368},"ee7a46425cd0",[6367],"2b274ab","people involved?",[6370,6371,6373,6375,6377],{"_30":6335,"_16":3566,"_3567":6328},{"_30":6343,"_16":3566,"_3567":6372},"00:10:47.010",{"_30":6351,"_16":3566,"_3567":6374},"00:10:49.930",{"_30":6359,"_16":3566,"_3567":6376},"00:10:52.770",{"_30":6367,"_16":3566,"_3567":6378},"00:10:55.610",{"_30":6380,"_16":265,"_266":6381,"_274":6387,"_276":277},"790a2789e737",[6382],{"_30":6383,"_16":270,"_271":6384,"_146":6386},"f006a9ab0ffd",[6385],"30c0ed3","And we've seen two approaches.",[6388],{"_30":6385,"_16":3566,"_3567":6378},{"_30":6390,"_16":265,"_266":6391,"_274":6413,"_276":277},"77c3eebb10b7",[6392,6397,6400,6405,6408],{"_30":6393,"_16":270,"_271":6394,"_146":6396},"2024bda5d184",[6395],"a5a717f","One is more bottom-up. Like I'm going to mandate that everybody build something",{"_30":6398,"_16":270,"_271":6399,"_146":3542},"637688dcfc17",[],{"_30":6401,"_16":270,"_271":6402,"_146":6404},"f9bfb19a0977",[6403],"ab401eb","with, with AI or we have some AI feature in every",{"_30":6406,"_16":270,"_271":6407,"_146":3542},"b4c506fff482",[],{"_30":6409,"_16":270,"_271":6410,"_146":6412},"c9e419dfa323",[6411],"027204e","product over the next domain.",[6414,6416,6418],{"_30":6395,"_16":3566,"_3567":6415},"00:10:58.690",{"_30":6403,"_16":3566,"_3567":6417},"00:11:01.570",{"_30":6411,"_16":3566,"_3567":6419},"00:11:04.380",{"_30":6421,"_16":265,"_266":6422,"_274":6436,"_276":277},"e87b09c2e714",[6423,6428,6431],{"_30":6424,"_16":270,"_271":6425,"_146":6427},"5072821b7a8c",[6426],"f3bf815","And the other approach we've seen is people",{"_30":6429,"_16":270,"_271":6430,"_146":3542},"e0c731b3022d",[],{"_30":6432,"_16":270,"_271":6433,"_146":6435},"732f33254806",[6434],"4e45a3c","centralizing a bit more.",[6437,6438],{"_30":6426,"_16":3566,"_3567":6419},{"_30":6434,"_16":3566,"_3567":6439},"00:11:07.260",{"_30":6441,"_16":265,"_266":6442,"_274":6464,"_276":277},"4c296771c553",[6443,6448,6451,6456,6459],{"_30":6444,"_16":270,"_271":6445,"_146":6447},"ef8f88cc9916",[6446],"f265311","And it tends",{"_30":6449,"_16":270,"_271":6450,"_146":3542},"c892f177c437",[],{"_30":6452,"_16":270,"_271":6453,"_146":6455},"3c5c6bc3181b",[6454],"442cbf5","to be that the centralized efforts have higher impact",{"_30":6457,"_16":270,"_271":6458,"_146":3542},"584c0e0d8cc9",[],{"_30":6460,"_16":270,"_271":6461,"_146":6463},"3ae0881f1df2",[6462],"cae129d","and sort of avoid the—",[6465,6466,6468],{"_30":6446,"_16":3566,"_3567":6439},{"_30":6454,"_16":3566,"_3567":6467},"00:11:10.220",{"_30":6462,"_16":3566,"_3567":6469},"00:11:12.940",{"_30":6471,"_16":265,"_266":6472,"_274":6486,"_276":277},"9bbb6f5a03fe",[6473,6478,6481],{"_30":6474,"_16":270,"_271":6475,"_146":6477},"409754e5b1fd",[6476],"c9135ea","What sometimes",{"_30":6479,"_16":270,"_271":6480,"_146":3542},"3b4bb4f621e3",[],{"_30":6482,"_16":270,"_271":6483,"_146":6485},"ad53cb6c1fe6",[6484],"a24f538","happens is you see people sprinkle AI everywhere.",[6487,6488],{"_30":6476,"_16":3566,"_3567":6469},{"_30":6484,"_16":3566,"_3567":6489},"00:11:15.940",{"_30":6491,"_16":265,"_266":6492,"_274":6522,"_276":277},"1b8328de02d4",[6493,6498,6501,6506,6509,6514,6517],{"_30":6494,"_16":270,"_271":6495,"_146":6497},"e0bc70dbeb36",[6496],"ff1205b","You see those little",{"_30":6499,"_16":270,"_271":6500,"_146":3542},"5fb87f268068",[],{"_30":6502,"_16":270,"_271":6503,"_146":6505},"b835c7f94f2c",[6504],"d64a792","magic, star symbols appear and there's",{"_30":6507,"_16":270,"_271":6508,"_146":3542},"0a4c7f226c05",[],{"_30":6510,"_16":270,"_271":6511,"_146":6513},"ce86edc45917",[6512],"54f2853","lots of incremental AI features like, oh,",{"_30":6515,"_16":270,"_271":6516,"_146":3542},"bfb5b9dfad2f",[],{"_30":6518,"_16":270,"_271":6519,"_146":6521},"f869b312d4ec",[6520],"329ccee","now when you go to type a message, it sort of populates the box for you.",[6523,6524,6526,6528],{"_30":6496,"_16":3566,"_3567":6489},{"_30":6504,"_16":3566,"_3567":6525},"00:11:18.860",{"_30":6512,"_16":3566,"_3567":6527},"00:11:21.660",{"_30":6520,"_16":3566,"_3567":6529},"00:11:24.580",{"_30":6531,"_16":265,"_266":6532,"_274":6554,"_276":277},"cf62cd43a144",[6533,6538,6541,6546,6549],{"_30":6534,"_16":270,"_271":6535,"_146":6537},"bcd341975c0b",[6536],"0fc1302","But no one has the central authority to say actually",{"_30":6539,"_16":270,"_271":6540,"_146":3542},"79ba95a79f0b",[],{"_30":6542,"_16":270,"_271":6543,"_146":6545},"023d4d1c0f86",[6544],"49ea411","what things, if we change them in our product, would",{"_30":6547,"_16":270,"_271":6548,"_146":3542},"1ff6d513f357",[],{"_30":6550,"_16":270,"_271":6551,"_146":6553},"29ccc353ea1c",[6552],"3fb6782","meaningfully move the needle for customers?",[6555,6557,6559],{"_30":6536,"_16":3566,"_3567":6556},"00:11:27.540",{"_30":6544,"_16":3566,"_3567":6558},"00:11:30.340",{"_30":6552,"_16":3566,"_3567":6560},"00:11:33.020",{"_30":6562,"_16":265,"_266":6563,"_274":6567,"_276":277},"497c6f2b0498",[6564],{"_30":6565,"_16":270,"_271":6566,"_146":445},"c075b4272ccf",[],[],{"_30":6569,"_16":265,"_266":6570,"_274":6575,"_276":349},"57703831aa75",[6571],{"_30":6572,"_16":270,"_271":6573,"_146":6574},"1229ae4ffc95",[],"The Right Team for AI Product Development",[],{"_30":6577,"_16":265,"_266":6578,"_274":6582,"_276":277},"0f1b131d7b02",[6579],{"_30":6580,"_16":270,"_271":6581,"_146":445},"e9aaea04363b",[],[],{"_30":6584,"_16":265,"_266":6585,"_274":6607,"_276":277},"d018c1423757",[6586,6591,6594,6599,6602],{"_30":6587,"_16":270,"_271":6588,"_146":6590},"1a9619f62244",[6589],"aaf2f0e","So an example of this was actually Gusto. So Gusto started with a",{"_30":6592,"_16":270,"_271":6593,"_146":3542},"a47954ec0f33",[],{"_30":6595,"_16":270,"_271":6596,"_146":6598},"9cfa1a9a383e",[6597],"ab4bf68","decentralized effort and then eventually pulled everything into a central",{"_30":6600,"_16":270,"_271":6601,"_146":3542},"a4ce70a44e30",[],{"_30":6603,"_16":270,"_271":6604,"_146":6606},"3e88b8c48212",[6605],"5825fb4","AI platform team.",[6608,6610,6612],{"_30":6589,"_16":3566,"_3567":6609},"00:11:35.960",{"_30":6597,"_16":3566,"_3567":6611},"00:11:38.840",{"_30":6605,"_16":3566,"_3567":6613},"00:11:41.600",{"_30":6615,"_16":265,"_266":6616,"_274":6654,"_276":277},"6237a2f82bfd",[6617,6622,6625,6630,6633,6638,6641,6646,6649],{"_30":6618,"_16":270,"_271":6619,"_146":6621},"a15b50d8a0bf",[6620],"047f642","And then they built an agent",{"_30":6623,"_16":270,"_271":6624,"_146":3542},"72b9cd40e677",[],{"_30":6626,"_16":270,"_271":6627,"_146":6629},"11641d7fea79",[6628],"582ea5b","called Gus that basically allows",{"_30":6631,"_16":270,"_271":6632,"_146":3542},"07129b56b85d",[],{"_30":6634,"_16":270,"_271":6635,"_146":6637},"70cbbfd0c1c9",[6636],"eaefb3c","the accountants in the Gusto HR system",{"_30":6639,"_16":270,"_271":6640,"_146":3542},"d395396f3ce4",[],{"_30":6642,"_16":270,"_271":6643,"_146":6645},"3152c2b49751",[6644],"a9e3fe8","to interact with it via an agent that",{"_30":6647,"_16":270,"_271":6648,"_146":3542},"5fa2aa5c7290",[],{"_30":6650,"_16":270,"_271":6651,"_146":6653},"f6aba0d9c51f",[6652],"97f5b05","can do what was hours of work for them automatically.",[6655,6656,6658,6660,6662],{"_30":6620,"_16":3566,"_3567":6613},{"_30":6628,"_16":3566,"_3567":6657},"00:11:44.400",{"_30":6636,"_16":3566,"_3567":6659},"00:11:47.400",{"_30":6644,"_16":3566,"_3567":6661},"00:11:50.360",{"_30":6652,"_16":3566,"_3567":6663},"00:11:53.320",{"_30":6665,"_16":265,"_266":6666,"_274":6704,"_276":277},"7f585a4f4dd9",[6667,6672,6675,6680,6683,6688,6691,6696,6699],{"_30":6668,"_16":270,"_271":6669,"_146":6671},"e2b777ce53a6",[6670],"ae9ba05","And so the difference between sort of like having—",{"_30":6673,"_16":270,"_271":6674,"_146":3542},"b51dfa2859fe",[],{"_30":6676,"_16":270,"_271":6677,"_146":6679},"bb83dd563e7e",[6678],"5d55ff6","I think their first product was like an auto CV generator for a",{"_30":6681,"_16":270,"_271":6682,"_146":3542},"19d2b4cd8279",[],{"_30":6684,"_16":270,"_271":6685,"_146":6687},"4b4bc4a76758",[6686],"10b09b3","job ad which was like useful but, but saved you a few",{"_30":6689,"_16":270,"_271":6690,"_146":3542},"9ecffa6016fb",[],{"_30":6692,"_16":270,"_271":6693,"_146":6695},"e28507bfd76c",[6694],"510c2e7","minutes versus eventually shipping an agent that now saves",{"_30":6697,"_16":270,"_271":6698,"_146":3542},"d6940ab2b71c",[],{"_30":6700,"_16":270,"_271":6701,"_146":6703},"de94a7660d29",[6702],"6e60110","hours of work for an accountant every time they interact with it.",[6705,6707,6709,6711,6713],{"_30":6670,"_16":3566,"_3567":6706},"00:11:56.120",{"_30":6678,"_16":3566,"_3567":6708},"00:11:59.000",{"_30":6686,"_16":3566,"_3567":6710},"00:12:01.960",{"_30":6694,"_16":3566,"_3567":6712},"00:12:04.600",{"_30":6702,"_16":3566,"_3567":6714},"00:12:07.560",{"_30":6716,"_16":265,"_266":6717,"_274":6747,"_276":277},"f5e190772488",[6718,6723,6726,6731,6734,6739,6742],{"_30":6719,"_16":270,"_271":6720,"_146":6722},"de26f687078c",[6721],"e808f7f","The impact you can have is much higher but it required one centralized",{"_30":6724,"_16":270,"_271":6725,"_146":3542},"beccbe3fea89",[],{"_30":6727,"_16":270,"_271":6728,"_146":6730},"4644a9ebafde",[6729],"8cc8b77","team who could have the",{"_30":6732,"_16":270,"_271":6733,"_146":3542},"456f38c93b44",[],{"_30":6735,"_16":270,"_271":6736,"_146":6738},"bab6f745c4ba",[6737],"e0f0436","overview of the whole product to be able to do that",{"_30":6740,"_16":270,"_271":6741,"_146":3542},"57744222a231",[],{"_30":6743,"_16":270,"_271":6744,"_146":6746},"124547292807",[6745],"3bc06bf","and the authority as well.",[6748,6750,6752,6754],{"_30":6721,"_16":3566,"_3567":6749},"00:12:10.480",{"_30":6729,"_16":3566,"_3567":6751},"00:12:13.480",{"_30":6737,"_16":3566,"_3567":6753},"00:12:16.280",{"_30":6745,"_16":3566,"_3567":6755},"00:12:19.000",{"_30":6757,"_16":265,"_266":6758,"_274":6788,"_276":277},"f041f9241dc5",[6759,6764,6767,6772,6775,6780,6783],{"_30":6760,"_16":270,"_271":6761,"_146":6763},"92eefee8a331",[6762],"c83a146","Okay, the next thing I want to chat about is the",{"_30":6765,"_16":270,"_271":6766,"_146":3542},"5fba6c539ad9",[],{"_30":6768,"_16":270,"_271":6769,"_146":6771},"73130e00531d",[6770],"67db625","evaluation itself. So I'm going to talk about GitHub Copilot a lot because it's still",{"_30":6773,"_16":270,"_271":6774,"_146":3542},"fb10d5745e91",[],{"_30":6776,"_16":270,"_271":6777,"_146":6779},"99c337921f5f",[6778],"e994e81","my favorite of the OG LLM",{"_30":6781,"_16":270,"_271":6782,"_146":3542},"4bd6e163ce2e",[],{"_30":6784,"_16":270,"_271":6785,"_146":6787},"44c416ef08c4",[6786],"81fa3e5","apps that I think paved the way.",[6789,6791,6793,6795],{"_30":6762,"_16":3566,"_3567":6790},"00:12:21.480",{"_30":6770,"_16":3566,"_3567":6792},"00:12:24.400",{"_30":6778,"_16":3566,"_3567":6794},"00:12:27.280",{"_30":6786,"_16":3566,"_3567":6796},"00:12:30.040",{"_30":6798,"_16":265,"_266":6799,"_274":6837,"_276":277},"f8e7d7ee2a21",[6800,6805,6808,6813,6816,6821,6824,6829,6832],{"_30":6801,"_16":270,"_271":6802,"_146":6804},"d2a7cb02235a",[6803],"c9b87a6","And Alex Gravely is one",{"_30":6806,"_16":270,"_271":6807,"_146":3542},"90d41f6da3ff",[],{"_30":6809,"_16":270,"_271":6810,"_146":6812},"7d7534cb1ca5",[6811],"41fee7f","of the co creators of it and I think this statement that he made",{"_30":6814,"_16":270,"_271":6815,"_146":3542},"d1350a699400",[],{"_30":6817,"_16":270,"_271":6818,"_146":6820},"fd8f6a12475b",[6819],"303f3a9","still stands true, which is in his opinion, \"Baseline",{"_30":6822,"_16":270,"_271":6823,"_146":3542},"92c4339a15e8",[],{"_30":6825,"_16":270,"_271":6826,"_146":6828},"540540cacb3c",[6827],"3451499","evaluation has to be the core of any AI",{"_30":6830,"_16":270,"_271":6831,"_146":3542},"46533322c2c0",[],{"_30":6833,"_16":270,"_271":6834,"_146":6836},"e8dc7c24f41d",[6835],"510ade5","application.\"",[6838,6839,6841,6843,6845],{"_30":6803,"_16":3566,"_3567":6796},{"_30":6811,"_16":3566,"_3567":6840},"00:12:33.000",{"_30":6819,"_16":3566,"_3567":6842},"00:12:35.740",{"_30":6827,"_16":3566,"_3567":6844},"00:12:38.660",{"_30":6835,"_16":3566,"_3567":6846},"00:12:41.180",{"_30":6848,"_16":265,"_266":6849,"_274":6863,"_276":277},"f2806dcd06ce",[6850,6855,6858],{"_30":6851,"_16":270,"_271":6852,"_146":6854},"31c5551c621b",[6853],"418d166","And so evaluation I think",{"_30":6856,"_16":270,"_271":6857,"_146":3542},"230d0557ad43",[],{"_30":6859,"_16":270,"_271":6860,"_146":6862},"3c1ddadb7758",[6861],"a19ea8d","matters at every stage of development, but it changes as you go.",[6864,6865],{"_30":6853,"_16":3566,"_3567":6846},{"_30":6861,"_16":3566,"_3567":6866},"00:12:44.060",{"_30":6868,"_16":265,"_266":6869,"_274":6891,"_276":277},"a0f063c048b1",[6870,6875,6878,6883,6886],{"_30":6871,"_16":270,"_271":6872,"_146":6874},"d8433246b747",[6873],"6b91278","So when you're starting a new project, you're just beginning out,",{"_30":6876,"_16":270,"_271":6877,"_146":3542},"62de0b7640aa",[],{"_30":6879,"_16":270,"_271":6880,"_146":6882},"445447b45a60",[6881],"3f73622","actually the first thing you're trying to check is just like, is it",{"_30":6884,"_16":270,"_271":6885,"_146":3542},"a4664f0d3cfa",[],{"_30":6887,"_16":270,"_271":6888,"_146":6890},"a03f2d1282e8",[6889],"58349ee","possible for the models to do this?",[6892,6894,6896],{"_30":6873,"_16":3566,"_3567":6893},"00:12:46.980",{"_30":6881,"_16":3566,"_3567":6895},"00:12:49.700",{"_30":6889,"_16":3566,"_3567":6897},"00:12:52.540",{"_30":6899,"_16":265,"_266":6900,"_274":6914,"_276":277},"bc794de36986",[6901,6906,6909],{"_30":6902,"_16":270,"_271":6903,"_146":6905},"58fed732d50d",[6904],"1b7b277","And that's a moving target",{"_30":6907,"_16":270,"_271":6908,"_146":3542},"85fe022c0291",[],{"_30":6910,"_16":270,"_271":6911,"_146":6913},"25d76394201c",[6912],"d6371fa","constantly. And so the first level tends to be very interactive.",[6915,6916],{"_30":6904,"_16":3566,"_3567":6897},{"_30":6912,"_16":3566,"_3567":6917},"00:12:55.460",{"_30":6919,"_16":265,"_266":6920,"_274":6942,"_276":277},"2cc2e524e488",[6921,6926,6929,6934,6937],{"_30":6922,"_16":270,"_271":6923,"_146":6925},"c94937a3c755",[6924],"33469e4","People are going into a notebook or a playground type environment,",{"_30":6927,"_16":270,"_271":6928,"_146":3542},"ea2355527904",[],{"_30":6930,"_16":270,"_271":6931,"_146":6933},"d5978a027913",[6932],"a320f56","they're trying things out by hand, and as they're doing that",{"_30":6935,"_16":270,"_271":6936,"_146":3542},"8bfc1fac7fcb",[],{"_30":6938,"_16":270,"_271":6939,"_146":6941},"3237a9ad9c73",[6940],"b7eb9a7","they're creating the evaluation criteria alongside.",[6943,6945,6947],{"_30":6924,"_16":3566,"_3567":6944},"00:12:58.420",{"_30":6932,"_16":3566,"_3567":6946},"00:13:01.140",{"_30":6940,"_16":3566,"_3567":6948},"00:13:04.100",{"_30":6950,"_16":265,"_266":6951,"_274":6973,"_276":277},"9ab1ac6debf2",[6952,6957,6960,6965,6968],{"_30":6953,"_16":270,"_271":6954,"_146":6956},"e6e251f96d08",[6955],"bf00dc2","Because as",{"_30":6958,"_16":270,"_271":6959,"_146":3542},"f8e440052d67",[],{"_30":6961,"_16":270,"_271":6962,"_146":6964},"08aa95c217bd",[6963],"af47177","you do it, that's when you realize, oh, actually",{"_30":6966,"_16":270,"_271":6967,"_146":3542},"e91ca9bbaaf6",[],{"_30":6969,"_16":270,"_271":6970,"_146":6972},"4112427f6fac",[6971],"8f4cbac","I have this particular constraint that I need the model to satisfy.",[6974,6976,6978],{"_30":6955,"_16":3566,"_3567":6975},"00:13:05.100",{"_30":6963,"_16":3566,"_3567":6977},"00:13:06.940",{"_30":6971,"_16":3566,"_3567":6979},"00:13:09.920",{"_30":6981,"_16":265,"_266":6982,"_274":7020,"_276":277},"99b9c093b5df",[6983,6988,6991,6996,6999,7004,7007,7012,7015],{"_30":6984,"_16":270,"_271":6985,"_146":6987},"a397990c314e",[6986],"44b220f","And so the spec or the evaluation co-evolves in this",{"_30":6989,"_16":270,"_271":6990,"_146":3542},"75f597ff7d13",[],{"_30":6992,"_16":270,"_271":6993,"_146":6995},"1a6baf1b6f0c",[6994],"9736c17","initial stage of exploration, at the end of",{"_30":6997,"_16":270,"_271":6998,"_146":3542},"0f933e1ce6bc",[],{"_30":7000,"_16":270,"_271":7001,"_146":7003},"f71cf11ee7b8",[7002],"5241d60","which you probably have the first version of the system,",{"_30":7005,"_16":270,"_271":7006,"_146":3542},"176ed40908a3",[],{"_30":7008,"_16":270,"_271":7009,"_146":7011},"b7b86b0758cb",[7010],"e32fa80","and then people will run it over a mini data set and they kind",{"_30":7013,"_16":270,"_271":7014,"_146":3542},"e704f57616e3",[],{"_30":7016,"_16":270,"_271":7017,"_146":7019},"fe2df75f75d1",[7018],"2c6c44d","of get into the development stage.",[7021,7023,7025,7027,7029],{"_30":6986,"_16":3566,"_3567":7022},"00:13:12.960",{"_30":6994,"_16":3566,"_3567":7024},"00:13:15.920",{"_30":7002,"_16":3566,"_3567":7026},"00:13:18.920",{"_30":7010,"_16":3566,"_3567":7028},"00:13:22.420",{"_30":7018,"_16":3566,"_3567":7030},"00:13:25.360",{"_30":7032,"_16":265,"_266":7033,"_274":7047,"_276":277},"ca089ccb0715",[7034,7039,7042],{"_30":7035,"_16":270,"_271":7036,"_146":7038},"1fd7fe71b44f",[7037],"1aa0079","They'll run it over that mini data set",{"_30":7040,"_16":270,"_271":7041,"_146":3542},"ca4dd910f03f",[],{"_30":7043,"_16":270,"_271":7044,"_146":7046},"520e035e950d",[7045],"d0d9e51","and they'll do an error analysis of like, what are the failure modes?",[7048,7049],{"_30":7037,"_16":3566,"_3567":7030},{"_30":7045,"_16":3566,"_3567":7050},"00:13:28.280",{"_30":7052,"_16":265,"_266":7053,"_274":7075,"_276":277},"66e79ecfd43e",[7054,7059,7062,7067,7070],{"_30":7055,"_16":270,"_271":7056,"_146":7058},"bea99d0425e8",[7057],"a6474f0","And every error mode becomes an",{"_30":7060,"_16":270,"_271":7061,"_146":3542},"e6ded7939f01",[],{"_30":7063,"_16":270,"_271":7064,"_146":7066},"4afb4df763ca",[7065],"f40f8f7","eval and usually it becomes some form of binary eval,",{"_30":7068,"_16":270,"_271":7069,"_146":3542},"6a2b82286ad7",[],{"_30":7071,"_16":270,"_271":7072,"_146":7074},"10ce2ada3d66",[7073],"e50e57c","a sort of pass-fail test.",[7076,7078,7080],{"_30":7057,"_16":3566,"_3567":7077},"00:13:31.200",{"_30":7065,"_16":3566,"_3567":7079},"00:13:34.080",{"_30":7073,"_16":3566,"_3567":7081},"00:13:37.600",{"_30":7083,"_16":265,"_266":7084,"_274":7088,"_276":277},"fe9edda5a0a8",[7085],{"_30":7086,"_16":270,"_271":7087,"_146":445},"176451291123",[],[],{"_30":7090,"_16":265,"_266":7091,"_274":7096,"_276":349},"054d33ce677c",[7092],{"_30":7093,"_16":270,"_271":7094,"_146":7095},"c58a24894671",[],"Centralized Versus Bottom-Up AI Strategy",[],{"_30":7098,"_16":265,"_266":7099,"_274":7103,"_276":277},"e73283e48cf9",[7100],{"_30":7101,"_16":270,"_271":7102,"_146":445},"9dafb96338a4",[],[],{"_30":7105,"_16":265,"_266":7106,"_274":7128,"_276":277},"3e8328b11fb0",[7107,7112,7115,7120,7123],{"_30":7108,"_16":270,"_271":7109,"_146":7111},"666f415a0938",[7110],"4862656","And this is one of the biggest differences",{"_30":7113,"_16":270,"_271":7114,"_146":3542},"d39db473a9e4",[],{"_30":7116,"_16":270,"_271":7117,"_146":7119},"eeede5023107",[7118],"94019a5","between coding for LLMs or AI versus",{"_30":7121,"_16":270,"_271":7122,"_146":3542},"0a7c5c0b9252",[],{"_30":7124,"_16":270,"_271":7125,"_146":7127},"bd54edc7fdab",[7126],"859e02e","traditional software.",[7129,7131,7133],{"_30":7110,"_16":3566,"_3567":7130},"00:13:40.360",{"_30":7118,"_16":3566,"_3567":7132},"00:13:43.120",{"_30":7126,"_16":3566,"_3567":7134},"00:13:46.000",{"_30":7136,"_16":265,"_266":7137,"_274":7151,"_276":277},"ec87dcb8af4e",[7138,7143,7146],{"_30":7139,"_16":270,"_271":7140,"_146":7142},"76638aff9f27",[7141],"22cefb4","If you're coming as a traditional software engineer,",{"_30":7144,"_16":270,"_271":7145,"_146":3542},"c07867707db2",[],{"_30":7147,"_16":270,"_271":7148,"_146":7150},"13e9490c45f5",[7149],"a388486","you're probably used to being able to write deterministic unit and integration tests.",[7152,7153],{"_30":7141,"_16":3566,"_3567":7134},{"_30":7149,"_16":3566,"_3567":7154},"00:13:48.960",{"_30":7156,"_16":265,"_266":7157,"_274":7179,"_276":277},"58fa04f6fce7",[7158,7163,7166,7171,7174],{"_30":7159,"_16":270,"_271":7160,"_146":7162},"4a98dffe8ab9",[7161],"60bbcec","And although this has been true in machine learning for a long",{"_30":7164,"_16":270,"_271":7165,"_146":3542},"8a6bd36fac94",[],{"_30":7167,"_16":270,"_271":7168,"_146":7170},"ed70ebdab6d9",[7169],"4953c51","time, for traditional software development, it's not the case that every time you run the",{"_30":7172,"_16":270,"_271":7173,"_146":3542},"6c4b6fb4a40a",[],{"_30":7175,"_16":270,"_271":7176,"_146":7178},"46064619f3bb",[7177],"1022d13","program you get a different result.",[7180,7182,7184],{"_30":7161,"_16":3566,"_3567":7181},"00:13:51.680",{"_30":7169,"_16":3566,"_3567":7183},"00:13:54.600",{"_30":7177,"_16":3566,"_3567":7185},"00:13:57.520",{"_30":7187,"_16":265,"_266":7188,"_274":7210,"_276":277},"84fbb6b2038d",[7189,7194,7197,7202,7205],{"_30":7190,"_16":270,"_271":7191,"_146":7193},"1047ecd0cd7e",[7192],"fec900d","And so just figuring out like, oh,",{"_30":7195,"_16":270,"_271":7196,"_146":3542},"5d9a4767f7dc",[],{"_30":7198,"_16":270,"_271":7199,"_146":7201},"c5c790a75ecd",[7200],"830b3b6","actually what I need to do is not write a unit test, but",{"_30":7203,"_16":270,"_271":7204,"_146":3542},"3020255f9a2e",[],{"_30":7206,"_16":270,"_271":7207,"_146":7209},"b1bf70df56b6",[7208],"95c8be1","I need to write an assertion against a data set.",[7211,7212,7214],{"_30":7192,"_16":3566,"_3567":7185},{"_30":7200,"_16":3566,"_3567":7213},"00:14:00.440",{"_30":7208,"_16":3566,"_3567":7215},"00:14:03.340",{"_30":7217,"_16":265,"_266":7218,"_274":7248,"_276":277},"3983d3bd1940",[7219,7224,7227,7232,7235,7240,7243],{"_30":7220,"_16":270,"_271":7221,"_146":7223},"3f09b47de2db",[7222],"ab6ac99","So when the",{"_30":7225,"_16":270,"_271":7226,"_146":3542},"92e5d3c3edb2",[],{"_30":7228,"_16":270,"_271":7229,"_146":7231},"589357efb600",[7230],"1842387","model gets run over that data set, I need to know if the accuracy stays",{"_30":7233,"_16":270,"_271":7234,"_146":3542},"46902dd5b962",[],{"_30":7236,"_16":270,"_271":7237,"_146":7239},"a6b7572247a0",[7238],"6126dec","above some percentage or I need to know that some",{"_30":7241,"_16":270,"_271":7242,"_146":3542},"bcc36e131d8c",[],{"_30":7244,"_16":270,"_271":7245,"_146":7247},"6703bc856b5c",[7246],"8be3cc4","criteria continues to be satisfied.",[7249,7250,7252,7254],{"_30":7222,"_16":3566,"_3567":7215},{"_30":7230,"_16":3566,"_3567":7251},"00:14:06.180",{"_30":7238,"_16":3566,"_3567":7253},"00:14:09.180",{"_30":7246,"_16":3566,"_3567":7255},"00:14:12.980",{"_30":7257,"_16":265,"_266":7258,"_274":7272,"_276":277},"12e081a72dc1",[7259,7264,7267],{"_30":7260,"_16":270,"_271":7261,"_146":7263},"dac7c20940cf",[7262],"69b79e8","And then once you move from development into production, evals",{"_30":7265,"_16":270,"_271":7266,"_146":3542},"90ccbde3ca3b",[],{"_30":7268,"_16":270,"_271":7269,"_146":7271},"6104a4b4004e",[7270],"6ea0b8f","don't stop.",[7273,7275],{"_30":7262,"_16":3566,"_3567":7274},"00:14:15.780",{"_30":7270,"_16":3566,"_3567":7276},"00:14:18.780",{"_30":7278,"_16":265,"_266":7279,"_274":7309,"_276":277},"4b25ef8159b5",[7280,7285,7288,7293,7296,7301,7304],{"_30":7281,"_16":270,"_271":7282,"_146":7284},"3b52f1ef5cf6",[7283],"9d0712f","Like, you still want to be able to monitor",{"_30":7286,"_16":270,"_271":7287,"_146":3542},"7b334cf8fb84",[],{"_30":7289,"_16":270,"_271":7290,"_146":7292},"53c03f502fb0",[7291],"7206f2b","your production traffic because you want to",{"_30":7294,"_16":270,"_271":7295,"_146":3542},"be157b094688",[],{"_30":7297,"_16":270,"_271":7298,"_146":7300},"a6e6a05ed90f",[7299],"dac0bc8","sort of understand the full generality of inputs",{"_30":7302,"_16":270,"_271":7303,"_146":3542},"d09f8682b0cf",[],{"_30":7305,"_16":270,"_271":7306,"_146":7308},"449368703d17",[7307],"150b9a0","that are coming in, how is that changing over time?",[7310,7311,7313,7315],{"_30":7283,"_16":3566,"_3567":7276},{"_30":7291,"_16":3566,"_3567":7312},"00:14:21.620",{"_30":7299,"_16":3566,"_3567":7314},"00:14:24.540",{"_30":7307,"_16":3566,"_3567":7316},"00:14:27.540",{"_30":7318,"_16":265,"_266":7319,"_274":7333,"_276":277},"1a4549d9f330",[7320,7325,7328],{"_30":7321,"_16":270,"_271":7322,"_146":7324},"b4fe39a3a1a7",[7323],"3c23dee","So the same evals that you use in development, some of them will require ground",{"_30":7326,"_16":270,"_271":7327,"_146":3542},"18d84cce0766",[],{"_30":7329,"_16":270,"_271":7330,"_146":7332},"b7af2de9e5a6",[7331],"5e510a6","truth, but a lot of them won't.",[7334,7336],{"_30":7323,"_16":3566,"_3567":7335},"00:14:30.690",{"_30":7331,"_16":3566,"_3567":7337},"00:14:33.690",{"_30":7339,"_16":265,"_266":7340,"_274":7362,"_276":277},"ea76c173bc82",[7341,7346,7349,7354,7357],{"_30":7342,"_16":270,"_271":7343,"_146":7345},"4a6f38fb30e2",[7344],"67b4aff","And so if they don't require ground truth, you",{"_30":7347,"_16":270,"_271":7348,"_146":3542},"8e9ba9e127f7",[],{"_30":7350,"_16":270,"_271":7351,"_146":7353},"9598253fa192",[7352],"c119fda","can actually run them as monitors. And so you can have a set of scores",{"_30":7355,"_16":270,"_271":7356,"_146":3542},"f0932dd30537",[],{"_30":7358,"_16":270,"_271":7359,"_146":7361},"0edc9e6bbf76",[7360],"e0c94df","that evolve over time.",[7363,7364,7366],{"_30":7344,"_16":3566,"_3567":7337},{"_30":7352,"_16":3566,"_3567":7365},"00:14:36.690",{"_30":7360,"_16":3566,"_3567":7367},"00:14:39.650",{"_30":7369,"_16":265,"_266":7370,"_274":7384,"_276":277},"7305ba362138",[7371,7376,7379],{"_30":7372,"_16":270,"_271":7373,"_146":7375},"83a7ee841e49",[7374],"57e2e87","Most of our customers",{"_30":7377,"_16":270,"_271":7378,"_146":3542},"410d7fd26894",[],{"_30":7380,"_16":270,"_271":7381,"_146":7383},"1bd6892bba28",[7382],"cf7eddf","now trigger an eval set every time they make a major change.",[7385,7386],{"_30":7374,"_16":3566,"_3567":7367},{"_30":7382,"_16":3566,"_3567":7387},"00:14:42.650",{"_30":7389,"_16":265,"_266":7390,"_274":7404,"_276":277},"98baa2168294",[7391,7396,7399],{"_30":7392,"_16":270,"_271":7393,"_146":7395},"3285ac3e3db0",[7394],"505aca3","So if you're trying to upgrade the model, if you're making a",{"_30":7397,"_16":270,"_271":7398,"_146":3542},"b1b92cb41a19",[],{"_30":7400,"_16":270,"_271":7401,"_146":7403},"c49a4a25a0be",[7402],"cc86b5b","particular change to a system prompt, this is just like running tests in CI/CD",[7405,7407],{"_30":7394,"_16":3566,"_3567":7406},"00:14:45.369",{"_30":7402,"_16":3566,"_3567":7408},"00:14:48.170",{"_30":7410,"_16":265,"_266":7411,"_274":7433,"_276":277},"0ea608ae3f53",[7412,7417,7420,7425,7428],{"_30":7413,"_16":270,"_271":7414,"_146":7416},"0da617940975",[7415],"e6490c5","For a lot of our",{"_30":7418,"_16":270,"_271":7419,"_146":3542},"b3c5f97509d8",[],{"_30":7421,"_16":270,"_271":7422,"_146":7424},"55d5c7feffde",[7423],"6f6584e","customers it actually happens as a GitHub action or a lot of people do it",{"_30":7426,"_16":270,"_271":7427,"_146":3542},"bed67f98bf15",[],{"_30":7429,"_16":270,"_271":7430,"_146":7432},"2afbaaa4f776",[7431],"34ec1fd","that way, and that then triggers an evaluation.",[7434,7436,7438],{"_30":7415,"_16":3566,"_3567":7435},"00:14:50.970",{"_30":7423,"_16":3566,"_3567":7437},"00:14:53.930",{"_30":7431,"_16":3566,"_3567":7439},"00:14:56.810",{"_30":7441,"_16":265,"_266":7442,"_274":7464,"_276":277},"dcd74355cfb2",[7443,7448,7451,7456,7459],{"_30":7444,"_16":270,"_271":7445,"_146":7447},"7b0bf750751a",[7446],"9f6df55","The evaluation runs and it comes back and it says the scores",{"_30":7449,"_16":270,"_271":7450,"_146":3542},"454b2b25a629",[],{"_30":7452,"_16":270,"_271":7453,"_146":7455},"acb063e08f60",[7454],"dc539bd","are, they're never the same but they're still within the range that's",{"_30":7457,"_16":270,"_271":7458,"_146":3542},"68932e106835",[],{"_30":7460,"_16":270,"_271":7461,"_146":7463},"0dc6f40a08a6",[7462],"35a8abc","acceptable or not.",[7465,7467,7469],{"_30":7446,"_16":3566,"_3567":7466},"00:14:59.650",{"_30":7454,"_16":3566,"_3567":7468},"00:15:02.630",{"_30":7462,"_16":3566,"_3567":7470},"00:15:05.630",{"_30":7472,"_16":265,"_266":7473,"_274":7537,"_276":2466},"14694f5d6831",[7474,7479,7484,7487,7492,7495,7500,7505,7508,7513,7516,7521,7524,7529,7532],{"_30":7475,"_16":270,"_271":7476,"_146":7478},"84fbb042ee3d",[7477],"a5bf6fd","Why is evaluation for this stuff so hard?",{"_30":7480,"_16":270,"_271":7481,"_146":7483},"bd1ebf1baaa1",[7482],"ae11502"," Well one thing is that foundation models are just very general and",{"_30":7485,"_16":270,"_271":7486,"_146":3542},"7ed5bdac9ee9",[],{"_30":7488,"_16":270,"_271":7489,"_146":7491},"4f942bbd9c82",[7490],"b6641e5","so the types of applications that they're applying to, they have a",{"_30":7493,"_16":270,"_271":7494,"_146":3542},"11e71679dbdc",[],{"_30":7496,"_16":270,"_271":7497,"_146":7499},"4b0189852e29",[7498],"69d1c09","very long tail of input that might come in.",{"_30":7501,"_16":270,"_271":7502,"_146":7504},"0625e62f608c",[7503],"dce33de"," If you're doing meeting",{"_30":7506,"_16":270,"_271":7507,"_146":3542},"9a34cf48179f",[],{"_30":7509,"_16":270,"_271":7510,"_146":7512},"83fefe63190e",[7511],"1799f16","summarization, the range of meetings that someone might check into",{"_30":7514,"_16":270,"_271":7515,"_146":3542},"6c6165a28365",[],{"_30":7517,"_16":270,"_271":7518,"_146":7520},"20d024923c6b",[7519],"802aa22","your model or something like that is so broad that it's hard to",{"_30":7522,"_16":270,"_271":7523,"_146":3542},"85a74fb5a183",[],{"_30":7525,"_16":270,"_271":7526,"_146":7528},"c1c39dc2060f",[7527],"e9ad176","anticipate all the different ways in which someone might try to use your",{"_30":7530,"_16":270,"_271":7531,"_146":3542},"3533234e9453",[],{"_30":7533,"_16":270,"_271":7534,"_146":7536},"442cb4f00e3e",[7535],"322e611","product.",[7538,7540,7542,7544,7546,7547,7549,7551,7553],{"_30":7477,"_16":3566,"_3567":7539},"00:15:08.630",{"_30":7482,"_16":3566,"_3567":7541},"00:15:11.390",{"_30":7490,"_16":3566,"_3567":7543},"00:15:14.150",{"_30":7498,"_16":3566,"_3567":7545},"00:15:16.990",{"_30":7503,"_16":3566,"_3567":7545},{"_30":7511,"_16":3566,"_3567":7548},"00:15:19.390",{"_30":7519,"_16":3566,"_3567":7550},"00:15:22.310",{"_30":7527,"_16":3566,"_3567":7552},"00:15:25.150",{"_30":7535,"_16":3566,"_3567":7554},"00:15:28.110",{"_30":7556,"_16":265,"_266":7557,"_274":7561,"_276":277},"02ce8fda2c36",[7558],{"_30":7559,"_16":270,"_271":7560,"_146":445},"7198888a229f",[],[],{"_30":7563,"_16":265,"_266":7564,"_274":7569,"_276":349},"f66c7de41ca6",[7565],{"_30":7566,"_16":270,"_271":7567,"_146":7568},"17b40ca302a0",[],"Evaluation as the Product Spec",[],{"_30":7571,"_16":265,"_266":7572,"_274":7576,"_276":277},"21d1b8021d0c",[7573],{"_30":7574,"_16":270,"_271":7575,"_146":445},"2b3d66bf4118",[],[],{"_30":7578,"_16":265,"_266":7579,"_274":7585,"_276":277},"a65e13399f10",[7580],{"_30":7581,"_16":270,"_271":7582,"_146":7584},"7fab06d09e58",[7583],"1a33eb2","I've already mentioned the stochasticity. Every time you run this, you're getting different results.",[7586],{"_30":7583,"_16":3566,"_3567":7587},"00:15:30.990",{"_30":7589,"_16":265,"_266":7590,"_274":7604,"_276":277},"071128e989ea",[7591,7596,7599],{"_30":7592,"_16":270,"_271":7593,"_146":7595},"0c8873fe4d9b",[7594],"142929c","Then compared to traditional machine learning applications, it's often a lot",{"_30":7597,"_16":270,"_271":7598,"_146":3542},"6c0786e96957",[],{"_30":7600,"_16":270,"_271":7601,"_146":7603},"1242e8ebeff3",[7602],"5ea3c4e","more subjective.",[7605,7607],{"_30":7594,"_16":3566,"_3567":7606},"00:15:33.920",{"_30":7602,"_16":3566,"_3567":7608},"00:15:36.840",{"_30":7610,"_16":265,"_266":7611,"_274":7681,"_276":2466},"c024cbd612d2",[7612,7617,7620,7625,7628,7633,7636,7641,7644,7649,7652,7657,7660,7665,7668,7673,7676],{"_30":7613,"_16":270,"_271":7614,"_146":7616},"241329490f10",[7615],"96ca9de","Even in traditional ML, you have accuracy,",{"_30":7618,"_16":270,"_271":7619,"_146":3542},"5572cf91b6a4",[],{"_30":7621,"_16":270,"_271":7622,"_146":7624},"e4c164742d2b",[7623],"3fae335","F1, precision recall, BLEU scores. There are quantitative",{"_30":7626,"_16":270,"_271":7627,"_146":3542},"71354c49d42f",[],{"_30":7629,"_16":270,"_271":7630,"_146":7632},"432a4a4eeda5",[7631],"e7a0312","metrics of how this compares to the ground",{"_30":7634,"_16":270,"_271":7635,"_146":3542},"d72444edc9c9",[],{"_30":7637,"_16":270,"_271":7638,"_146":7640},"71d37e0731b0",[7639],"b8c3dff","truth. With a lot of these more creative",{"_30":7642,"_16":270,"_271":7643,"_146":3542},"166e03dba7bf",[],{"_30":7645,"_16":270,"_271":7646,"_146":7648},"7f7671aba1f1",[7647],"867d084","subjective writing applications, there isn't one",{"_30":7650,"_16":270,"_271":7651,"_146":3542},"cfe195887c22",[],{"_30":7653,"_16":270,"_271":7654,"_146":7656},"0df5093ac20b",[7655],"df4c888","correct answer and so you can't specify",{"_30":7658,"_16":270,"_271":7659,"_146":3542},"9c106bf84c4b",[],{"_30":7661,"_16":270,"_271":7662,"_146":7664},"894fd988c49b",[7663],"665353e","this is the right answer to check against.",{"_30":7666,"_16":270,"_271":7667,"_146":3542},"902b3f7a581f",[],{"_30":7669,"_16":270,"_271":7670,"_146":7672},"bbaef37a2641",[7671],"94ad8a2","And so that subjectivity makes it hard to",{"_30":7674,"_16":270,"_271":7675,"_146":3542},"9d72a18b0ae0",[],{"_30":7677,"_16":270,"_271":7678,"_146":7680},"7d2e472ab1d0",[7679],"a433c13","necessarily score these things.",[7682,7684,7686,7688,7690,7692,7694,7696,7698],{"_30":7615,"_16":3566,"_3567":7683},"00:15:39.680",{"_30":7623,"_16":3566,"_3567":7685},"00:15:42.280",{"_30":7631,"_16":3566,"_3567":7687},"00:15:45.280",{"_30":7639,"_16":3566,"_3567":7689},"00:15:48.240",{"_30":7647,"_16":3566,"_3567":7691},"00:15:51.000",{"_30":7655,"_16":3566,"_3567":7693},"00:15:53.840",{"_30":7663,"_16":3566,"_3567":7695},"00:15:57.200",{"_30":7671,"_16":3566,"_3567":7697},"00:16:00.070",{"_30":7679,"_16":3566,"_3567":7699},"00:16:02.750",{"_30":7701,"_16":265,"_266":7702,"_274":7708,"_276":277},"84eb0fc944d5",[7703],{"_30":7704,"_16":270,"_271":7705,"_146":7707},"4684f1732be2",[7706],"a87ddbf","And there's sort of a couple of ways around this.",[7709],{"_30":7706,"_16":3566,"_3567":7699},{"_30":7711,"_16":265,"_266":7712,"_274":7742,"_276":277},"ab3c34474753",[7713,7718,7721,7726,7729,7734,7737],{"_30":7714,"_16":270,"_271":7715,"_146":7717},"5b9c8d96a0a5",[7716],"1af7261","One is human labeling and the other is trying to use",{"_30":7719,"_16":270,"_271":7720,"_146":3542},"8fc08c5af484",[],{"_30":7722,"_16":270,"_271":7723,"_146":7725},"885a101ea231",[7724],"330ddbd","the models as judge and actually having some sort of",{"_30":7727,"_16":270,"_271":7728,"_146":3542},"e62386325dc9",[],{"_30":7730,"_16":270,"_271":7731,"_146":7733},"ec493dc1a3c3",[7732],"7bacee6","scalable— The snake eats its own tail and the",{"_30":7735,"_16":270,"_271":7736,"_146":3542},"e6fd1027016a",[],{"_30":7738,"_16":270,"_271":7739,"_146":7741},"67e35eded841",[7740],"bb2704f","model scores itself.",[7743,7745,7747,7749],{"_30":7716,"_16":3566,"_3567":7744},"00:16:05.790",{"_30":7724,"_16":3566,"_3567":7746},"00:16:08.790",{"_30":7732,"_16":3566,"_3567":7748},"00:16:11.670",{"_30":7740,"_16":3566,"_3567":7750},"00:16:14.670",{"_30":7752,"_16":265,"_266":7753,"_274":7767,"_276":277},"bb78a37e195b",[7754,7759,7762],{"_30":7755,"_16":270,"_271":7756,"_146":7758},"948c7af5f2e9",[7757],"7125fe8","And I see",{"_30":7760,"_16":270,"_271":7761,"_146":3542},"1256ea1f6978",[],{"_30":7763,"_16":270,"_271":7764,"_146":7766},"8fed83535bb8",[7765],"7fdbdac","someone in the audience smiling at me.",[7768,7770],{"_30":7757,"_16":3566,"_3567":7769},"00:16:15.670",{"_30":7765,"_16":3566,"_3567":7771},"00:16:17.590",{"_30":7773,"_16":265,"_266":7774,"_274":7796,"_276":277},"7e19a2a0dac5",[7775,7780,7783,7788,7791],{"_30":7776,"_16":270,"_271":7777,"_146":7779},"227fdc692e4a",[7778],"c3466c7","So the reason why",{"_30":7781,"_16":270,"_271":7782,"_146":3542},"445b4770144b",[],{"_30":7784,"_16":270,"_271":7785,"_146":7787},"7745dff72c41",[7786],"bfd01dd","that's not crazy is that oftentimes there's a sort of",{"_30":7789,"_16":270,"_271":7790,"_146":3542},"88179f4bfec4",[],{"_30":7792,"_16":270,"_271":7793,"_146":7795},"1709caa0259f",[7794],"7d78318","gap between what the models can generate and what they can evaluate.",[7797,7798,7800],{"_30":7778,"_16":3566,"_3567":7771},{"_30":7786,"_16":3566,"_3567":7799},"00:16:20.470",{"_30":7794,"_16":3566,"_3567":7801},"00:16:23.430",{"_30":7803,"_16":265,"_266":7804,"_274":7818,"_276":277},"a7fee332dde6",[7805,7810,7813],{"_30":7806,"_16":270,"_271":7807,"_146":7809},"abdb9070319b",[7808],"66ac250","And this has kind of been true throughout the history of machine",{"_30":7811,"_16":270,"_271":7812,"_146":3542},"b1cc049087fb",[],{"_30":7814,"_16":270,"_271":7815,"_146":7817},"7d9fab903058",[7816],"d159cc7","learning, generation is a harder task than evaluation.",[7819,7821],{"_30":7808,"_16":3566,"_3567":7820},"00:16:26.350",{"_30":7816,"_16":3566,"_3567":7822},"00:16:29.350",{"_30":7824,"_16":265,"_266":7825,"_274":7839,"_276":277},"722712b805d7",[7826,7831,7834],{"_30":7827,"_16":270,"_271":7828,"_146":7830},"238444ac9869",[7829],"df336fe","And so they'll often be very good classifiers of like is this thing",{"_30":7832,"_16":270,"_271":7833,"_146":3542},"578a3b632cb0",[],{"_30":7835,"_16":270,"_271":7836,"_146":7838},"10dc367703de",[7837],"b4e5234","present? But they might not be able to write the answer correctly.",[7840,7842],{"_30":7829,"_16":3566,"_3567":7841},"00:16:32.290",{"_30":7837,"_16":3566,"_3567":7843},"00:16:35.050",{"_30":7845,"_16":265,"_266":7846,"_274":7871,"_276":277},"80a18f94b5e8",[7847,7850,7855,7858,7863,7866],{"_30":7848,"_16":270,"_271":7849,"_146":4448},"9868d71f179d",[],{"_30":7851,"_16":270,"_271":7852,"_146":7854},"10ee7638f439",[7853],"7f9be43","nd then the subjectivity and the lack",{"_30":7856,"_16":270,"_271":7857,"_146":3542},"a1ec6fca7e7f",[],{"_30":7859,"_16":270,"_271":7860,"_146":7862},"d9f6f8374eff",[7861],"557d16c","of well defined metrics is similar. And then the last one I've put here",{"_30":7864,"_16":270,"_271":7865,"_146":3542},"f11a7e7e6813",[],{"_30":7867,"_16":270,"_271":7868,"_146":7870},"74eeab4d6952",[7869],"81c4c2b","is complex chains or agents.",[7872,7874,7876],{"_30":7853,"_16":3566,"_3567":7873},"00:16:38.240",{"_30":7861,"_16":3566,"_3567":7875},"00:16:41.210",{"_30":7869,"_16":3566,"_3567":7877},"00:16:44.050",{"_30":7879,"_16":265,"_266":7880,"_274":7902,"_276":277},"b00d23749a28",[7881,7886,7889,7894,7897],{"_30":7882,"_16":270,"_271":7883,"_146":7885},"0cd4f6e72804",[7884],"f4f922b","So if you imagine an agent that might have a multi-step interaction",{"_30":7887,"_16":270,"_271":7888,"_146":3542},"b50aff241ddd",[],{"_30":7890,"_16":270,"_271":7891,"_146":7893},"10f718d6a5a2",[7892],"3bac61d","in order to evaluate it and run it, you might actually have to mock",{"_30":7895,"_16":270,"_271":7896,"_146":3542},"858fa44c4f44",[],{"_30":7898,"_16":270,"_271":7899,"_146":7901},"5779ccb2ddf7",[7900],"b8a3560","steps that are coming from other systems.",[7903,7905,7907],{"_30":7884,"_16":3566,"_3567":7904},"00:16:46.850",{"_30":7892,"_16":3566,"_3567":7906},"00:16:49.610",{"_30":7900,"_16":3566,"_3567":7908},"00:16:52.530",{"_30":7910,"_16":265,"_266":7911,"_274":7941,"_276":277},"34a07e02e55e",[7912,7917,7920,7925,7928,7933,7936],{"_30":7913,"_16":270,"_271":7914,"_146":7916},"15a7dfcf390c",[7915],"f730a37","So if you imagine that you're",{"_30":7918,"_16":270,"_271":7919,"_146":3542},"9ef21a818c78",[],{"_30":7921,"_16":270,"_271":7922,"_146":7924},"64fa5e0ef68e",[7923],"1b028d8","evaluating an agent that will converse with your users,",{"_30":7926,"_16":270,"_271":7927,"_146":3542},"62c53019b78f",[],{"_30":7929,"_16":270,"_271":7930,"_146":7932},"d328daa1041f",[7931],"0f61615","then when they ask the user a question, you need something that's going to come",{"_30":7934,"_16":270,"_271":7935,"_146":3542},"d0f28830a415",[],{"_30":7937,"_16":270,"_271":7938,"_146":7940},"60e4cc833d1a",[7939],"a690ea2","in to fill that step.",[7942,7943,7945,7947],{"_30":7915,"_16":3566,"_3567":7908},{"_30":7923,"_16":3566,"_3567":7944},"00:16:55.530",{"_30":7931,"_16":3566,"_3567":7946},"00:16:58.340",{"_30":7939,"_16":3566,"_3567":7948},"00:17:01.260",{"_30":7950,"_16":265,"_266":7951,"_274":7973,"_276":277},"82b9146971ee",[7952,7957,7960,7965,7968],{"_30":7953,"_16":270,"_271":7954,"_146":7956},"6e0fe7a1958c",[7955],"6fbbd12","And so there's sort of an infrastructure",{"_30":7958,"_16":270,"_271":7959,"_146":3542},"1b8c2406c148",[],{"_30":7961,"_16":270,"_271":7962,"_146":7964},"63ac299d6620",[7963],"3abe56a","complexity as to how I mock all of the systems that my",{"_30":7966,"_16":270,"_271":7967,"_146":3542},"a11ea2756590",[],{"_30":7969,"_16":270,"_271":7970,"_146":7972},"6edb54ebadd2",[7971],"f5bdb1f","agent might interact with.",[7974,7975,7977],{"_30":7955,"_16":3566,"_3567":7948},{"_30":7963,"_16":3566,"_3567":7976},"00:17:03.980",{"_30":7971,"_16":3566,"_3567":7978},"00:17:06.860",{"_30":7980,"_16":265,"_266":7981,"_274":8003,"_276":277},"21c279f26503",[7982,7987,7990,7995,7998],{"_30":7983,"_16":270,"_271":7984,"_146":7986},"fb77122d2719",[7985],"eb90e4d","And also just like a",{"_30":7988,"_16":270,"_271":7989,"_146":3542},"0239fb975622",[],{"_30":7991,"_16":270,"_271":7992,"_146":7994},"56f45735f273",[7993],"0f5f0e0","combinatorial explosion of branching paths. And the way you can deal",{"_30":7996,"_16":270,"_271":7997,"_146":3542},"76c27a764f1a",[],{"_30":7999,"_16":270,"_271":8000,"_146":8002},"c6e7bc095364",[8001],"15650d4","with that is by just evaluating single turns.",[8004,8005,8007],{"_30":7985,"_16":3566,"_3567":7978},{"_30":7993,"_16":3566,"_3567":8006},"00:17:09.740",{"_30":8001,"_16":3566,"_3567":8008},"00:17:12.700",{"_30":8010,"_16":265,"_266":8011,"_274":8033,"_276":277},"80aa7bc5b5a4",[8012,8017,8020,8025,8028],{"_30":8013,"_16":270,"_271":8014,"_146":8016},"96d3feb1c495",[8015],"1a5d828","So you can break up the",{"_30":8018,"_16":270,"_271":8019,"_146":3542},"ece150ff491c",[],{"_30":8021,"_16":270,"_271":8022,"_146":8024},"880a56f931e2",[8023],"ee06957","agent trajectory and evaluate each turn",{"_30":8026,"_16":270,"_271":8027,"_146":3542},"56a91b2d1a9d",[],{"_30":8029,"_16":270,"_271":8030,"_146":8032},"4a411fccef60",[8031],"b9d6ce8","independently rather than trying to evaluate the whole.",[8034,8036,8038],{"_30":8015,"_16":3566,"_3567":8035},"00:17:15.260",{"_30":8023,"_16":3566,"_3567":8037},"00:17:17.860",{"_30":8031,"_16":3566,"_3567":8039},"00:17:20.580",{"_30":8041,"_16":265,"_266":8042,"_274":8064,"_276":277},"94d27d376d37",[8043,8048,8051,8056,8059],{"_30":8044,"_16":270,"_271":8045,"_146":8047},"30be4b9386d0",[8046],"d8195a9","I think that for a lot of subjective use",{"_30":8049,"_16":270,"_271":8050,"_146":3542},"f129f92bf7ed",[],{"_30":8052,"_16":270,"_271":8053,"_146":8055},"e496d74f61f3",[8054],"6eeca37","cases, end user feedback is the most priceless eval you can",{"_30":8057,"_16":270,"_271":8058,"_146":3542},"31335ddefad4",[],{"_30":8060,"_16":270,"_271":8061,"_146":8063},"eb87623d83b5",[8062],"8cede2f","get. But you only get it in production.",[8065,8067,8069],{"_30":8046,"_16":3566,"_3567":8066},"00:17:23.570",{"_30":8054,"_16":3566,"_3567":8068},"00:17:26.310",{"_30":8062,"_16":3566,"_3567":8070},"00:17:29.270",{"_30":8072,"_16":265,"_266":8073,"_274":8111,"_276":277},"e1e1aea891c3",[8074,8079,8082,8087,8090,8095,8098,8103,8106],{"_30":8075,"_16":270,"_271":8076,"_146":8078},"f44d621cafd7",[8077],"edebcaa","But once you are in production,",{"_30":8080,"_16":270,"_271":8081,"_146":3542},"16abf9cf49c5",[],{"_30":8083,"_16":270,"_271":8084,"_146":8086},"3478d2c60321",[8085],"87ac615","then GitHub Copilot, the way that they used to do this is they would",{"_30":8088,"_16":270,"_271":8089,"_146":3542},"463d2977cb8e",[],{"_30":8091,"_16":270,"_271":8092,"_146":8094},"17d945493308",[8093],"eb44df1","measure: Was this suggestion accepted? Did the code stick",{"_30":8096,"_16":270,"_271":8097,"_146":3542},"01698fb0500e",[],{"_30":8099,"_16":270,"_271":8100,"_146":8102},"68279270b5ab",[8101],"4d36d57","around or stick around substantially and",{"_30":8104,"_16":270,"_271":8105,"_146":3542},"c8fc3596db6a",[],{"_30":8107,"_16":270,"_271":8108,"_146":8110},"cd6203cadedc",[8109],"431507e","for how long?",[8112,8113,8115,8117,8119],{"_30":8077,"_16":3566,"_3567":8070},{"_30":8085,"_16":3566,"_3567":8114},"00:17:32.270",{"_30":8093,"_16":3566,"_3567":8116},"00:17:35.270",{"_30":8101,"_16":3566,"_3567":8118},"00:17:38.230",{"_30":8109,"_16":3566,"_3567":8120},"00:17:41.070",{"_30":8122,"_16":265,"_266":8123,"_274":8145,"_276":277},"de4785fb9d9a",[8124,8129,8132,8137,8140],{"_30":8125,"_16":270,"_271":8126,"_146":8128},"f883eec52779",[8127],"2ff6106","And other applications",{"_30":8130,"_16":270,"_271":8131,"_146":3542},"fc30da0a44fd",[],{"_30":8133,"_16":270,"_271":8134,"_146":8136},"48cd65be4abe",[8135],"5266574","also gather it with these thumbs up, thumbs down, they",{"_30":8138,"_16":270,"_271":8139,"_146":3542},"4e25d2a63701",[],{"_30":8141,"_16":270,"_271":8142,"_146":8144},"9d2104a4952f",[8143],"b6d0a1d","gather actions.",[8146,8147,8149],{"_30":8127,"_16":3566,"_3567":8120},{"_30":8135,"_16":3566,"_3567":8148},"00:17:44.030",{"_30":8143,"_16":3566,"_3567":8150},"00:17:46.750",{"_30":8152,"_16":265,"_266":8153,"_274":8157,"_276":277},"75ed7002ab61",[8154],{"_30":8155,"_16":270,"_271":8156,"_146":445},"f3c670faa006",[],[],{"_30":8159,"_16":265,"_266":8160,"_274":8165,"_276":349},"786c264cd7d7",[8161],{"_30":8162,"_16":270,"_271":8163,"_146":8164},"36b2e4478121",[],"Tooling & Operational Best Practices",[],{"_30":8167,"_16":265,"_266":8168,"_274":8172,"_276":277},"803abf668dfa",[8169],{"_30":8170,"_16":270,"_271":8171,"_146":445},"5b1f893c6a6a",[],[],{"_30":8174,"_16":265,"_266":8175,"_274":8197,"_276":277},"ba37cfcaa642",[8176,8181,8184,8189,8192],{"_30":8177,"_16":270,"_271":8178,"_146":8180},"4267851fb97d",[8179],"d22314e","Did the user proceed",{"_30":8182,"_16":270,"_271":8183,"_146":3542},"cba90fced732",[],{"_30":8185,"_16":270,"_271":8186,"_146":8188},"32366e43b83c",[8187],"7c5efe2","in the way that I expected? If I",{"_30":8190,"_16":270,"_271":8191,"_146":3542},"aff43109f381",[],{"_30":8193,"_16":270,"_271":8194,"_146":8196},"65d9b57d3700",[8195],"db16920","generated text for them, did they copy it? If I generated an email, did they send it?",[8198,8200,8202],{"_30":8179,"_16":3566,"_3567":8199},"00:17:47.750",{"_30":8187,"_16":3566,"_3567":8201},"00:17:49.670",{"_30":8195,"_16":3566,"_3567":8203},"00:17:52.310",{"_30":8205,"_16":265,"_266":8206,"_274":8228,"_276":277},"9497dad4deb7",[8207,8212,8215,8220,8223],{"_30":8208,"_16":270,"_271":8209,"_146":8211},"cc9d8b96e11f",[8210],"b458ca6","But it's hard to get",{"_30":8213,"_16":270,"_271":8214,"_146":3542},"0a3181acd238",[],{"_30":8216,"_16":270,"_271":8217,"_146":8219},"bdec7ebbaa3f",[8218],"e976fda","and it's often especially things like thumbs up, thumbs down, you're",{"_30":8221,"_16":270,"_271":8222,"_146":3542},"bda738f06c90",[],{"_30":8224,"_16":270,"_271":8225,"_146":8227},"458cafff63c3",[8226],"b97286e","getting a biased sample from the people who care the most.",[8229,8231,8233],{"_30":8210,"_16":3566,"_3567":8230},"00:17:55.610",{"_30":8218,"_16":3566,"_3567":8232},"00:17:58.540",{"_30":8226,"_16":3566,"_3567":8234},"00:18:01.530",{"_30":8236,"_16":265,"_266":8237,"_274":8251,"_276":277},"7c13e2ebf4e3",[8238,8243,8246],{"_30":8239,"_16":270,"_271":8240,"_146":8242},"12ededfc2f88",[8241],"8cd90ba","So it's a useful monitor of how well things are working, but it's not perfect",{"_30":8244,"_16":270,"_271":8245,"_146":3542},"538aa7cb13fa",[],{"_30":8247,"_16":270,"_271":8248,"_146":8250},"3bccab508716",[8249],"905b99e","on its own.",[8252,8254],{"_30":8241,"_16":3566,"_3567":8253},"00:18:04.490",{"_30":8249,"_16":3566,"_3567":8255},"00:18:07.450",{"_30":8257,"_16":265,"_266":8258,"_274":8288,"_276":277},"e8ac466dd424",[8259,8264,8267,8272,8275,8280,8283],{"_30":8260,"_16":270,"_271":8261,"_146":8263},"3ed193a6ab24",[8262],"044ac92","And so these are the types",{"_30":8265,"_16":270,"_271":8266,"_146":3542},"eaa3bd3e9e91",[],{"_30":8268,"_16":270,"_271":8269,"_146":8271},"c21df09e2427",[8270],"7856e0d","of feedback we tend to see",{"_30":8273,"_16":270,"_271":8274,"_146":3542},"723517843a13",[],{"_30":8276,"_16":270,"_271":8277,"_146":8279},"61d87b896b9c",[8278],"0fe99b4","people capturing. So corrections, actions and",{"_30":8281,"_16":270,"_271":8282,"_146":3542},"1a2da9081736",[],{"_30":8284,"_16":270,"_271":8285,"_146":8287},"cc981f3b1013",[8286],"19c4474","votes tend to be the main ones.",[8289,8290,8292,8294],{"_30":8262,"_16":3566,"_3567":8255},{"_30":8270,"_16":3566,"_3567":8291},"00:18:10.410",{"_30":8278,"_16":3566,"_3567":8293},"00:18:13.410",{"_30":8286,"_16":3566,"_3567":8295},"00:18:16.330",{"_30":8297,"_16":265,"_266":8298,"_274":8320,"_276":277},"5b55caeb159c",[8299,8304,8307,8312,8315],{"_30":8300,"_16":270,"_271":8301,"_146":8303},"51eb23f4cea0",[8302],"3140235","But it's sort of necessary in practice to have some kind of",{"_30":8305,"_16":270,"_271":8306,"_146":3542},"945a1aa2df41",[],{"_30":8308,"_16":270,"_271":8309,"_146":8311},"475c9fd767c5",[8310],"5a58e44","scorecard of different evals during development",{"_30":8313,"_16":270,"_271":8314,"_146":3542},"ec800956aad8",[],{"_30":8316,"_16":270,"_271":8317,"_146":8319},"4cdd02549844",[8318],"0bdeac0","that overall give you a picture of how well the system is working.",[8321,8323,8325],{"_30":8302,"_16":3566,"_3567":8322},"00:18:19.690",{"_30":8310,"_16":3566,"_3567":8324},"00:18:22.290",{"_30":8318,"_16":3566,"_3567":8326},"00:18:25.050",{"_30":8328,"_16":265,"_266":8329,"_274":8351,"_276":277},"e00bee8a81c2",[8330,8335,8338,8343,8346],{"_30":8331,"_16":270,"_271":8332,"_146":8334},"25790a82ada5",[8333],"b196778","And there's usually three types of eval",{"_30":8336,"_16":270,"_271":8337,"_146":3542},"4d64893aea78",[],{"_30":8339,"_16":270,"_271":8340,"_146":8342},"d1f2f873a67f",[8341],"4c75469","and these range from the cheapest being code, which is",{"_30":8344,"_16":270,"_271":8345,"_146":3542},"f61b43400185",[],{"_30":8347,"_16":270,"_271":8348,"_146":8350},"14284634a74c",[8349],"8172130","closest to your unit tests.",[8352,8354,8356],{"_30":8333,"_16":3566,"_3567":8353},"00:18:27.990",{"_30":8341,"_16":3566,"_3567":8355},"00:18:31.510",{"_30":8349,"_16":3566,"_3567":8357},"00:18:34.510",{"_30":8359,"_16":265,"_266":8360,"_274":8382,"_276":277},"1be6f3ed5656",[8361,8366,8369,8374,8377],{"_30":8362,"_16":270,"_271":8363,"_146":8365},"10a5b399f4f5",[8364],"c381f18","This is like: Is it valid",{"_30":8367,"_16":270,"_271":8368,"_146":3542},"bde1176a57aa",[],{"_30":8370,"_16":270,"_271":8371,"_146":8373},"ce7375317359",[8372],"6cf8b02","JSON? Is the length of my summary actually shorter than",{"_30":8375,"_16":270,"_271":8376,"_146":3542},"318349f00bf4",[],{"_30":8378,"_16":270,"_271":8379,"_146":8381},"d56e22362ffe",[8380],"bd9b214","the thing I'm trying to summarize?",[8383,8384,8386],{"_30":8364,"_16":3566,"_3567":8357},{"_30":8372,"_16":3566,"_3567":8385},"00:18:37.310",{"_30":8380,"_16":3566,"_3567":8387},"00:18:40.230",{"_30":8389,"_16":265,"_266":8390,"_274":8412,"_276":277},"361692b1bc48",[8391,8396,8399,8404,8407],{"_30":8392,"_16":270,"_271":8393,"_146":8395},"c3f82b7ec54d",[8394],"48c842e","Not an eval you tend to need",{"_30":8397,"_16":270,"_271":8398,"_146":3542},"a7548ab8db01",[],{"_30":8400,"_16":270,"_271":8401,"_146":8403},"9b0f845b88da",[8402],"4b6bb2e","anymore because the models have gotten smart, but was it one that",{"_30":8405,"_16":270,"_271":8406,"_146":3542},"fceecb836f2d",[],{"_30":8408,"_16":270,"_271":8409,"_146":8411},"82b5bf4f08fc",[8410],"20d0445","a year and a half ago was not guaranteed.",[8413,8414,8416],{"_30":8394,"_16":3566,"_3567":8387},{"_30":8402,"_16":3566,"_3567":8415},"00:18:42.990",{"_30":8410,"_16":3566,"_3567":8417},"00:18:45.750",{"_30":8419,"_16":265,"_266":8420,"_274":8442,"_276":277},"4c5cd47f42cf",[8421,8426,8429,8434,8437],{"_30":8422,"_16":270,"_271":8423,"_146":8425},"2f4cdb169a0d",[8424],"79034be","And then once you go past the code ones, when you're trying to",{"_30":8427,"_16":270,"_271":8428,"_146":3542},"5e02a22a096e",[],{"_30":8430,"_16":270,"_271":8431,"_146":8433},"5a26e3c476b0",[8432],"89b0d78","include more subjective things like tone of voice",{"_30":8435,"_16":270,"_271":8436,"_146":3542},"066d703a8efb",[],{"_30":8438,"_16":270,"_271":8439,"_146":8441},"94a83386abb8",[8440],"4d46b78","or even things like hallucination checking—",[8443,8445,8447],{"_30":8424,"_16":3566,"_3567":8444},"00:18:49.670",{"_30":8432,"_16":3566,"_3567":8446},"00:18:52.590",{"_30":8440,"_16":3566,"_3567":8448},"00:18:55.430",{"_30":8450,"_16":265,"_266":8451,"_274":8473,"_276":277},"18380fa7b909",[8452,8457,8460,8465,8468],{"_30":8453,"_16":270,"_271":8454,"_146":8456},"c7ac1c429388",[8455],"15a594a","Like, I have a retrieved context, I have the model's",{"_30":8458,"_16":270,"_271":8459,"_146":3542},"78febe1ad3cb",[],{"_30":8461,"_16":270,"_271":8462,"_146":8464},"6bf3936690e4",[8463],"4e9d8f6","output. Does the model's output follow logically from the",{"_30":8466,"_16":270,"_271":8467,"_146":3542},"12f5e0e551fb",[],{"_30":8469,"_16":270,"_271":8470,"_146":8472},"ce2e487df9ec",[8471],"0d3fcf2","retrieved context? You can create a binary classifier for that.",[8474,8476,8478],{"_30":8455,"_16":3566,"_3567":8475},"00:18:58.050",{"_30":8463,"_16":3566,"_3567":8477},"00:19:01.050",{"_30":8471,"_16":3566,"_3567":8479},"00:19:03.890",{"_30":8481,"_16":265,"_266":8482,"_274":8533,"_276":2466},"10a48d2bc59d",[8483,8488,8491,8496,8499,8504,8507,8512,8515,8520,8525,8528],{"_30":8484,"_16":270,"_271":8485,"_146":8487},"d5a15f141d7f",[8486],"490c45c","Most of the time, the way that people actually do this when they succeed",{"_30":8489,"_16":270,"_271":8490,"_146":3542},"4b8f8b875846",[],{"_30":8492,"_16":270,"_271":8493,"_146":8495},"4da9610073fe",[8494],"aa18ef1","is that they will create a data set, have a",{"_30":8497,"_16":270,"_271":8498,"_146":3542},"ff0ed5391856",[],{"_30":8500,"_16":270,"_271":8501,"_146":8503},"920612434cc0",[8502],"49d7415","human expert, the most trusted expert, do the eval",{"_30":8505,"_16":270,"_271":8506,"_146":3542},"ddd236a691c5",[],{"_30":8508,"_16":270,"_271":8509,"_146":8511},"f3da60bd5375",[8510],"7050ad2","and provide critiques or explanations for why they agree",{"_30":8513,"_16":270,"_271":8514,"_146":3542},"4e625c9edc62",[],{"_30":8516,"_16":270,"_271":8517,"_146":8519},"2ac4b3897482",[8518],"489812a","or disagree with the model's output.",{"_30":8521,"_16":270,"_271":8522,"_146":8524},"974db9335328",[8523],"8bd3455"," And then they'll use that",{"_30":8526,"_16":270,"_271":8527,"_146":3542},"bd73bf42c684",[],{"_30":8529,"_16":270,"_271":8530,"_146":8532},"44fb9d740cdf",[8531],"a31bf47","to write the prompt for the LLM as judge.",[8534,8536,8538,8540,8542,8544,8545],{"_30":8486,"_16":3566,"_3567":8535},"00:19:08.010",{"_30":8494,"_16":3566,"_3567":8537},"00:19:11.250",{"_30":8502,"_16":3566,"_3567":8539},"00:19:14.010",{"_30":8510,"_16":3566,"_3567":8541},"00:19:17.250",{"_30":8518,"_16":3566,"_3567":8543},"00:19:20.130",{"_30":8523,"_16":3566,"_3567":8543},{"_30":8531,"_16":3566,"_3567":8546},"00:19:23.050",{"_30":8548,"_16":265,"_266":8549,"_274":8571,"_276":277},"88c5d482e21b",[8550,8555,8558,8563,8566],{"_30":8551,"_16":270,"_271":8552,"_146":8554},"3be766d5cb1b",[8553],"b3dfec7","And they'll",{"_30":8556,"_16":270,"_271":8557,"_146":3542},"91b4763d050e",[],{"_30":8559,"_16":270,"_271":8560,"_146":8562},"ff2009b8651e",[8561],"8c4b370","iterate this process until the LLM judge and",{"_30":8564,"_16":270,"_271":8565,"_146":3542},"f3eb4ad0abf8",[],{"_30":8567,"_16":270,"_271":8568,"_146":8570},"72bea630a921",[8569],"d0715af","the human have high correlation.",[8572,8573,8575],{"_30":8553,"_16":3566,"_3567":8546},{"_30":8561,"_16":3566,"_3567":8574},"00:19:25.920",{"_30":8569,"_16":3566,"_3567":8576},"00:19:28.840",{"_30":8578,"_16":265,"_266":8579,"_274":8609,"_276":277},"1cfae64b08a3",[8580,8585,8588,8593,8596,8601,8604],{"_30":8581,"_16":270,"_271":8582,"_146":8584},"9b608218aa62",[8583],"5681877","And at that point you",{"_30":8586,"_16":270,"_271":8587,"_146":3542},"68b7eb6c4eaf",[],{"_30":8589,"_16":270,"_271":8590,"_146":8592},"007b4b9eb161",[8591],"6877125","trust that the LLM judge is a good proxy,",{"_30":8594,"_16":270,"_271":8595,"_146":3542},"d27eb1d22ce2",[],{"_30":8597,"_16":270,"_271":8598,"_146":8600},"e5870f3682fc",[8599],"39431d4","not a perfect proxy, but a cheap and scalable one compared",{"_30":8602,"_16":270,"_271":8603,"_146":3542},"34f8755f5c0a",[],{"_30":8605,"_16":270,"_271":8606,"_146":8608},"a7bdc6e4c16a",[8607],"52308d0","to the human for doing evaluation.",[8610,8611,8613,8615],{"_30":8583,"_16":3566,"_3567":8576},{"_30":8591,"_16":3566,"_3567":8612},"00:19:31.760",{"_30":8599,"_16":3566,"_3567":8614},"00:19:34.840",{"_30":8607,"_16":3566,"_3567":8616},"00:19:37.760",{"_30":8618,"_16":265,"_266":8619,"_274":8649,"_276":277},"e9cd8fc70355",[8620,8625,8628,8633,8636,8641,8644],{"_30":8621,"_16":270,"_271":8622,"_146":8624},"0907875b0774",[8623],"7e0b390","So actually where human evaluation",{"_30":8626,"_16":270,"_271":8627,"_146":3542},"20e0050abe31",[],{"_30":8629,"_16":270,"_271":8630,"_146":8632},"b90cd4479e25",[8631],"93cad10","ends up being used most, usually in my experience now, is",{"_30":8634,"_16":270,"_271":8635,"_146":3542},"0fe76eb3947e",[],{"_30":8637,"_16":270,"_271":8638,"_146":8640},"26ef678baf2c",[8639],"a033d4c","to get the LLM evaluators tuned, so",{"_30":8642,"_16":270,"_271":8643,"_146":3542},"184501b8e5e6",[],{"_30":8645,"_16":270,"_271":8646,"_146":8648},"f81c4c331191",[8647],"c71ff71","that you can trust them.",[8650,8651,8653,8655],{"_30":8623,"_16":3566,"_3567":8616},{"_30":8631,"_16":3566,"_3567":8652},"00:19:40.720",{"_30":8639,"_16":3566,"_3567":8654},"00:19:43.680",{"_30":8647,"_16":3566,"_3567":8656},"00:19:46.680",{"_30":8658,"_16":265,"_266":8659,"_274":8689,"_276":277},"5b60da340963",[8660,8665,8668,8673,8676,8681,8684],{"_30":8661,"_16":270,"_271":8662,"_146":8664},"fd323743b12a",[8663],"3bf2b98","And then also,",{"_30":8666,"_16":270,"_271":8667,"_146":3542},"047128819463",[],{"_30":8669,"_16":270,"_271":8670,"_146":8672},"1b27b8e321d7",[8671],"91464da","people do it on batches of samples. So",{"_30":8674,"_16":270,"_271":8675,"_146":3542},"a749a43bbe3f",[],{"_30":8677,"_16":270,"_271":8678,"_146":8680},"c55872bdb240",[8679],"535378c","they will take on some interval, a batch of production logs and",{"_30":8682,"_16":270,"_271":8683,"_146":3542},"923434c4e722",[],{"_30":8685,"_16":270,"_271":8686,"_146":8688},"68790211e854",[8687],"aa657be","get them manually annotated as a kind of spot check of quality.",[8690,8691,8693,8695],{"_30":8663,"_16":3566,"_3567":8656},{"_30":8671,"_16":3566,"_3567":8692},"00:19:49.490",{"_30":8679,"_16":3566,"_3567":8694},"00:19:52.400",{"_30":8687,"_16":3566,"_3567":8696},"00:19:55.360",{"_30":8698,"_16":265,"_266":8699,"_274":8703,"_276":277},"9d55e5f9aa32",[8700],{"_30":8701,"_16":270,"_271":8702,"_146":445},"01f02e3ca205",[],[],{"_30":8705,"_16":265,"_266":8706,"_274":8711,"_276":349},"b217972bf352",[8707],{"_30":8708,"_16":270,"_271":8709,"_146":8710},"a0370b5c3c6f",[],"Key Pitfalls & Practical Recommendations",[],{"_30":8713,"_16":265,"_266":8714,"_274":8718,"_276":277},"4ad5814a6fbc",[8715],{"_30":8716,"_16":270,"_271":8717,"_146":445},"5cdd2807bf4c",[],[],{"_30":8720,"_16":265,"_266":8721,"_274":8751,"_276":277},"b056f9341632",[8722,8727,8730,8735,8738,8743,8746],{"_30":8723,"_16":270,"_271":8724,"_146":8726},"6a3e44e2b5a9",[8725],"8a719e3","And so the last thing I want to talk about is",{"_30":8728,"_16":270,"_271":8729,"_146":3542},"6e08eb2db00a",[],{"_30":8731,"_16":270,"_271":8732,"_146":8734},"1b1df72d3147",[8733],"1d3c492","the types of tooling that we've seen people building",{"_30":8736,"_16":270,"_271":8737,"_146":3542},"be4bccb48fb0",[],{"_30":8739,"_16":270,"_271":8740,"_146":8742},"d2b7cd137080",[8741],"59c41ee","and using to kind of get to success here and what are some of the",{"_30":8744,"_16":270,"_271":8745,"_146":3542},"80d5c1e48ece",[],{"_30":8747,"_16":270,"_271":8748,"_146":8750},"02cce2585138",[8749],"b984ba6","workflows that are different than before.",[8752,8754,8756,8758],{"_30":8725,"_16":3566,"_3567":8753},"00:19:58.930",{"_30":8733,"_16":3566,"_3567":8755},"00:20:01.750",{"_30":8741,"_16":3566,"_3567":8757},"00:20:04.660",{"_30":8749,"_16":3566,"_3567":8759},"00:20:07.500",{"_30":8761,"_16":265,"_266":8762,"_274":8792,"_276":2466},"d038fc3b41f6",[8763,8768,8771,8776,8779,8784,8787],{"_30":8764,"_16":270,"_271":8765,"_146":8767},"8c3fc6ccc3f8",[8766],"8d504ff","So compared to traditional software development,",{"_30":8769,"_16":270,"_271":8770,"_146":3542},"55bfa2334130",[],{"_30":8772,"_16":270,"_271":8773,"_146":8775},"454b73907fe9",[8774],"46514e8","there's a few things that have changed. One is you now have all these natural language artifacts,",{"_30":8777,"_16":270,"_271":8778,"_146":3542},"a2ff2aac71da",[],{"_30":8780,"_16":270,"_271":8781,"_146":8783},"f6c9ecd655a8",[8782],"59e9248","the prompts themselves, that are sitting around",{"_30":8785,"_16":270,"_271":8786,"_146":3542},"e755bb2f56a1",[],{"_30":8788,"_16":270,"_271":8789,"_146":8791},"7af0c56b58aa",[8790],"5cc8a19","that are somewhere between data and code.",[8793,8795,8797,8799],{"_30":8766,"_16":3566,"_3567":8794},"00:20:09.500",{"_30":8774,"_16":3566,"_3567":8796},"00:20:13.219",{"_30":8782,"_16":3566,"_3567":8798},"00:20:16.060",{"_30":8790,"_16":3566,"_3567":8800},"00:20:18.980",{"_30":8802,"_16":265,"_266":8803,"_274":8817,"_276":277},"7c0e1d9997cf",[8804,8809,8812],{"_30":8805,"_16":270,"_271":8806,"_146":8808},"b3784792deb2",[8807],"12d2e59","They're like configuration, right? They do have",{"_30":8810,"_16":270,"_271":8811,"_146":3542},"f5ab5cf5cd4c",[],{"_30":8813,"_16":270,"_271":8814,"_146":8816},"5ee2e6f77a52",[8815],"bdbe8cf","instructions in them.",[8818,8820],{"_30":8807,"_16":3566,"_3567":8819},"00:20:21.660",{"_30":8815,"_16":3566,"_3567":8821},"00:20:24.100",{"_30":8823,"_16":265,"_266":8824,"_274":8838,"_276":277},"c7dbab4dd36c",[8825,8830,8833],{"_30":8826,"_16":270,"_271":8827,"_146":8829},"63b5f473efe8",[8828],"1c8b9dd","So in that sense they feel",{"_30":8831,"_16":270,"_271":8832,"_146":3542},"966661a0ee43",[],{"_30":8834,"_16":270,"_271":8835,"_146":8837},"9dbe8fb9899d",[8836],"8bf43cf","like code, but they're really more like data.",[8839,8840],{"_30":8828,"_16":3566,"_3567":8821},{"_30":8836,"_16":3566,"_3567":8841},"00:20:27.060",{"_30":8843,"_16":265,"_266":8844,"_274":8858,"_276":277},"cf1d84edfab5",[8845,8850,8853],{"_30":8846,"_16":270,"_271":8847,"_146":8849},"e029adde74f5",[8848],"5e89102","And for most applications you want these things to be accessible",{"_30":8851,"_16":270,"_271":8852,"_146":3542},"7a6277b36ebb",[],{"_30":8854,"_16":270,"_271":8855,"_146":8857},"6ecf165d812a",[8856],"db7350e","to non-technical people.",[8859,8861],{"_30":8848,"_16":3566,"_3567":8860},"00:20:30.420",{"_30":8856,"_16":3566,"_3567":8862},"00:20:33.300",{"_30":8864,"_16":265,"_266":8865,"_274":8887,"_276":277},"b5bdf08f0484",[8866,8871,8874,8879,8882],{"_30":8867,"_16":270,"_271":8868,"_146":8870},"2b9316329eb5",[8869],"0b2386e","You want them to be able to iterate on them,",{"_30":8872,"_16":270,"_271":8873,"_146":3542},"c380777142a4",[],{"_30":8875,"_16":270,"_271":8876,"_146":8878},"d835d4f709fb",[8877],"2b2619f","but you also want to have the same rigor of",{"_30":8880,"_16":270,"_271":8881,"_146":3542},"015e671d392e",[],{"_30":8883,"_16":270,"_271":8884,"_146":8886},"266c06d541f2",[8885],"f417cdb","versioning and tracking that you have for normal code.",[8888,8889,8891],{"_30":8869,"_16":3566,"_3567":8862},{"_30":8877,"_16":3566,"_3567":8890},"00:20:36.300",{"_30":8885,"_16":3566,"_3567":8892},"00:20:39.020",{"_30":8894,"_16":265,"_266":8895,"_274":8925,"_276":277},"e8218fb331ec",[8896,8901,8904,8909,8912,8917,8920],{"_30":8897,"_16":270,"_271":8898,"_146":8900},"67a8e786c99e",[8899],"1693210","And so you need",{"_30":8902,"_16":270,"_271":8903,"_146":3542},"4fc3d5d345bc",[],{"_30":8905,"_16":270,"_271":8906,"_146":8908},"2be8e518dee4",[8907],"6cf46b0","some solution for that. And we've seen",{"_30":8910,"_16":270,"_271":8911,"_146":3542},"433395d04430",[],{"_30":8913,"_16":270,"_271":8914,"_146":8916},"f490a0965294",[8915],"9c224e9","a variety of different approaches, but that's one very common problem that",{"_30":8918,"_16":270,"_271":8919,"_146":3542},"c87566f6ca9c",[],{"_30":8921,"_16":270,"_271":8922,"_146":8924},"e81dc51810ad",[8923],"08f0259","comes up that you're going to want to think about.",[8926,8927,8929,8931],{"_30":8899,"_16":3566,"_3567":8892},{"_30":8907,"_16":3566,"_3567":8928},"00:20:41.980",{"_30":8915,"_16":3566,"_3567":8930},"00:20:44.940",{"_30":8923,"_16":3566,"_3567":8932},"00:20:47.860",{"_30":8934,"_16":265,"_266":8935,"_274":8965,"_276":277},"325bd5c9b3d8",[8936,8941,8944,8949,8952,8957,8960],{"_30":8937,"_16":270,"_271":8938,"_146":8940},"f21f13492346",[8939],"948a9d5","Another thing",{"_30":8942,"_16":270,"_271":8943,"_146":3542},"e2a4839e5c1d",[],{"_30":8945,"_16":270,"_271":8946,"_146":8948},"1cff89efa054",[8947],"2cd3d0e","is making sure that you have a good system for doing",{"_30":8950,"_16":270,"_271":8951,"_146":3542},"63fba4128779",[],{"_30":8953,"_16":270,"_271":8954,"_146":8956},"948a9899e2fd",[8955],"efd2bba","evaluation, both during prototyping in development",{"_30":8958,"_16":270,"_271":8959,"_146":3542},"70b791577e7c",[],{"_30":8961,"_16":270,"_271":8962,"_146":8964},"667f9266be65",[8963],"af9ac06","and post production.",[8966,8967,8969,8971],{"_30":8939,"_16":3566,"_3567":8932},{"_30":8947,"_16":3566,"_3567":8968},"00:20:50.780",{"_30":8955,"_16":3566,"_3567":8970},"00:20:53.420",{"_30":8963,"_16":3566,"_3567":8972},"00:20:56.310",{"_30":8974,"_16":265,"_266":8975,"_274":8997,"_276":277},"cdd2a3c1ab8e",[8976,8981,8984,8989,8992],{"_30":8977,"_16":270,"_271":8978,"_146":8980},"092d1d959c5e",[8979],"50064f0","I think if there's one thing",{"_30":8982,"_16":270,"_271":8983,"_146":3542},"12b0d58d7df5",[],{"_30":8985,"_16":270,"_271":8986,"_146":8988},"ed768a416446",[8987],"a869463","you remember, if you forget everything else I say, I think if you",{"_30":8990,"_16":270,"_271":8991,"_146":3542},"81df3ebd3563",[],{"_30":8993,"_16":270,"_271":8994,"_146":8996},"c6589041dba3",[8995],"0bbb2bf","don't build good evals at the start, it's likely your project will fail.",[8998,8999,9001],{"_30":8979,"_16":3566,"_3567":8972},{"_30":8987,"_16":3566,"_3567":9000},"00:20:59.230",{"_30":8995,"_16":3566,"_3567":9002},"00:21:01.990",{"_30":9004,"_16":265,"_266":9005,"_274":9019,"_276":277},"bc6bd01c33ab",[9006,9011,9014],{"_30":9007,"_16":270,"_271":9008,"_146":9010},"71e0c1817da3",[9009],"76f2c95","And the evals don't have to be super complicated.",{"_30":9012,"_16":270,"_271":9013,"_146":3542},"a0054e4c8c6b",[],{"_30":9015,"_16":270,"_271":9016,"_146":9018},"f7c3c03ca631",[9017],"84ef45a","A lot of people are doing this with spreadsheets and having great success.",[9020,9022],{"_30":9009,"_16":3566,"_3567":9021},"00:21:04.830",{"_30":9017,"_16":3566,"_3567":9023},"00:21:07.990",{"_30":9025,"_16":265,"_266":9026,"_274":9064,"_276":277},"f232db438156",[9027,9032,9035,9040,9043,9048,9051,9056,9059],{"_30":9028,"_16":270,"_271":9029,"_146":9031},"5a124dd77873",[9030],"ff281d4","So it's not necessary that this becomes complicated. I would",{"_30":9033,"_16":270,"_271":9034,"_146":3542},"0b76c8993d0d",[],{"_30":9036,"_16":270,"_271":9037,"_146":9039},"cf423724eb26",[9038],"0f13ffa","only reach for complicated tools or more powerful",{"_30":9041,"_16":270,"_271":9042,"_146":3542},"dec8691afbdd",[],{"_30":9044,"_16":270,"_271":9045,"_146":9047},"64e8555894af",[9046],"14bd974","tools if and when your simple spreadsheet methods aren't",{"_30":9049,"_16":270,"_271":9050,"_146":3542},"2f2ac1126803",[],{"_30":9052,"_16":270,"_271":9053,"_146":9055},"44d5ae98d3ef",[9054],"27fb83d","working or the simplest things to phase stops to work,",{"_30":9057,"_16":270,"_271":9058,"_146":3542},"8a6102a0292c",[],{"_30":9060,"_16":270,"_271":9061,"_146":9063},"b0cc484eaaf6",[9062],"51b9bca","but you definitely need them.",[9065,9067,9069,9071,9073],{"_30":9030,"_16":3566,"_3567":9066},"00:21:11.160",{"_30":9038,"_16":3566,"_3567":9068},"00:21:14.110",{"_30":9046,"_16":3566,"_3567":9070},"00:21:16.950",{"_30":9054,"_16":3566,"_3567":9072},"00:21:19.910",{"_30":9062,"_16":3566,"_3567":9074},"00:21:23.090",{"_30":9076,"_16":265,"_266":9077,"_274":9107,"_276":277},"2df2522584f5",[9078,9083,9086,9091,9094,9099,9102],{"_30":9079,"_16":270,"_271":9080,"_146":9082},"ce9284214de2",[9081],"aff19fa","And then lastly, I think you definitely want",{"_30":9084,"_16":270,"_271":9085,"_146":3542},"d3af77ff7d19",[],{"_30":9087,"_16":270,"_271":9088,"_146":9090},"5f548a32e781",[9089],"f681187","comprehensive logging.",{"_30":9092,"_16":270,"_271":9093,"_146":3542},"6e52df6f6fd9",[],{"_30":9095,"_16":270,"_271":9096,"_146":9098},"2c5490f3a045",[9097],"61a5707","There is so much that you",{"_30":9100,"_16":270,"_271":9101,"_146":3542},"d1184ce12ea6",[],{"_30":9103,"_16":270,"_271":9104,"_146":9106},"5cdbec70b694",[9105],"52e36b7","gain from just looking at your data.",[9108,9109,9111,9113],{"_30":9081,"_16":3566,"_3567":9074},{"_30":9089,"_16":3566,"_3567":9110},"00:21:26.090",{"_30":9097,"_16":3566,"_3567":9112},"00:21:29.040",{"_30":9105,"_16":3566,"_3567":9114},"00:21:31.770",{"_30":9116,"_16":265,"_266":9117,"_274":9166,"_276":277},"beabb3e70e03",[9118,9123,9126,9131,9134,9139,9142,9147,9150,9155,9158,9163],{"_30":9119,"_16":270,"_271":9120,"_146":9122},"52be794f0361",[9121],"383c733","And again maybe this is",{"_30":9124,"_16":270,"_271":9125,"_146":3542},"467d15d22e35",[],{"_30":9127,"_16":270,"_271":9128,"_146":9130},"01ed3ef909a8",[9129],"18f8c5a","something that if you come from a data science or a machine learning background is",{"_30":9132,"_16":270,"_271":9133,"_146":3542},"92e8fbd38db6",[],{"_30":9135,"_16":270,"_271":9136,"_146":9138},"676cccdf147c",[9137],"a3f6497","obvious, but if you come from a more traditional software engineering",{"_30":9140,"_16":270,"_271":9141,"_146":3542},"46d444e921ea",[],{"_30":9143,"_16":270,"_271":9144,"_146":9146},"2997cc5544ab",[9145],"4c2d3c0","background, it's not necessarily the case that just staring at",{"_30":9148,"_16":270,"_271":9149,"_146":3542},"64795ef07fad",[],{"_30":9151,"_16":270,"_271":9152,"_146":9154},"c84439b3a43c",[9153],"f8df57d","logs is a super, super high value activity",{"_30":9156,"_16":270,"_271":9157,"_146":3542},"ba5c1ad633dd",[],{"_30":9159,"_16":270,"_271":9160,"_146":9162},"97bd2fe0bb4b",[9161],"ce0be79","or not as high value.",{"_30":9164,"_16":270,"_271":9165,"_146":2702},"04af975e6c12",[],[9167,9168,9170,9172,9174,9176],{"_30":9121,"_16":3566,"_3567":9114},{"_30":9129,"_16":3566,"_3567":9169},"00:21:34.690",{"_30":9137,"_16":3566,"_3567":9171},"00:21:37.490",{"_30":9145,"_16":3566,"_3567":9173},"00:21:40.330",{"_30":9153,"_16":3566,"_3567":9175},"00:21:43.090",{"_30":9161,"_16":3566,"_3567":9177},"00:21:46.050",{"_30":9179,"_16":265,"_266":9180,"_274":9210,"_276":277},"1ff40d17d41a",[9181,9186,9189,9194,9197,9202,9205],{"_30":9182,"_16":270,"_271":9183,"_146":9185},"0ea8f6e75b79",[9184],"2381f93","But actually just",{"_30":9187,"_16":270,"_271":9188,"_146":3542},"fc4baadc768d",[],{"_30":9190,"_16":270,"_271":9191,"_146":9193},"275c927f0c78",[9192],"7f68faa","stepping through and looking at what the model did, why did it",{"_30":9195,"_16":270,"_271":9196,"_146":3542},"c0539d84bd1e",[],{"_30":9198,"_16":270,"_271":9199,"_146":9201},"15e7b3a4fd3e",[9200],"570768c","succeed, why did it fail, in the cases that it went wrong,",{"_30":9203,"_16":270,"_271":9204,"_146":3542},"55bf20dd5b5f",[],{"_30":9206,"_16":270,"_271":9207,"_146":9209},"07c0b9ecdbb2",[9208],"abca94e","why did it go wrong?",[9211,9212,9214,9216],{"_30":9184,"_16":3566,"_3567":9177},{"_30":9192,"_16":3566,"_3567":9213},"00:21:48.900",{"_30":9200,"_16":3566,"_3567":9215},"00:21:51.820",{"_30":9208,"_16":3566,"_3567":9217},"00:21:54.700",{"_30":9219,"_16":265,"_266":9220,"_274":9250,"_276":277},"637e61023c77",[9221,9226,9229,9234,9237,9242,9245],{"_30":9222,"_16":270,"_271":9223,"_146":9225},"53ff968d8407",[9224],"bf6803a","That tends to",{"_30":9227,"_16":270,"_271":9228,"_146":3542},"1c82baff5b66",[],{"_30":9230,"_16":270,"_271":9231,"_146":9233},"b31195242156",[9232],"28231d7","be an extremely high value activity that then feeds back",{"_30":9235,"_16":270,"_271":9236,"_146":3542},"521c4fe70857",[],{"_30":9238,"_16":270,"_271":9239,"_146":9241},"a4a608a60142",[9240],"6e3509c","into the round of improvement and",{"_30":9243,"_16":270,"_271":9244,"_146":3542},"0dccfea1852d",[],{"_30":9246,"_16":270,"_271":9247,"_146":9249},"c37e7f960669",[9248],"2c18905","iteration.",[9251,9252,9254,9256],{"_30":9224,"_16":3566,"_3567":9217},{"_30":9232,"_16":3566,"_3567":9253},"00:21:57.660",{"_30":9240,"_16":3566,"_3567":9255},"00:22:00.460",{"_30":9248,"_16":3566,"_3567":9257},"00:22:02.980",{"_30":9259,"_16":265,"_266":9260,"_274":9274,"_276":277},"213e63229698",[9261,9266,9269],{"_30":9262,"_16":270,"_271":9263,"_146":9265},"7645a27f4db8",[9264],"2b3f7d9","There are different options for this. Obviously we are one of them. But there",{"_30":9267,"_16":270,"_271":9268,"_146":3542},"2b5b6c83105e",[],{"_30":9270,"_16":270,"_271":9271,"_146":9273},"d9741993c5b4",[9272],"d970cf5","are open source tools.",[9275,9277],{"_30":9264,"_16":3566,"_3567":9276},"00:22:05.740",{"_30":9272,"_16":3566,"_3567":9278},"00:22:08.740",{"_30":9280,"_16":265,"_266":9281,"_274":9303,"_276":277},"d9857ea5b7cd",[9282,9287,9290,9295,9298],{"_30":9283,"_16":270,"_271":9284,"_146":9286},"f3559a0964d8",[9285],"36268a9","So Ironclad built this tool called Rivet",{"_30":9288,"_16":270,"_271":9289,"_146":3542},"7be078457890",[],{"_30":9291,"_16":270,"_271":9292,"_146":9294},"f6a8cca6e27b",[9293],"5602597","that gives you a visual builder for",{"_30":9296,"_16":270,"_271":9297,"_146":3542},"b64b1b7e64a3",[],{"_30":9299,"_16":270,"_271":9300,"_146":9302},"b045f31cdcf9",[9301],"e233252","this and there's another one out there called Vellum that does that.",[9304,9305,9307],{"_30":9285,"_16":3566,"_3567":9278},{"_30":9293,"_16":3566,"_3567":9306},"00:22:11.740",{"_30":9301,"_16":3566,"_3567":9308},"00:22:14.490",{"_30":9310,"_16":265,"_266":9311,"_274":9341,"_276":277},"95d654b368c2",[9312,9317,9320,9325,9328,9333,9336],{"_30":9313,"_16":270,"_271":9314,"_146":9316},"53c2a9428d76",[9315],"1774c19","And so there's a variety of tools and I think",{"_30":9318,"_16":270,"_271":9319,"_146":3542},"93d4fcc389a4",[],{"_30":9321,"_16":270,"_271":9322,"_146":9324},"86cdf15caa3c",[9323],"7d5c4a5","that it is helpful to get some form of LLM",{"_30":9326,"_16":270,"_271":9327,"_146":3542},"1c2fc4cc4fdc",[],{"_30":9329,"_16":270,"_271":9330,"_146":9332},"6be706c0af6b",[9331],"1b1e272","Ops platform in, but only if you've got the other",{"_30":9334,"_16":270,"_271":9335,"_146":3542},"063635e6e39c",[],{"_30":9337,"_16":270,"_271":9338,"_146":9340},"1d52a1652e56",[9339],"9fbb594","things in place and you have them right.",[9342,9344,9346,9348],{"_30":9315,"_16":3566,"_3567":9343},"00:22:17.410",{"_30":9323,"_16":3566,"_3567":9345},"00:22:20.410",{"_30":9331,"_16":3566,"_3567":9347},"00:22:23.050",{"_30":9339,"_16":3566,"_3567":9349},"00:22:25.890",{"_30":9351,"_16":265,"_266":9352,"_274":9374,"_276":277},"f4c8ee28bd8c",[9353,9358,9361,9366,9369],{"_30":9354,"_16":270,"_271":9355,"_146":9357},"c5cf616de616",[9356],"6ea86c3","It's not a silver",{"_30":9359,"_16":270,"_271":9360,"_146":3542},"6fc7be2deaf8",[],{"_30":9362,"_16":270,"_271":9363,"_146":9365},"98dc015aed19",[9364],"0992888","bullet. So, that's the",{"_30":9367,"_16":270,"_271":9368,"_146":3542},"262cd71d3fe8",[],{"_30":9370,"_16":270,"_271":9371,"_146":9373},"b8e60f66c923",[9372],"3d3cee4","end of the concrete stuff that I had to say.",[9375,9376,9378],{"_30":9356,"_16":3566,"_3567":9349},{"_30":9364,"_16":3566,"_3567":9377},"00:22:28.650",{"_30":9372,"_16":3566,"_3567":9379},"00:22:32.210",{"_30":9381,"_16":265,"_266":9382,"_274":9396,"_276":277},"6aa48b53bd67",[9383,9388,9391],{"_30":9384,"_16":270,"_271":9385,"_146":9387},"a041310f8f5b",[9386],"64ee810","I'm delighted to take questions and",{"_30":9389,"_16":270,"_271":9390,"_146":3542},"039a7966059e",[],{"_30":9392,"_16":270,"_271":9393,"_146":9395},"2e01c32b99f6",[9394],"11d3939","we can see where the conversation goes. But thank you so much.",[9397,9399],{"_30":9386,"_16":3566,"_3567":9398},"00:22:35.250",{"_30":9394,"_16":3566,"_3567":9400},"00:22:38.130",{"_16":9402,"_111":9403},"mux.video",{"_11":9404,"_13":9405,"_14":9406,"_16":9407,"_17":9408,"_9409":9410,"_7":9411,"_9496":9435,"_9472":9473,"_9497":9493,"_116":9498},"2025-09-19T15:28:17Z","a0bb0634-bab9-4d24-92c0-25b502908fa0","ouh5UxJ2ABFaVjvI34gENf","mux.videoAsset","2025-09-19T15:32:33Z","assetId","e01LQKDvgTCucsD018lnu9PIh9ky02rlANl",{"_9412":9413,"_9414":9415,"_294":9416,"_9417":9418,"_131":9410,"_9419":9420,"_9421":9422,"_9423":9424,"_9425":9426,"_9427":9428,"_9429":9430,"_9431":9405,"_9432":9433,"_9438":9439,"_9443":9444,"_9445":9446,"_9472":9473,"_9474":9475,"_9492":9493,"_9494":9495},"aspect_ratio","16:9","created_at","1758295741",1365.530833,"encoding_tier","smart","ingest_type","on_demand_direct_upload","master_access","none","max_resolution_tier","1080p","max_stored_frame_rate",29.97,"max_stored_resolution","HD","mp4_support","standard","passthrough","playback_ids",[9434],{"_131":9435,"_9436":9437},"00PvwAKE02pl7Pl01mA002qCgI34x6Ak9W9j","policy","public","progress",{"_9438":9440,"_9441":9442},100,"state","completed","resolution_tier","720p","static_renditions",{"_9447":9448,"_9472":9473},"files",[9449,9461,9467],{"_9450":9451,"_9452":9453,"_9454":9455,"_9456":9457,"_3466":9458,"_9459":9460},"bitrate",369864,"ext","mp4","filesize","63132797","height",480,"medium.mp4","width",854,{"_9450":9462,"_9452":9453,"_9454":9463,"_9456":9464,"_3466":9465,"_9459":9466},509272,"86928682",720,"high.mp4",1280,{"_9450":9468,"_9452":9453,"_9454":9469,"_9456":9470,"_3466":9471,"_9459":9457},236544,"40376304",270,"low.mp4","status","ready","tracks",[9476,9486],{"_294":9477,"_131":9478,"_9479":9480,"_9481":9482,"_9483":143,"_9484":9485},1365.519093,"5jkrzcsQeooy6cyxFUeAWfAlfpyPQzMswuXQ6kfv24g","max_channel_layout","stereo","max_channels",2,"primary","type","audio",{"_294":9487,"_131":9488,"_9489":9426,"_9490":9464,"_9491":9466,"_9484":252},1365.497467,"rBn8pTVxdfVKTgDaIwUkat4H02YNFHptf","max_frame_rate","max_height","max_width","upload_id","6ULvuHs7vJJwQr2NpOOaRs58eyEILYQL","video_quality","plus","playbackId","uploadId","https://stream.mux.com/00PvwAKE02pl7Pl01mA002qCgI34x6Ak9W9j","preview","pageNumber","actionData","errors"]

AI-First Organization Design feat. Raza Habib

Speakers