1083 lines
6.2 MiB
HTML
1083 lines
6.2 MiB
HTML
|
<!DOCTYPE html> <html lang=en-AU dir=ltr><!--
|
|||
|
|
Page saved with SingleFile
|
|||
|
|
url: https://gemini.google.com/app/4089c2ad7c8dde16?hl=en-AU
|
|||
|
|
saved date: Sat Jun 06 2026 14:18:24 GMT+1000 (Australian Eastern Standard Time)
|
|||
|
|
--><meta charset=utf-8><meta name=og-profile-acct content=stephenlohning@gmail.com><meta name=viewport content="initial-scale=1,minimum-scale=1,width=device-width,interactive-widget=resizes-content,viewport-fit=cover"><meta name=color-scheme content=only><meta name=google-site-verification content=NDfDvTdbFxsd1TbRRZVQZULBTRT2-fatyuQ1TT8gack><meta name=twitter:card content=summary_large_image><meta property=og:site_name content=Gemini><link rel=icon href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAYAAAD0eNT6AADxRklEQVR4AezBgQAAAACAoP2pF6kCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIDZu4McOa46DOC/r7omjpWARuICwwliTkCzSnaeNRuHE5AbxDkBvgEcxUcIJyArVhEYiOUZT09/ZIU8uC1YUs3/Jz2pX6m3pXpV3/uqzkWMMf5v9Hp/efd6uU591rqE7JZX7fGPF7uPXuI75w9jjPSLL4wxztuNm6vdIb9XexCAkFCqkvzhYnfxDb4zxpgFwBhju+7ubn6b5EWhgECAhNIAEgvP8Y0xxtlaD8eDMcaZOh6+Ds8LBQQACaUAUI48l8UsAsY4X+vR4vyMMZbj26/Vc4n3BUBBEQRIVJ/L8je8cHbGGLn5/HPnZYzR3F6td/4ETWiA9ET2j0AANAESy279Bb51VsYY67LbOS9jjOXW14UEAOXkrAgQNCgCjvf3v8OvnJUxxnq8v3c+xhiPcnt14Eso1EMCFJAAaBEIAcr+48ePL/HKGONsrI8ePzbGOB93f7/dJzQBBCUQ0IRCEYRAKALRInX75s2XeGGMcTbWH09sY4zzscbTI0ChxH9UKAIBQCT5zBjjrKxJjDHOx5EnEgpBAd6/+08QQAAIhQQ0ucZvjDHOxtrEGOM8LIc3T8JVARQAwWkVp+/+izi2l+uyPMG3xhhnYV0SY4zzcLy3X5Z44GT2jwQBIEADACEBh3Z/PguAMcZ6aI0xzsPF4mmhQEJ5f5JTkQAAigQUROopXhhjnIU1NcY4E5W9QlAFcbr3HxKKEBwFSIAEERzjydtHj6YOOMaZWG8+fmSMsX2f/vBqXwGUgA8++i8UgGPigUQBR5DLj28OT/DSGGPz1h9PaGOM7WuXa+qheKiIQkKdEFAQ77rP8RovjTE2b73P0Rhj+3bpLzUo+XDtD0ABQYJQhAYCQIIIT/GVMcbmrRFjjG1b+vpK84QSgKhTgnhXoQgFeTgHlKtdL67wnTHGpq1LL4wxtm33dtlLvatwOvsHIFQQAkiAUEgQFcK9u2u8MMbWTQRwZ4yxbbulzxQBKidrfwUBFAsBqArQaACiAqDLcgZ1wDHG2mUxxtiuT1//4/Jul70AFacFASDRQEAFEAJIFEFTtdA8OXy0TB1wjI1bDxc7Y4ztut/t9hw9kFBAIApBaQgIRZAApUECiKoKQnK5HkwdcIyNW9eDMcaGHfWpABWgENQDBYImQCiAaABoQ2iCgEI9w0tjjM1aWmbMmLHdQfdQAQRQAlFB4MQFvYBoAUFDQgBCAFwbY2zaaoyxWY9vX+2bXHlPUPWQUIDSQEioD1cIBVRA4/LIHi+NMbbaAjDG2Kh2eSZVIQEKJRAARJWcrv01gEQbEkJBCFABZN4KOMaGrcQYY6OW7hUABQQCJJSmCIB6P+uHCqEICKGigCbSPMNXxhibtEaMMbbnk5u/7CtXFfhw578IIBAFSBASlAQQQgUBQAhpNLnsLhMDjLFRa9fFGGN7muVZC0goUJD3jyUIRQBoESQqCCCggaiH/wOHTgywUWOsDrU9Y4zq3uk3/gEFBAnQ01k/gCI0Tm/+g9IggYkBNmqMVWJbxhifvvl+f7RcEQoIBUGAUICeet9/0AeLgYIgGgiQd4+hNLnMspsYYIPGWO1W2zLGaHbPtICgCgInP+ZzeuMflROLgiig4l0RFU1IHI/3EwNs0Bjr8XhvW8YY1b2EQjUQQAj8l4/+g4bQAIVQAQ3QhAQQRbNMDLBBY6zNYjvGGD/54ftryRWAOKkAQpxQ5MRmvxAIghDaAKhFA0guy8QAGzPG2sR2jDGyeNrC6V3/QiEBLU7f/WtQmiAAiAqiQSOJiooCWBCR+TbAxoyR769/bRvGGD979efLt+vurwTVAAQhUT1xse/DOYQ2mhAIoqENWQgNtQCiWRQSzQKa5dXai5/jlU0YY6w7H9mGMcbNxXK9FEoAQgIokNAQKAkQQGhDEAioKCQn2gPRLASlKGRRLm/dfYkXNmGMsd66sw1jjE+6PKsSCoJQCkECxANtSB78biCIQooF+dfvJiQqCqVZSBAFLMlTvLAJY4x1SWzAGLP57833V2UPBQEgEVVAmxO5P9CgFACFRAENRMH73wGAoiiIZtm/OS6XEwNswxjr667GGP/7fnp395X498ofoi3/ZO8sYCTJlit6bnRO73xus0VWmUFgZhqzLTCJZdg2s2fN7Fqj0MzMzGyL/heZmZmZcWGm64VpqvMq9P56PNtQkuJIrcytfAkf73vxIm6ohv7nJX9Q9/3lyYQm8PJV/v7cxgj8KHEzxiPAozRNc/AsN2PQNM0BY9n/OCqh/2r2g+oqv7CKu1cBuPjXaEMWT4AEEn9O3O4JQNO0D0DTNBfES/zLn98asEnVkj/v4icyfZUPzFb/CIrhD3h438Vftt/vYh/mLyBPHjzZ0dbAh0/TwLLTwmHTNM1QPmwrcavdT/bkpO2vJAZV/P1cHvoHicwSBZD/c50gxBoVUJAIie3hTwCaplkkDpimaV7s3/58A5xSe/h7vT9l3x8gmfgEUFbyCQTgpkECCVLn4wYA2HaBTyJAiLF+0a27d886GfDAaZrl7t0zDpimaXZ3brkRD57vP3fwg7SJgO/l49a/6SJOonp/bSgECoRIiUSWC7BeT4k4Pu5kwKY5cJY4PqZpmsNF/6FtTfpjYvUrIMVc/Kf7/gKE9teRPUfeG8DL/gDIsvpPWB0CEZI6GbBpDpxFEk3THCYv9fe//84Z2rh3P2Qx9AEQoyb9uc+/izm27+/9/T0KIPl4AFDYNUB70fd3iiEBOhk68AZBTdNJgKJpmsMkQ7cRmKKXUj0M2TXAxR3Zb3a/ewcAKlsB6c/ylf/MP4DqExCdDNg0B8wCRzRNc3i89D//wQZxi5w3+cHFfR76L+V7QBH/lFifb7+DjZ9k/2sv8vVejy5wS9zYAH9M0zQHxwI3aJrm8IjcbRNV8Qdm4q/Jvr8dTdgTob2AJyAX+iBdzBMy5FEBqxLYXwsTf/cIEEPjtHMBmuYwWUYMmqY5LF7uL3/15MnjG7dAJEap93djIAeK1a/SQ/QM8xIQkIj0skDJ9/2RWwRXTwAAwfCIgEQqGHB7OY7P75LApjnIKoCgaQ6Nbvt7452VbFLAPPS/CnDeOwiEGAioPv+a7PvXCIEn/sHwqMH+mmwiIEj7TcDAtwAA4uTJu3Sb4IOk6XbAd2ma5sCQ2CaCSeh/Xu9fxF8Un39IhATDe/yrlvnB+tiS2Gc5AxnyPAJAjPMqgfDtAoRu9wSgaQ6PRQRN0xwOL/O3v3aKtKninwjImvTnboB2Psv4F8PE3EU6/TxFhqxM0MP9kDY+FYA/1ycdwUAgbQbqksCmOTCWoaBpmsNB8PDc7MerAT0jHyirdxAr8pvtuok8wDx6UKyARcrFXlAnEefX3aZYXRLYNAfGgkTTNIfB5u9+bYO4NTf7ydrfv6zk78Pq18Qdea+AKAIPKFzAwa7lLCIg1ioAyX4XSdzaHXUUoGkOieXs6IimaQ6E3G0TanvfaateF3yq05+v4i3pLwEAKchiE5yYaG
|
|||
|
|
(stephenlohning@gmail.com)" href="https://accounts.google.com/SignOutOptions?hl=en&continue=https://gemini.google.com/app%3Fhl%3Den-AU&ec=GBRAkgU" tabindex=0 role=button></a></div></div></div><div style=overflow:hidden;position:absolute;top:0px;visibility:hidden;width:436px;z-index:991;height:0px;margin-top:57px;right:0px;margin-right:4px></div><div style=overflow:hidden;position:absolute;top:0px;visibility:hidden;width:420px;z-index:991;height:280px;margin-top:70px;right:0px;margin-right:25px></div></div></div></div><div style=display:none></div><chat-app id=app-root ng-version=0.0.0-PLACEHOLDER _nghost-ng-c4057416054 class="enable-lr26-markdown-styling lr26 ng-tns-c4057416054-0 side-nav-open" style=--internal-bar-offset:0px;--announcement-banner-padding-top:0px data-sf-nesting-track-id=1.10><main _ngcontent-ng-c4057416054 class=chat-app data-sf-nesting-track-id=1.10.1><div _ngcontent-ng-c4057416054 data-test-id=chat-app class="side-nav-menu-button with-pill-ui"><chat-app-side-nav-menu-button _ngcontent-ng-c4057416054 _nghost-ng-c478118268 class="ng-tns-c478118268-1 ng-tns-c4057416054-0 ng-star-inserted"><side-nav-sparkle-button _ngcontent-ng-c478118268 _nghost-ng-c3252813403 class="ng-tns-c478118268-1 lr26 ng-star-inserted"><a _ngcontent-ng-c3252813403 href=https://gemini.google.com/ data-test-id=side-nav-sparkle-button class=side-nav-sparkle-button aria-label="New chat" jslog=323333;track:generic_click,impression style=display:flex><div _ngcontent-ng-c3252813403 class=xap-count-badge-container><img _ngcontent-ng-c3252813403 alt aria-hidden=true class="sparkle-image ng-star-inserted" src="data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiBpZD0iTGF5ZXJfMSIgdmlld0JveD0iMCAwIDE5MiAxOTIiPjxkZWZzPjxjbGlwUGF0aCBpZD0iY2xpcHBhdGgiPjxwYXRoIGQ9Ik0xNjQuOTMgODYuNjhjLTEzLjU2LTUuODQtMjUuNDItMTMuODQtMzUuNi0yNC4wMS0xMC4xNy0xMC4xNy0xOC4xOC0yMi4wNC0yNC4wMS0zNS42LTIuMjMtNS4xOS00LjA0LTEwLjU0LTUuNDItMTYuMDJDOTkuNDUgOS4yNiA5Ny44NSA4IDk2IDhzLTMuNDUgMS4yNi0zLjkgMy4wNWMtMS4zOCA1LjQ4LTMuMTggMTAuODEtNS40MiAxNi4wMi01Ljg0IDEzLjU2LTEzLjg0IDI1LjQzLTI0LjAxIDM1LjYtMTAuMTcgMTAuMTYtMjIuMDQgMTguMTctMzUuNiAyNC4wMS01LjE5IDIuMjMtMTAuNTQgNC4wNC0xNi4wMiA1LjQyQzkuMjYgOTIuNTUgOCA5NC4xNSA4IDk2czEuMjYgMy40NSAzLjA1IDMuOWM1LjQ4IDEuMzggMTAuODEgMy4xOCAxNi4wMiA1LjQyIDEzLjU2IDUuODQgMjUuNDIgMTMuODQgMzUuNiAyNC4wMSAxMC4xNyAxMC4xNyAxOC4xOCAyMi4wNCAyNC4wMSAzNS42IDIuMjQgNS4yIDQuMDQgMTAuNTQgNS40MiAxNi4wMkE0LjAzIDQuMDMgMCAwIDAgOTYgMTg0YzEuODUgMCAzLjQ1LTEuMjYgMy45LTMuMDUgMS4zOC01LjQ4IDMuMTgtMTAuODEgNS40Mi0xNi4wMiA1Ljg0LTEzLjU2IDEzLjg0LTI1LjQyIDI0LjAxLTM1LjYgMTAuMTctMTAuMTcgMjIuMDQtMTguMTggMzUuNi0yNC4wMSA1LjItMi4yNCAxMC41NC00LjA0IDE2LjAyLTUuNDJBNC4wMyA0LjAzIDAgMCAwIDE4NCA5NmMwLTEuODUtMS4yNi0zLjQ1LTMuMDUtMy45LTUuNDgtMS4zOC0xMC44MS0zLjE4LTE2LjAyLTUuNDIiIGNsYXNzPSJzdDAiLz48L2NsaXBQYXRoPjxjbGlwUGF0aCBpZD0iY2xpcHBhdGgtMSI+PHBhdGggZD0iTTE2NC45MyA4Ni42OGMtMTMuNTYtNS44NC0yNS40Mi0xMy44NC0zNS42LTI0LjAxLTEwLjE3LTEwLjE3LTE4LjE4LTIyLjA0LTI0LjAxLTM1LjYtMi4yMy01LjE5LTQuMDQtMTAuNTQtNS40Mi0xNi4wMkM5OS40NSA5LjI2IDk3Ljg1IDggOTYgOHMtMy40NSAxLjI2LTMuOSAzLjA1Yy0xLjM4IDUuNDgtMy4xOCAxMC44MS01LjQyIDE2LjAyLTUuODQgMTMuNTYtMTMuODQgMjUuNDMtMjQuMDEgMzUuNi0xMC4xNyAxMC4xNi0yMi4wNCAxOC4xNy0zNS42IDI0LjAxLTUuMTkgMi4yMy0xMC41NCA0LjA0LTE2LjAyIDUuNDJDOS4yNiA5Mi41NSA4IDk0LjE1IDggOTZzMS4yNiAzLjQ1IDMuMDUgMy45YzUuNDggMS4zOCAxMC44MSAzLjE4IDE2LjAyIDUuNDIgMTMuNTYgNS44NCAyNS40MiAxMy44NCAzNS42IDI0LjAxIDEwLjE3IDEwLjE3IDE4LjE4IDIyLjA0IDI0LjAxIDM1LjYgMi4yNCA1LjIgNC4wNCAxMC41NCA1LjQyIDE2LjAyQTQuMDMgNC4wMyAwIDAgMCA5NiAxODRjMS44NSAwIDMuNDUtMS4yNiAzLjktMy4wNSAxLjM4LTUuNDggMy4xOC0xMC44MSA1LjQyLTE2LjAyIDUuODQtMTMuNTYgMTMuODQtMjUuNDIgMjQuMDEtMzUuNiAxMC4xNy0xMC4xNyAyMi4wNC0xOC4xOCAzNS42LTI0LjAxIDUuMi0yLjI0IDEwLjU0LTQuMDQgMTYuMDItNS40MkE0LjAzIDQuMDMgMCAwIDAgMTg0IDk2YzAtMS44NS0xLjI2LTMuNDUtMy4wNS0zLjktNS40OC0xLjM4LTEwLjgxLTMuMTgtMTYuMDItNS40MiIgY2xhc3M9InN0MCIvPjwvY2xpcFBhdGg+PHJhZGlhbEdyYWRpZW50IGlkPSJyYWRpYWwtZ3JhZGllbnQiIGN4PSItMTIyLjQ5IiBjeT0iLTIyMy41MyIgcj0iMTEwLjk4IiBmeD0iLTEyMi40OSIgZnk9Ii0yMjMuNTMiIGdyYWRpZW50VHJhbnNmb3JtPSJtYXRyaXgoMSA
|
|||
|
|
│
|
|||
|
|
▼ (Audio waveform data)
|
|||
|
|
┌────────────────────────────────────────┐
|
|||
|
|
│ Gemma 4 12B │ <── Understands the audio directly
|
|||
|
|
└────────────────────────────────────────┘
|
|||
|
|
│
|
|||
|
|
▼ (Generates written Spanish reply)
|
|||
|
|
┌────────────────────────────────────────┐
|
|||
|
|
│ Piper or Kokoro (Spanish) │ <── Synthesizes the text to voice
|
|||
|
|
└────────────────────────────────────────┘
|
|||
|
|
│
|
|||
|
|
▼ (Plays native Spanish audio back to you)
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=14>Which one should you pick?</h3><sources-carousel-inline ng-version=0.0.0-PLACEHOLDER _nghost-ng-c1747638706> <source-inline-chip _ngcontent-ng-c1747638706 _nghost-ng-c4141473053 class=ng-star-inserted><div _ngcontent-ng-c4141473053 class="source-inline-chip-container luminous-sources hide-from-message-actions ng-star-inserted"><button _ngcontent-ng-c4141473053 cdkoverlayorigin class="button multiple-button ng-star-inserted" aria-label="View source details for citation from Level Up Coding - GitConnected. Opens side panel." jslog='262206;track:generic_click;BardVeMetadataKey:[["r_3ab80f71e82c520c","c_4089c2ad7c8dde16",null,"rc_9f6f6f95d7918163",null,null,"en",null,1,null,null,1,0],null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,[0]]'><div _ngcontent-ng-c4141473053 class="source-label-container gds-body-s ng-star-inserted" dir=ltr><bdi _ngcontent-ng-c4141473053 class=source-title style=max-width:20ch>Level Up Coding - GitConnected</bdi></div></button></div></source-inline-chip></sources-carousel-inline><ul data-path-to-node=15><li><p data-path-to-node=15,0,0>Go with <b data-path-to-node=15,0,0 data-index-in-node=8>Piper</b> if you want something that works with zero friction immediately. You can download an <code data-path-to-node=15,0,0 data-index-in-node=99>es_ES</code> or <code data-path-to-node=15,0,0 data-index-in-node=108>es_MX</code> voice model, feed it the text output from Gemma, and it will read it back to you instantly.</p><li><p data-path-to-node=15,1,0>Go with <b data-path-to-node=15,1,0 data-index-in-node=8>Kokoro (via a Spanish community model)</b> if you want to focus on high-fidelity audio realism where the voice has emotional inflection and natural pacing.</p></ul></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"><overview-carousel _nghost-ng-c672104571 class="lm-enabled ng-star-inserted"><div _ngcontent-ng-c672104571 jslog=268370;track:generic_click,impression></div></overview-carousel></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-564 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-565 hide-action-bar ng-tns-c3135600295-564 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-565"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-565 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-565 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_3ab80f71e82c520c","c_4089c2ad7c8dde16",null,"rc_9f6f6f95d7918163",null,null,"en",null,1,null,null,1,0]];mutable:true'
|
|||
|
|
┌────────────────────────────────────────────────────────┐
|
|||
|
|
│ Kokoro/Piper reads standard Spanish prompt out loud. │
|
|||
|
|
└───────────────────────────┬────────────────────────────┘
|
|||
|
|
│
|
|||
|
|
2. HUMAN INPUT & PHONETIC ANALYSIS
|
|||
|
|
┌───────────────────────────▼────────────────────────────┐
|
|||
|
|
│ User repeats prompt -> Audio captured -> Wav2Vec2 │
|
|||
|
|
│ extracts raw Spanish phonemes (No autocorrect). │
|
|||
|
|
└───────────────────────────┬────────────────────────────┘
|
|||
|
|
│
|
|||
|
|
3. COMPARISON & EVALUATION (LLM)
|
|||
|
|
┌───────────────────────────▼────────────────────────────┐
|
|||
|
|
│ Diff Algorithm aligns Target Phonemes vs User Phonemes. │
|
|||
|
|
│ Gemma 4 evaluates the diff & explains the correction. │
|
|||
|
|
└────────────────────────────────────────────────────────┘
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=7>Stage 1: The Target Generation (System Speaker)</h3><p data-path-to-node=8>The system serves a native Spanish target sentence to the user visually and audio-visually.<ul data-path-to-node=9><li><p data-path-to-node=9,0,0><b data-path-to-node=9,0,0 data-index-in-node=0>The Tech:</b> Use <b data-path-to-node=9,0,0 data-index-in-node=14>Kokoro</b> (with community Spanish weights) or <b data-path-to-node=9,0,0 data-index-in-node=57>Piper (<code data-path-to-node=9,0,0 data-index-in-node=64>es_ES</code> or <code data-path-to-node=9,0,0 data-index-in-node=73>es_MX</code>)</b> to synthesize the ground-truth audio.</p><li><p data-path-to-node=9,1,0><b data-path-to-node=9,1,0 data-index-in-node=0>The Data:</b> Generate a text string of the expected Spanish phonemes for this phrase using a library like <code data-path-to-node=9,1,0 data-index-in-node=103>epitran</code> (which maps text to International Phonetic Alphabet / IPA tokens specifically for Spanish). This acts as your <b data-path-to-node=9,1,0 data-index-in-node=221>Ground Truth Sequence</b>.</p></ul><h3 data-path-to-node=10>Stage 2: Phonetic Analysis (The Listening Engine)</h3><p data-path-to-node=11>To prevent the "autocorrect" flaw of standard ASR, bypass word-level transcription entirely. Instead, use an acoustic model fine-tuned for raw phonetic character extraction (Chen et al., 2025; Lin, 2025).<ul data-path-to-node=12><li><p data-path-to-node=12,0,0><b data-path-to-node=12,0,0 data-index-in-node=0>The Tech:</b> A native-level Spanish model like <b data-path-to-node=12,0,0 data-index-in-node=44>Wav2Vec2-Phoneme</b> or a specialized local instance of <b data-path-to-node=12,0,0 data-index-in-node=96>Whisper</b> running with its language model decoding constraints disabled (forcing it to output raw characters without contextual guessing).</p><li><p data-path-to-node=12,1,0><b data-path-to-node=12,1,0 data-index-in-node=0>The Output:</b> When the human speaks into the microphone, this engine converts their audio into a string of raw Spanish phonetic sounds (the <b data-path-to-node=12,1,0 data-index-in-node=138>Hypothesis Sequence</b>). If they mispronounce the rolled "rr", the phoneme sequence will accurately reflect a single "r" or a substituted sound (Lin, 2025).</p></ul><h3 data-path-to-node=13>Stage 3: The Pronunciation Comparison (The Evaluator)</h3><p data-path-to-node=14>Now you have two clean string arrays representing the ideal pronunciation and the human's actual pronunciation.<ul data-path-to-node=15><li><p data-path-to-node=15,0,0><b data-path-to-node=15,0,0 data-index-in-node=0>The Pre-Processing:</b> Run a basic <b data-path-to-node=15,0,0 data-index-in-node=32>Needleman-Wunsch alignment algorithm</b> (a classic dynamic programming algorithm used in bioinformatics and text-diffing) to line up the two phoneme strings side-by-side (Lin, 2025). This instantly isolates exactly where letters were added, dropped, or substituted.</p><li><p data-path-to-node=15,1,0><b data-path-to-node=15,1,0 data-index-in-node=0>The LLM Analysis:</b> Pass this aligned phonetic mapping directly to <b data-path-to-node=15,1,0 data-index-in-node=65>Gemma 4 12B</b> using a structured prompt (Chen et al., 2025; Lin, 2025).</p></ul><h2 data-path-to-node=17>The Evaluation Prompt (For Gemma 4)</h2><p data-path-to-node=18>Because you are passing a highly structured text-based acoustic description into Gemma 4, it can pinpoint exactly where the learner deviated and offer targeted feedback (Chen et al., 2025). Here is a layout of how to frame your prompt:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-658 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-658 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_edf196fc03484c52","c_4089c2ad7c8dde16",null,"rc_8aa425cfdcf4bda8",null,null,"en",null,1,n
|
|||
|
|
Analyze the misalignment between the Target Spanish pronunciation phonemes and the Student's actual spoken phonemes.
|
|||
|
|
|
|||
|
|
[<span class=hljs-symbol>TARGET PHONEMES</span>]: <span class=hljs-link>/p/ /e/ /r/ /r/ /o/</span>
|
|||
|
|
[<span class=hljs-symbol>STUDENT PHONEMES</span>]: <span class=hljs-link>/p/ /e/ /r/ /o/</span>
|
|||
|
|
|
|||
|
|
Provide feedback in the following strict format:
|
|||
|
|
<span class=hljs-bullet>1.</span> Score: (1-5 based on phonetic alignment accuracy)
|
|||
|
|
<span class=hljs-bullet>2.</span> Error Identification: Identify the exact sound that was missed or substituted.
|
|||
|
|
<span class=hljs-bullet>3.</span> Actionable Advice: Explain the physical mechanics required to fix the error.
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=20>Example of Gemma's Evaluation Response:</h3><blockquote data-path-to-node=21><p data-path-to-node=21,0><b data-path-to-node=21,0 data-index-in-node=0>Score:</b> 3.5/5
|
|||
|
|
<b data-path-to-node=21,0 data-index-in-node=13>Error Identification:</b> You substituted the Spanish trill (<i data-path-to-node=21,0 data-index-in-node=70>erre doble</i> /rr/) with a single flap (<i data-path-to-node=21,0 data-index-in-node=107>ere</i> /r/), turning the word <i data-path-to-node=21,0 data-index-in-node=134>"perro"</i> (dog) into <i data-path-to-node=21,0 data-index-in-node=153>"pero"</i> (but).
|
|||
|
|
<b data-path-to-node=21,0 data-index-in-node=167>Actionable Advice:</b> To produce the correct trill, relax your tongue and place the tip lightly against the alveolar ridge (the bumpy area right behind your upper front teeth). Exhale a steady stream of air to let the tongue tip vibrate freely against the roof of your mouth.</p></blockquote><h2 data-path-to-node=23>Practical Setup Steps</h2><p data-path-to-node=24>Since you are targeting a local environment, this structure keeps everything self-contained, high-performance, and completely private:<ol start=1 data-path-to-node=25><li><p data-path-to-node=25,0,0><b data-path-to-node=25,0,0 data-index-in-node=0>Python Backend:</b> Wrap the pipeline in a lightweight Python backend (<code data-path-to-node=25,0,0 data-index-in-node=67>FastAPI</code> or a native CLI script).</p><li><p data-path-to-node=25,1,0><b data-path-to-node=25,1,0 data-index-in-node=0>Audio Processing:</b> Use <code data-path-to-node=25,1,0 data-index-in-node=22>sounddevice</code> or <code data-path-to-node=25,1,0 data-index-in-node=37>pyaudio</code> to capture the microphone input as a mono 16kHz WAV file.</p><li><p data-path-to-node=25,2,0><b data-path-to-node=25,2,0 data-index-in-node=0>Acoustic Layer:</b> Load a local Hugging Face model like <code data-path-to-node=25,2,0 data-index-in-node=53>facebook/wav2vec2-xlsr-53-spanish</code> to handle phoneme transcription.</p><li><p data-path-to-node=25,3,0><b data-path-to-node=25,3,0 data-index-in-node=0>LLM Orchestration:</b> Interface with your running local instance of Gemma 4 12B via an OpenAI-compatible API layer (provided natively by Ollama or llama.cpp).</p></ol><p data-path-to-node=26>This separation of concerns ensures your system doesn't just guess what the user <i data-path-to-node=26 data-index-in-node=81>meant</i> to say, but systematically analyzes exactly what they <i data-path-to-node=26 data-index-in-node=141>actually</i> said (Lin, 2025).<h2 data-path-to-node=28>References</h2><ul data-path-to-node=29><li><p data-path-to-node=29,0,0>Chen, Y.-W., Ma, M., & Hirschberg, J. (2025). Read to Hear: A Zero-Shot Pronunciation Assessment Using Textual Descriptions and LLMs. <i data-path-to-node=29,0,0 data-index-in-node=134>arXiv</i>. <response-element ng-version=0.0.0-PLACEHOLDER><link-block _nghost-ng-c227220131 class=ng-star-inserted><a _ngcontent-ng-c227220131 target=_blank rel=noopener externallink _nghost-ng-c1003445734 jslog='197247;track:generic_click,impression,attention;BardVeMetadataKey:[["r_edf196fc03484c52","c_4089c2ad7c8dde16",null,"rc_8aa425cfdcf4bda8",null,null,"en",null,1,null,null,1,0]]' href=https://aclanthology.org/2025.emnlp-main.134.pdf class=ng-star-inserted data-hveid=7>https://aclanthology.org/2025.emnlp-main.134.pdf</a></link-block></response-element>
|
|||
|
|
<code data-path-to-node=29,0,0 data-index-in-node=190>Cited by: 1</code></p><li><p data-path-to-node=29,1,0>Lin, I. (2025). Democratizing Pronunciation Training through Large Language Models. <i data-path-to-node=29,1,0 data-index-in-node=84>TechRxiv</i>. <response-element ng-version=0.0.0-PLACEHOLDER><link-block _nghost-ng-c227220131 class=ng-star-inserted><a _ngcontent-ng-c227220131 target=_blank rel=noopener externallink _nghost-ng-c1003445734 jslog='197247;track:generic_click,impression,attention;BardVeMetadataKey:[["r_edf196fc03484c52","c_4089c2ad7c8dde16",null,"rc_8aa425cfdcf4bda8",null,null,"en",null,1,null,null,1,0]]' href="https://www.google.com/search?q=https%3A%2F%2Fwww.techrxiv.org%2Fdoi%2Fpdf%2F10.36227%2Ftechrxiv.176240380.07002909" class=ng-star-inserted data-hveid=8>https://www.techrxiv.org/doi/pdf/10.36227/techrxiv.176240380.07002909</a></link-block></response-element>
|
|||
|
|
<code data-path-to-node=29,1,0 data-index-in-node=164>Cited by: 0</code></p></ul></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-566 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-567 hide-action-bar ng-tns-c3135600295-566 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-567"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-567 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-567 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_edf196fc03484c52","c_4089c2ad7c8dde16",null,"rc_8aa425cfdcf4bda8",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-567 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_edf196fc03484c52","c_4089c2ad7c8dde16",null,"rc_8aa425cfdcf4bda8",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=fo
|
|||
|
|
<span class=hljs-keyword>export</span> <span class=hljs-keyword>interface</span> PhonemeAlignment {
|
|||
|
|
<span class=hljs-attr>target</span>: <span class=hljs-built_in>string</span>; <span class=hljs-comment>// Ground truth phoneme (e.g., "r")</span>
|
|||
|
|
student: <span class=hljs-built_in>string</span>; <span class=hljs-comment>// What the user actually said (e.g., "o" or "-")</span>
|
|||
|
|
match: <span class=hljs-built_in>boolean</span>; <span class=hljs-comment>// Did they get it right?</span>
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>export</span> <span class=hljs-keyword>interface</span> EvaluationResult {
|
|||
|
|
<span class=hljs-attr>score</span>: <span class=hljs-built_in>number</span>;
|
|||
|
|
errorIdentification: <span class=hljs-built_in>string</span>;
|
|||
|
|
actionableAdvice: <span class=hljs-built_in>string</span>;
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=8>Step 2: The Core Alignment Engine</h3><p data-path-to-node=9>To figure out exactly where the student drifted, use a sequence alignment algorithm (like Levenshtein distance or Needleman-Wunsch). This calculates whether a phoneme was an exact match, a <b data-path-to-node=9 data-index-in-node=189>substitution</b>, an <b data-path-to-node=9 data-index-in-node=206>insertion</b>, or a <b data-path-to-node=9 data-index-in-node=222>deletion</b> (<code data-path-to-node=9 data-index-in-node=232>-</code>).</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-660 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-660 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_8ffc1913dbc42772","c_4089c2ad7c8dde16",null,"rc_5b0acb2320e33d97",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-660"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-660"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-660 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-660>TypeScript</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-660 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-660 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-660 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_8ffc1913dbc42772","c_4089c2ad7c8dde16",null,"rc_5b0acb2320e33d97",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-
|
|||
|
|
<span class=hljs-keyword>import</span> { PhonemeAlignment } <span class=hljs-keyword>from</span> <span class=hljs-string>'./types'</span>;
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>export</span> <span class=hljs-function><span class=hljs-keyword>function</span> <span class=hljs-title>alignPhonemes</span>(<span class=hljs-params>target: <span class=hljs-built_in>string</span>[], student: <span class=hljs-built_in>string</span>[]</span>): <span class=hljs-title>PhonemeAlignment</span>[] </span>{
|
|||
|
|
<span class=hljs-keyword>const</span> result: PhonemeAlignment[] = [];
|
|||
|
|
<span class=hljs-keyword>const</span> maxLength = <span class=hljs-built_in>Math</span>.max(target.length, student.length);
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>for</span> (<span class=hljs-keyword>let</span> i = <span class=hljs-number>0</span>; i < maxLength; i++) {
|
|||
|
|
<span class=hljs-keyword>const</span> t = target[i] || <span class=hljs-string>'-'</span>; <span class=hljs-comment>// '-' represents an omission or extra insertion</span>
|
|||
|
|
<span class=hljs-keyword>const</span> s = student[i] || <span class=hljs-string>'-'</span>;
|
|||
|
|
|
|||
|
|
result.push({
|
|||
|
|
<span class=hljs-attr>target</span>: t,
|
|||
|
|
<span class=hljs-attr>student</span>: s,
|
|||
|
|
<span class=hljs-attr>match</span>: t.toLowerCase() === s.toLowerCase() && t !== <span class=hljs-string>'-'</span>
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>return</span> result;
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=12>Step 3: Orchestrating the Gemma 4 Evaluation Loop</h3><p data-path-to-node=13>Using an OpenAI-compatible interface (which Ollama exposes natively at <code data-path-to-node=13 data-index-in-node=71>http://localhost:11434/v1</code>), send the phonetic alignment data to Gemma 4 for linguistic analysis.<p data-path-to-node=14>By utilizing <b data-path-to-node=14 data-index-in-node=13>JSON Mode</b>, you guarantee that Gemma returns data structurally matching your TypeScript interfaces.</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-661 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-661 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_8ffc1913dbc42772","c_4089c2ad7c8dde16",null,"rc_5b0acb2320e33d97",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=9><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-661"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-661"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-661 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-661>TypeScript</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-661 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-661 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-661 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_8ffc1913dbc42772","c_4089c2ad7c8dde16",null,"rc_5b0acb2320e33d97",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"
|
|||
|
|
<span class=hljs-keyword>import</span> OpenAI <span class=hljs-keyword>from</span> <span class=hljs-string>'openai'</span>;
|
|||
|
|
<span class=hljs-keyword>import</span> { PhonemeAlignment, EvaluationResult } <span class=hljs-keyword>from</span> <span class=hljs-string>'./types'</span>;
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>const</span> openai = <span class=hljs-keyword>new</span> OpenAI({
|
|||
|
|
<span class=hljs-attr>baseURL</span>: <span class=hljs-string>'http://localhost:11434/v1'</span>, <span class=hljs-comment>// Points to your local Ollama instance</span>
|
|||
|
|
<span class=hljs-attr>apiKey</span>: <span class=hljs-string>'ollama'</span>, <span class=hljs-comment>// Required placeholder</span>
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>export</span> <span class=hljs-keyword>async</span> <span class=hljs-function><span class=hljs-keyword>function</span> <span class=hljs-title>evaluatePronunciation</span>(<span class=hljs-params>
|
|||
|
|
phraseText: <span class=hljs-built_in>string</span>,
|
|||
|
|
alignment: PhonemeAlignment[]
|
|||
|
|
</span>): <span class=hljs-title>Promise</span><<span class=hljs-title>EvaluationResult</span>> </span>{
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>const</span> prompt = <span class=hljs-string>`
|
|||
|
|
You are an expert Spanish phonetics instructor evaluating a student practicing pronunciation.
|
|||
|
|
The target Spanish phrase is: "<span class=hljs-subst>${phraseText}</span>".
|
|||
|
|
|
|||
|
|
Analyze the following array showing the target phonemes vs the student's actual spoken phonemes:
|
|||
|
|
<span class=hljs-subst>${<span class=hljs-built_in>JSON</span>.stringify(alignment, <span class=hljs-literal>null</span>, <span class=hljs-number>2</span>)}</span>
|
|||
|
|
|
|||
|
|
Identify specific articulation errors (e.g., substituting a tapped /r/ for a trilled /rr/, dropping terminal consonants). Provide explicit, actionable feedback on how to physically position their mouth/tongue to fix it.
|
|||
|
|
|
|||
|
|
Respond STRICTLY in this JSON format:
|
|||
|
|
{
|
|||
|
|
"score": <number between 1 and 5>,
|
|||
|
|
"errorIdentification": "<string describing what sound was missed>",
|
|||
|
|
"actionableAdvice": "<string describing the physical jaw/tongue movement required to correct it>"
|
|||
|
|
}
|
|||
|
|
`</span>;
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>const</span> response = <span class=hljs-keyword>await</span> openai.chat.completions.create({
|
|||
|
|
<span class=hljs-attr>model</span>: <span class=hljs-string>'gemma4:12b'</span>, <span class=hljs-comment>// Replace with the exact local name of your model tag</span>
|
|||
|
|
<span class=hljs-attr>messages</span>: [{ <span class=hljs-attr>role</span>: <span class=hljs-string>'user'</span>, <span class=hljs-attr>content</span>: prompt }],
|
|||
|
|
<span class=hljs-attr>response_format</span>: { <span class=hljs-attr>type</span>: <span class=hljs-string>'json_object'</span> } <span class=hljs-comment>// Enforces structured JSON output</span>
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>const</span> content = response.choices[<span class=hljs-number>0</span>].message.content;
|
|||
|
|
<span class=hljs-keyword>if</span> (!content) <span class=hljs-keyword>throw</span> <span class=hljs-keyword>new</span> <span class=hljs-built_in>Error</span>(<span class=hljs-string>"Empty response from local LLM"</span>);
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>return</span> <span class=hljs-built_in>JSON</span>.parse(content) <span class=hljs-keyword>as</span> EvaluationResult;
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=17>Step 4: Putting It All Together (The Loop)</h3><p data-path-to-node=18>Here is how your main program coordinates the pipeline. It takes the text, converts it to targets, extracts the audio features, and runs the assessment.</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-662 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-662 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_8ffc1913dbc42772","c_4089c2ad7c8dde16",null,"rc_5b0acb2320e33d97",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=12><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-662"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-662"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-662 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-662>TypeScript</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-662 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-662 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-662 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_8ffc1913dbc42772","c_4089c2ad7c8dde16",null,"rc_5b0acb2320e33d97",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-
|
|||
|
|
<span class=hljs-keyword>import</span> { alignPhonemes } <span class=hljs-keyword>from</span> <span class=hljs-string>'./alignment'</span>;
|
|||
|
|
<span class=hljs-keyword>import</span> { evaluatePronunciation } <span class=hljs-keyword>from</span> <span class=hljs-string>'./evaluator'</span>;
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>async</span> <span class=hljs-function><span class=hljs-keyword>function</span> <span class=hljs-title>runPronunciationSession</span>(<span class=hljs-params></span>) </span>{
|
|||
|
|
<span class=hljs-keyword>const</span> targetText = <span class=hljs-string>"Perro"</span>;
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// 1. Generate text phonemes using a utility/dictionary (e.g., Spanish IPA mapping)</span>
|
|||
|
|
<span class=hljs-keyword>const</span> targetPhonemes = [<span class=hljs-string>"p"</span>, <span class=hljs-string>"e"</span>, <span class=hljs-string>"rr"</span>, <span class=hljs-string>"o"</span>];
|
|||
|
|
|
|||
|
|
<span class=hljs-built_in>console</span>.log(<span class=hljs-string>"System Voice (Piper/Kokoro): 'Perro'"</span>);
|
|||
|
|
<span class=hljs-built_in>console</span>.log(<span class=hljs-string>"Listening to student response..."</span>);
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// 2. Extract raw student phonemes from your local acoustic model (e.g. Wav2Vec2 via transformers.js)</span>
|
|||
|
|
<span class=hljs-comment>// Let's pretend the user said "pero" (single flap instead of rolling the double r)</span>
|
|||
|
|
<span class=hljs-keyword>const</span> studentPhonemes = [<span class=hljs-string>"p"</span>, <span class=hljs-string>"e"</span>, <span class=hljs-string>"r"</span>, <span class=hljs-string>"o"</span>];
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// 3. Align sequences</span>
|
|||
|
|
<span class=hljs-keyword>const</span> alignmentData = alignPhonemes(targetPhonemes, studentPhonemes);
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// 4. Pass to Gemma 4 for pedagogical insight</span>
|
|||
|
|
<span class=hljs-built_in>console</span>.log(<span class=hljs-string>"Analyzing speech mechanics locally via Gemma 4..."</span>);
|
|||
|
|
<span class=hljs-keyword>const</span> evaluation = <span class=hljs-keyword>await</span> evaluatePronunciation(targetText, alignmentData);
|
|||
|
|
|
|||
|
|
<span class=hljs-built_in>console</span>.log(<span class=hljs-string>"\n--- Evaluation Results ---"</span>);
|
|||
|
|
<span class=hljs-built_in>console</span>.log(<span class=hljs-string>`Score: <span class=hljs-subst>${evaluation.score}</span>/5`</span>);
|
|||
|
|
<span class=hljs-built_in>console</span>.log(<span class=hljs-string>`Error: <span class=hljs-subst>${evaluation.errorIdentification}</span>`</span>);
|
|||
|
|
<span class=hljs-built_in>console</span>.log(<span class=hljs-string>`Correction Guide: <span class=hljs-subst>${evaluation.actionableAdvice}</span>`</span>);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
runPronunciationSession();
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=20>Tips for Local TypeScript Performance</h3><ol start=1 data-path-to-node=21><li><p data-path-to-node=21,0,0><b data-path-to-node=21,0,0 data-index-in-node=0>Audio Buffers:</b> If running in Node.js, use native <code data-path-to-node=21,0,0 data-index-in-node=49>Buffer</code> or <code data-path-to-node=21,0,0 data-index-in-node=59>Float32Array</code> objects to stream raw 16kHz audio from the mic straight into your inference step to keep latencies low.</p><li><p data-path-to-node=21,1,0><b data-path-to-node=21,1,0 data-index-in-node=0>Web Workers:</b> If you ever decide to move the acoustic layer (Wav2Vec2) directly into a web browser frontend, wrap it in a Web Worker using TypeScript so it doesn't block your main UI frame while processing matrix multiplications.</p></ol></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response complete has-thoughts gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-568 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-569 hide-action-bar ng-tns-c3135600295-568 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-569"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-569 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-569 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_8ffc1913dbc42772","c_4089c2ad7c8dde16",null,"rc_5b0acb2320e33d97",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-569 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-
|
|||
|
|
<span class=hljs-built_in>cd</span> llama.cpp
|
|||
|
|
make
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><li><p data-path-to-node=6,1,0><b data-path-to-node=6,1,0 data-index-in-node=0>Download the GGUF model:</b> Instead of abstracting files away, you download <code data-path-to-node=6,1,0 data-index-in-node=73>.gguf</code> files directly from Hugging Face. For instance, to get a 4-bit quantized Gemma 4 model, look for files named <code data-path-to-node=6,1,0 data-index-in-node=188>gemma-4-12b-it-Q4_K_M.gguf</code>.</p><li><p data-path-to-node=6,2,0><b data-path-to-node=6,2,0 data-index-in-node=0>Run a basic CLI inference:</b> Test that your build works by running the model directly through the command line:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-664 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-664 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_d17799e078dcaabb","c_4089c2ad7c8dde16",null,"rc_bf93e9f4f85a3c54",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-664"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-664"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-664 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-664>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-664 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-664 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-664 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_d17799e078dcaabb","c_4089c2ad7c8dde16",null,"rc_bf93e9f4f85a3c54",null,null,"en",null,1,null
|
|||
|
|
</code></pre></div></div></div></code-block></response-element></ol><h2 data-path-to-node=8>Project 2: Separating the Model via an Open Standard API</h2><p data-path-to-node=9>Instead of writing bespoke code that hooks into custom <code data-path-to-node=9 data-index-in-node=55>llama.cpp</code> bindings, you can leverage its built-in server. <code data-path-to-node=9 data-index-in-node=113>llama.cpp</code> includes <b data-path-to-node=9 data-index-in-node=132><code data-path-to-node=9 data-index-in-node=132>llama-server</code></b>, which spins up a local background process that exposes an <b data-path-to-node=9 data-index-in-node=204>OpenAI-Compatible REST API</b>.<p data-path-to-node=10>This is the ultimate abstraction layer. Your TypeScript code can now use the official, production-grade <code data-path-to-node=10 data-index-in-node=104>openai</code> npm package, pointing it entirely at your local machine. If you ever switch your backend model from <code data-path-to-node=10 data-index-in-node=211>llama.cpp</code> to another local engine, your application code won't need to change.<h3 data-path-to-node=11>Launching the open standard server:</h3><p data-path-to-node=12>Run this command from your compiled <code data-path-to-node=12 data-index-in-node=36>llama.cpp</code> directory:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-665 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-665 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_d17799e078dcaabb","c_4089c2ad7c8dde16",null,"rc_bf93e9f4f85a3c54",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=9><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-665"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-665"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-665 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-665>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-665 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-665 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-665 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><ul data-path-to-node=14><li><p data-path-to-node=14,0,0><code data-path-to-node=14,0,0 data-index-in-node=0>-c 4096</code>: Sets the context window.</p><li><p data-path-to-node=14,1,0><code data-path-to-node=14,1,0 data-index-in-node=0>--n-gpu-layers 99</code>: On a Mac, this forces all layers directly into Unified Memory (VRAM) via the Metal backend for fast performance.</p></ul><p data-path-to-node=15>Your local machine now hosts a standardized endpoint at <code data-path-to-node=15 data-index-in-node=56>http://localhost:8080/v1/chat/completions</code>.<h2 data-path-to-node=17>Project 3: Learning Zod for Runtime Type Checking</h2><p data-path-to-node=18>In Python, Pydantic uses type hints to parse and validate data structures at runtime. In TypeScript, because standard types are fully erased at compilation, <b data-path-to-node=18 data-index-in-node=157>Zod</b> steps in to handle that exact runtime validation role.<p data-path-to-node=19>You define a runtime schema, and Zod infers the TypeScript static type from it automatically. This prevents "type drift."<h3 data-path-to-node=20>Implementation Example:</h3><p data-path-to-node=21>First, install the library:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-666 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-666 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_d17799e078dcaabb","c_4089c2ad7c8dde16",null,"rc_bf93e9f4f85a3c54",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=12><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-666"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-666"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-666 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-666>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-666 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-666 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-666 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-desc
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=23>Here is how you use Zod to validate the JSON payloads coming back from your open standard local API server:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-667 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-667 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_d17799e078dcaabb","c_4089c2ad7c8dde16",null,"rc_bf93e9f4f85a3c54",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=15><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-667"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-667"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-667 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-667>TypeScript</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-667 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-667 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-667 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_d17799e078dcaabb","c_4089c2ad7c8dde16",null,"rc_bf93e9f4f85a3c54",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=copy data-mat-icon-namespace=lumi-symbols fonticon=copy></mat-icon></gem-icon><span class=mat-focus-ind
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// 1. Define the runtime validation schema (Equivalent to Pydantic BaseModel)</span>
|
|||
|
|
<span class=hljs-keyword>export</span> <span class=hljs-keyword>const</span> PronunciationFeedbackSchema = z.object({
|
|||
|
|
<span class=hljs-attr>score</span>: z.number().min(<span class=hljs-number>1</span>).max(<span class=hljs-number>5</span>),
|
|||
|
|
<span class=hljs-attr>errorIdentified</span>: z.string(),
|
|||
|
|
<span class=hljs-attr>actionableAdvice</span>: z.string(),
|
|||
|
|
<span class=hljs-attr>detectedPhonemes</span>: z.array(z.string())
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// 2. Extract the compile-time TypeScript type automatically from the schema</span>
|
|||
|
|
<span class=hljs-keyword>export</span> <span class=hljs-keyword>type</span> PronunciationFeedback = z.infer<<span class=hljs-keyword>typeof</span> PronunciationFeedbackSchema>;
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// 3. Example of runtime parsing and error handling</span>
|
|||
|
|
<span class=hljs-function><span class=hljs-keyword>function</span> <span class=hljs-title>handleLLMResponse</span>(<span class=hljs-params>rawJsonString: <span class=hljs-built_in>string</span></span>) </span>{
|
|||
|
|
<span class=hljs-keyword>try</span> {
|
|||
|
|
<span class=hljs-keyword>const</span> rawData = <span class=hljs-built_in>JSON</span>.parse(rawJsonString);
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// .parse() validates the schema at runtime. If data fails, it throws an error.</span>
|
|||
|
|
<span class=hljs-keyword>const</span> cleanData: PronunciationFeedback = PronunciationFeedbackSchema.parse(rawData);
|
|||
|
|
|
|||
|
|
<span class=hljs-built_in>console</span>.log(<span class=hljs-string>"Validated Data safely:"</span>, cleanData.score);
|
|||
|
|
} <span class=hljs-keyword>catch</span> (error) {
|
|||
|
|
<span class=hljs-keyword>if</span> (error <span class=hljs-keyword>instanceof</span> z.ZodError) {
|
|||
|
|
<span class=hljs-comment>// Zod gives you highly granular structural error logging</span>
|
|||
|
|
<span class=hljs-built_in>console</span>.error(<span class=hljs-string>"JSON did not match required schema structure:"</span>, error.errors);
|
|||
|
|
} <span class=hljs-keyword>else</span> {
|
|||
|
|
<span class=hljs-built_in>console</span>.error(<span class=hljs-string>"Invalid JSON string received"</span>);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h2 data-path-to-node=26>Project 4: Extracting Text from an EPUB Document</h2><p data-path-to-node=27>An <code data-path-to-node=27 data-index-in-node=3>.epub</code> file is essentially a standard <code data-path-to-node=27 data-index-in-node=40>.zip</code> file containing compressed XHTML content, CSS files, images, and XML metadata documents (<code data-path-to-node=27 data-index-in-node=134>.opf</code>). To pull raw text out of it cleanly, you must unpack the archive, find the reading sequence inside the spine document, and strip out the HTML tags.<p data-path-to-node=28>Instead of writing that low-level parsing logic by hand, you can use a native TypeScript parsing wrapper like <b data-path-to-node=28 data-index-in-node=110><code data-path-to-node=28 data-index-in-node=110>epubts</code></b> or <b data-path-to-node=28 data-index-in-node=120><code data-path-to-node=28 data-index-in-node=120>@lingo-reader/epub-parser</code></b>.<h3 data-path-to-node=29>Implementation Example (Node.js/TypeScript):</h3><p data-path-to-node=30>This script safely extracts text chapter-by-chapter without bringing along heavy browser-based DOM dependencies:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-668 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-668 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_d17799e078dcaabb","c_4089c2ad7c8dde16",null,"rc_bf93e9f4f85a3c54",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=18><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-668"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-668"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-668 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-668>TypeScript</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-668 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-668 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-668 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><bu
|
|||
|
|
<span class=hljs-keyword>import</span> { readFileSync } <span class=hljs-keyword>from</span> <span class=hljs-string>'node:fs'</span>;
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>async</span> <span class=hljs-function><span class=hljs-keyword>function</span> <span class=hljs-title>extractTextFromEpub</span>(<span class=hljs-params>filePath: <span class=hljs-built_in>string</span></span>): <span class=hljs-title>Promise</span><<span class=hljs-title>string</span>[]> </span>{
|
|||
|
|
<span class=hljs-comment>// 1. Read file into a raw buffer</span>
|
|||
|
|
<span class=hljs-keyword>const</span> fileBuffer = readFileSync(filePath);
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// Create a standard File blob structure that the parser accepts</span>
|
|||
|
|
<span class=hljs-keyword>const</span> epubFile = <span class=hljs-keyword>new</span> File([fileBuffer.buffer], <span class=hljs-string>'book.epub'</span>);
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// 2. Load and unpack the EPUB structure</span>
|
|||
|
|
<span class=hljs-keyword>const</span> epub = <span class=hljs-keyword>await</span> Epub.load(epubFile);
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// 3. Map out the linear reading order (the Spine)</span>
|
|||
|
|
<span class=hljs-keyword>const</span> chapterIds = epub.spine.contents.map(<span class=hljs-function><span class=hljs-params>item</span> =></span> item.id);
|
|||
|
|
<span class=hljs-keyword>const</span> fullBookText: <span class=hljs-built_in>string</span>[] = [];
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// 4. Iterate through and clean up HTML tags</span>
|
|||
|
|
<span class=hljs-keyword>for</span> (<span class=hljs-keyword>const</span> id <span class=hljs-keyword>of</span> chapterIds) {
|
|||
|
|
<span class=hljs-keyword>const</span> rawChapter = <span class=hljs-keyword>await</span> epub.getChapter(id);
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>if</span> (rawChapter && rawChapter.content) {
|
|||
|
|
<span class=hljs-comment>// Regex to strip HTML tags, leaving behind clean raw text sentences</span>
|
|||
|
|
<span class=hljs-keyword>const</span> cleanText = rawChapter.content
|
|||
|
|
.replace(<span class=hljs-regexp>/<[^>]*>/g</span>, <span class=hljs-string>''</span>) <span class=hljs-comment>// Strip tags</span>
|
|||
|
|
.replace(<span class=hljs-regexp>/\s+/g</span>, <span class=hljs-string>' '</span>) <span class=hljs-comment>// Normalize whitespace spacing</span>
|
|||
|
|
.trim();
|
|||
|
|
|
|||
|
|
fullBookText.push(cleanText);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>return</span> fullBookText;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// Usage</span>
|
|||
|
|
extractTextFromEpub(<span class=hljs-string>'./materials/spanish_course.epub'</span>).then(<span class=hljs-function>(<span class=hljs-params>chapters</span>) =></span> {
|
|||
|
|
<span class=hljs-built_in>console</span>.log(<span class=hljs-string>`Successfully extracted <span class=hljs-subst>${chapters.length}</span> chapters.`</span>);
|
|||
|
|
<span class=hljs-built_in>console</span>.log(<span class=hljs-string>"Sample text from Chapter 1:"</span>, chapters[<span class=hljs-number>0</span>].substring(<span class=hljs-number>0</span>, <span class=hljs-number>300</span>));
|
|||
|
|
});
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=32>Which of these four projects are you planning to spin up first to establish your base layer?</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete has-thoughts gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-570 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-571 hide-action-bar ng-tns-c3135600295-570 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-571"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-571 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-571 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_d17799e078dcaabb","c_4089c2ad7c8dde16",null,"rc_bf93e9f4f85a3c54",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-571 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_d17799e078dcaabb","c_4089c2ad7c8dde16",null,"rc_bf93e9f4f85a3c54",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-li
|
|||
|
|
│ React Client Component │ <-- Captures speech/text input
|
|||
|
|
└────────────────┬────────────────┘
|
|||
|
|
│
|
|||
|
|
│ (Secure internal fetch)
|
|||
|
|
▼
|
|||
|
|
┌─────────────────────────────────┐
|
|||
|
|
│ Next.js Server Route │ <-- Parses files, runs Zod schemas
|
|||
|
|
└────────────────┬────────────────┘
|
|||
|
|
│
|
|||
|
|
│ (OpenAI-Standard REST API)
|
|||
|
|
▼
|
|||
|
|
┌─────────────────────────────────┐
|
|||
|
|
│ llama-server Backend │ <-- Executes heavy GGUF inference
|
|||
|
|
└─────────────────────────────────┘
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=7>Step 1: The Server-Side API Endpoint (<code data-path-to-node=7 data-index-in-node=38>/app/api/evaluate/route.ts</code>)</h3><p data-path-to-node=8>By placing the local LLM logic inside a Next.js Route Handler, your frontend code doesn't need to know anything about <code data-path-to-node=8 data-index-in-node=118>llama.cpp</code>. It simply POSTs data to a standard URL path, and the server route ensures the schema is strictly typed with Zod before responding.</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-670 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-670 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_a9e11da211aee173","c_4089c2ad7c8dde16",null,"rc_d745e6a2f12a1262",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=4><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-670"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-670"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-670 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-670>TypeScript</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-670 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-670 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-670 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_a9e11da211aee173","c_4089c2ad7c8dde16",null,"rc_d745e6a2f12a1262",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcon
|
|||
|
|
<span class=hljs-keyword>import</span> OpenAI <span class=hljs-keyword>from</span> <span class=hljs-string>'openai'</span>;
|
|||
|
|
<span class=hljs-keyword>import</span> { PronunciationFeedbackSchema } <span class=hljs-keyword>from</span> <span class=hljs-string>'@/schemas/validation'</span>;
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// Points to your decoupled llama-server API bridge</span>
|
|||
|
|
<span class=hljs-keyword>const</span> localLLM = <span class=hljs-keyword>new</span> OpenAI({
|
|||
|
|
<span class=hljs-attr>baseURL</span>: <span class=hljs-string>'http://localhost:8080/v1'</span>,
|
|||
|
|
<span class=hljs-attr>apiKey</span>: <span class=hljs-string>'local-token'</span>,
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>export</span> <span class=hljs-keyword>async</span> <span class=hljs-function><span class=hljs-keyword>function</span> <span class=hljs-title>POST</span>(<span class=hljs-params>request: Request</span>) </span>{
|
|||
|
|
<span class=hljs-keyword>try</span> {
|
|||
|
|
<span class=hljs-keyword>const</span> body = <span class=hljs-keyword>await</span> request.json();
|
|||
|
|
<span class=hljs-keyword>const</span> { phraseText, alignmentData } = body;
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// Send structure to local llama.cpp server</span>
|
|||
|
|
<span class=hljs-keyword>const</span> response = <span class=hljs-keyword>await</span> localLLM.chat.completions.create({
|
|||
|
|
<span class=hljs-attr>model</span>: <span class=hljs-string>'gemma-4-12b'</span>,
|
|||
|
|
<span class=hljs-attr>messages</span>: [
|
|||
|
|
{
|
|||
|
|
<span class=hljs-attr>role</span>: <span class=hljs-string>'user'</span>,
|
|||
|
|
<span class=hljs-attr>content</span>: <span class=hljs-string>`Evaluate this Spanish alignment: <span class=hljs-subst>${<span class=hljs-built_in>JSON</span>.stringify(alignmentData)}</span>`</span>
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
<span class=hljs-attr>response_format</span>: { <span class=hljs-attr>type</span>: <span class=hljs-string>'json_object'</span> }
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>const</span> rawContent = response.choices[<span class=hljs-number>0</span>].message.content || <span class=hljs-string>'{}'</span>;
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// Validate the response using Zod at runtime</span>
|
|||
|
|
<span class=hljs-keyword>const</span> validatedFeedback = PronunciationFeedbackSchema.parse(<span class=hljs-built_in>JSON</span>.parse(rawContent));
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>return</span> NextResponse.json({ <span class=hljs-attr>success</span>: <span class=hljs-literal>true</span>, <span class=hljs-attr>data</span>: validatedFeedback });
|
|||
|
|
} <span class=hljs-keyword>catch</span> (error) {
|
|||
|
|
<span class=hljs-built_in>console</span>.error(<span class=hljs-string>"API Pipeline Error:"</span>, error);
|
|||
|
|
<span class=hljs-keyword>return</span> NextResponse.json({ <span class=hljs-attr>success</span>: <span class=hljs-literal>false</span>, <span class=hljs-attr>error</span>: <span class=hljs-string>'Validation failed'</span> }, { <span class=hljs-attr>status</span>: <span class=hljs-number>500</span> });
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=11>Step 2: The Interactive React Frontend Component</h3><p data-path-to-node=12>On the client side, you can build a standard React state loop. This component displays the target Spanish sentence, lets the user interact with it, sends the payload to the Next.js API, and renders the structured feedback safely.</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-671 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-671 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_a9e11da211aee173","c_4089c2ad7c8dde16",null,"rc_d745e6a2f12a1262",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=7><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-671"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-671"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-671 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-671>TypeScript</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-671 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-671 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-671 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_a9e11da211aee173","c_4089c2ad7c8dde16",null,"rc_d745e6a2f12a1262",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font
|
|||
|
|
|
|||
|
|
import { useState } from 'react';
|
|||
|
|
import type { PronunciationFeedback } from '@/schemas/validation';
|
|||
|
|
|
|||
|
|
export default function SpanishPracticeCard() {
|
|||
|
|
const [targetSentence] = useState("El perro corre rápido.");
|
|||
|
|
const [feedback, setFeedback] = useState<PronunciationFeedback | null>(null);
|
|||
|
|
const [loading, setLoading] = useState(false);
|
|||
|
|
|
|||
|
|
const handleSimulateAssessment = async () => {
|
|||
|
|
setLoading(true);
|
|||
|
|
|
|||
|
|
// Mock data mimicking an extracted phonetic array sequence
|
|||
|
|
const mockAlignment = [
|
|||
|
|
{ target: "p", student: "p", match: true },
|
|||
|
|
{ target: "e", student: "e", match: true },
|
|||
|
|
{ target: "rr", student: "r", match: false }, // Missed the rolled double-R
|
|||
|
|
{ target: "o", student: "o", match: true }
|
|||
|
|
];
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
const res = await fetch('/api/evaluate', {
|
|||
|
|
method: 'POST',
|
|||
|
|
headers: { 'Content-Type': 'application/json' },
|
|||
|
|
body: JSON.stringify({ phraseText: targetSentence, alignmentData: mockAlignment })
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
const result = await res.json();
|
|||
|
|
if (result.success) {
|
|||
|
|
setFeedback(result.data);
|
|||
|
|
}
|
|||
|
|
} catch (err) {
|
|||
|
|
console.error("Error fetching evaluation:", err);
|
|||
|
|
} finally {
|
|||
|
|
setLoading(false);
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
return (
|
|||
|
|
<div className="max-w-md mx-auto my-8 p-6 bg-white rounded-xl shadow-md border border-gray-100">
|
|||
|
|
<h2 className="text-sm font-semibold tracking-wide uppercase text-indigo-600">
|
|||
|
|
Spanish Pronunciation Coach
|
|||
|
|
</h2>
|
|||
|
|
<p className="mt-2 text-2xl font-bold text-gray-900 font-serif">
|
|||
|
|
{targetSentence}
|
|||
|
|
</p>
|
|||
|
|
|
|||
|
|
<button
|
|||
|
|
onClick={handleSimulateAssessment}
|
|||
|
|
disabled={loading}
|
|||
|
|
className="mt-4 px-4 py-2 bg-indigo-600 text-white rounded-lg hover:bg-indigo-700 disabled:bg-gray-400 transition-colors"
|
|||
|
|
>
|
|||
|
|
{loading ? 'Analyzing Speech...' : 'Simulate Speaking Input'}
|
|||
|
|
</button>
|
|||
|
|
|
|||
|
|
{feedback && (
|
|||
|
|
<div className="mt-6 p-4 bg-slate-50 rounded-lg border-l-4 border-amber-500">
|
|||
|
|
<div className="flex justify-between items-center">
|
|||
|
|
<h3 className="font-bold text-gray-800">Pronunciation Feedback</h3>
|
|||
|
|
<span className="px-2 py-1 bg-amber-100 text-amber-800 font-mono text-sm rounded">
|
|||
|
|
Score: {feedback.score}/5
|
|||
|
|
</span>
|
|||
|
|
</div>
|
|||
|
|
<p className="mt-2 text-sm text-gray-700 font-medium">
|
|||
|
|
<span className="font-semibold text-gray-900">Observation:</span> {feedback.errorIdentified}
|
|||
|
|
</p>
|
|||
|
|
<p className="mt-1 text-sm text-gray-600 italic">
|
|||
|
|
<span className="font-semibold text-gray-900 not-italic">How to correct:</span> {feedback.actionableAdvice}
|
|||
|
|
</p>
|
|||
|
|
</div>
|
|||
|
|
)}
|
|||
|
|
</div>
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=15>What This Strategy Unlocks</h3><ol start=1 data-path-to-node=16><li><p data-path-to-node=16,0,0><b data-path-to-node=16,0,0 data-index-in-node=0>Clean Project Separation:</b> Your Next.js workspace remains lightweight. It doesn't care if <code data-path-to-node=16,0,0 data-index-in-node=89>llama.cpp</code> is written in C++ or running on a completely separate server box on your network; it just sends standard network requests over HTTP.</p><li><p data-path-to-node=16,1,0><b data-path-to-node=16,1,0 data-index-in-node=0>Deterministic UI Rendering:</b> Because Zod guarantees the structure of the data coming into your component, you never have to worry about standard LLM text anomalies (like unexpected trailing strings or bad Markdown parsing) blowing up your React component rendering logic.</p><li><p data-path-to-node=16,2,0><b data-path-to-node=16,2,0 data-index-in-node=0>Smooth UX:</b> While the local LLM is crunching token strings on the background hardware, React handles the interface loading states dynamically, providing a fluid user experience.</p></ol><p data-path-to-node=17>This structure bridges the gap between raw machine learning binaries and clean, production-grade application engineering. How are you thinking about managing the layout—will you embed the EPUB document text selection right inside this web interface?</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-572 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-573 hide-action-bar ng-tns-c3135600295-572 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-573"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-573 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-573 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_a9e11da211aee173","c_4089c2ad7c8dde16",null,"rc_d745e6a2f12a1262",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-573 luminous-theme ng-star-inserted"
|
|||
|
|
id String @id @default(uuid())
|
|||
|
|
title String
|
|||
|
|
filePath String // Path to the stored file or unpacked EPUB text
|
|||
|
|
content String @db.Text
|
|||
|
|
|
|||
|
|
// Localized metadata
|
|||
|
|
locale String @default("es-ES") // Enforces BCP 47 strings
|
|||
|
|
region String? // Optional descriptive field: e.g., "Andalucía", "CDMX"
|
|||
|
|
|
|||
|
|
createdAt DateTime @default(now())
|
|||
|
|
updatedAt DateTime @updatedAt
|
|||
|
|
|
|||
|
|
@@index([locale]) // Highly efficient queries when fetching Spanish-only texts
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=12>3. Automating the Text-to-Speech Engine via Metadata</h3><p data-path-to-node=13>Storing BCP 47 values simplifies your server-side API logic. When a user opens a document to practice their pronunciation, your Next.js route can map the database <code data-path-to-node=13 data-index-in-node=163>locale</code> directly to the corresponding voice model parameter for <b data-path-to-node=13 data-index-in-node=226>Piper</b> or <b data-path-to-node=13 data-index-in-node=235>Kokoro</b>.</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-673 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-673 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_57d6442303392bb5","c_4089c2ad7c8dde16",null,"rc_21a1e58bda47dd59",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-673"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-673"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-673 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-673>TypeScript</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-673 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-673 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-673 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_57d6442303392bb5","c_4089c2ad7c8dde16",null,"rc_21a1e58bda47dd59",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></s
|
|||
|
|
<span class=hljs-keyword>import</span> { getPiperModelPath } <span class=hljs-keyword>from</span> <span class=hljs-string>'@/utils/audio-mapping'</span>;
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>export</span> <span class=hljs-keyword>async</span> <span class=hljs-function><span class=hljs-keyword>function</span> <span class=hljs-title>prepareSpeechTarget</span>(<span class=hljs-params>documentId: <span class=hljs-built_in>string</span></span>) </span>{
|
|||
|
|
<span class=hljs-keyword>const</span> doc = <span class=hljs-keyword>await</span> db.document.findUnique({ <span class=hljs-attr>where</span>: { <span class=hljs-attr>id</span>: documentId } });
|
|||
|
|
<span class=hljs-keyword>if</span> (!doc) <span class=hljs-keyword>throw</span> <span class=hljs-keyword>new</span> <span class=hljs-built_in>Error</span>(<span class=hljs-string>"Document not found"</span>);
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// Dynamically resolve voice paths based on database localization tags</span>
|
|||
|
|
<span class=hljs-keyword>const</span> voiceConfig = {
|
|||
|
|
<span class=hljs-attr>text</span>: doc.content.substring(<span class=hljs-number>0</span>, <span class=hljs-number>200</span>), <span class=hljs-comment>// Pull initial segment</span>
|
|||
|
|
<span class=hljs-attr>modelFile</span>: doc.locale === <span class=hljs-string>'es-MX'</span> ? <span class=hljs-string>'es_MX-medium.onnx'</span> : <span class=hljs-string>'es_ES-low.onnx'</span>,
|
|||
|
|
<span class=hljs-attr>speed</span>: <span class=hljs-number>1.0</span>
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>return</span> voiceConfig;
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=15>4. Injecting the Metadata into the Front-End DOM</h3><p data-path-to-node=16>Storing this metadata benefits the user experience and interface accessibility. When Next.js serves the document content to your React text editor or reading panel, pass that locale directly into your HTML tags.</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-674 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-674 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_57d6442303392bb5","c_4089c2ad7c8dde16",null,"rc_21a1e58bda47dd59",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=9><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-674"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-674"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-674 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-674>TypeScript</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-674 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-674 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-674 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_57d6442303392bb5","c_4089c2ad7c8dde16",null,"rc_21a1e58bda47dd59",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color
|
|||
|
|
export default function ReadingPanel({ document }: { document: Document }) {
|
|||
|
|
return (
|
|||
|
|
<article className="prose max-w-none">
|
|||
|
|
<h1 className="text-xl font-bold">{document.title}</h1>
|
|||
|
|
|
|||
|
|
{/* Setting the lang attribute optimizes browser text-to-speech accessibility extensions
|
|||
|
|
and ensures proper dictionary hyphenation rules are applied natively.
|
|||
|
|
*/}
|
|||
|
|
<div lang={document.locale} className="p-4 bg-white border font-serif text-lg leading-relaxed">
|
|||
|
|
{document.content}
|
|||
|
|
</div>
|
|||
|
|
</article>
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=18>Next Steps for Your CMS Expansion</h3><p data-path-to-node=19>Since your backend already supports managing file paths and blobs, adding this field allows you to filter your document collection screen using a localized query: <code data-path-to-node=19 data-index-in-node=163>const documents = await db.document.findMany({ where: { locale: 'es-ES' } })</code>.<p data-path-to-node=20>This setup allows you to categorize materials systematically, such as grouping documents under an <code data-path-to-node=20 data-index-in-node=98>es-MX</code> tag for specific target dialects.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete has-thoughts gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-574 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-575 hide-action-bar ng-tns-c3135600295-574 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-575"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-575 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-575 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_57d6442303392bb5","c_4089c2ad7c8dde16",null,"rc_21a1e58bda47dd59",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-575 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-
|
|||
|
|
id String @id @default(uuid())
|
|||
|
|
name String @unique // e.g., "Gemma-4-12B-Balanced"
|
|||
|
|
modelPath String // e.g., "models/gemma-4-12b-it-Q4_K_M.gguf"
|
|||
|
|
port Int @default(8080)
|
|||
|
|
contextSize Int @default(4096) // The -c flag
|
|||
|
|
gpuLayers Int @default(99) // The --n-gpu-layers flag
|
|||
|
|
|
|||
|
|
// Advanced flags stored as JSON for future-proofing
|
|||
|
|
additionalFlags Json? // e.g., { "--threads": 8, "--flash-attn": true }
|
|||
|
|
|
|||
|
|
isActive Boolean @default(false)
|
|||
|
|
createdAt DateTime @default(now())
|
|||
|
|
updatedAt DateTime @updatedAt
|
|||
|
|
|
|||
|
|
@@index([isActive])
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=8>2. Runtime Validation & Command Generator</h3><p data-path-to-node=9>Using your target stack (<b data-path-to-node=9 data-index-in-node=25>Prisma + Zod</b>), you can create a utility class that extracts the configuration row from Postgres, validates the types, and compiles the clean CLI startup string.</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-676 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-676 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_68bead323aac4f4e","c_4089c2ad7c8dde16",null,"rc_618023398f493abe",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-676"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-676"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-676 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-676>TypeScript</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-676 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-676 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-676 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_68bead323aac4f4e","c_4089c2ad7c8dde16",null,"rc_618023398f493abe",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font
|
|||
|
|
<span class=hljs-keyword>import</span> { z } <span class=hljs-keyword>from</span> <span class=hljs-string>'zod'</span>;
|
|||
|
|
<span class=hljs-keyword>import</span> { PrismaClient } <span class=hljs-keyword>from</span> <span class=hljs-string>'@prisma/client'</span>;
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>const</span> prisma = <span class=hljs-keyword>new</span> PrismaClient();
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// 1. Validate incoming DB configurations using Zod to ensure CLI safety</span>
|
|||
|
|
<span class=hljs-keyword>export</span> <span class=hljs-keyword>const</span> LlamaConfigValidator = z.object({
|
|||
|
|
<span class=hljs-attr>modelPath</span>: z.string().min(<span class=hljs-number>1</span>),
|
|||
|
|
<span class=hljs-attr>port</span>: z.number().int().min(<span class=hljs-number>1024</span>).max(<span class=hljs-number>65535</span>),
|
|||
|
|
<span class=hljs-attr>contextSize</span>: z.number().int().positive(),
|
|||
|
|
<span class=hljs-attr>gpuLayers</span>: z.number().int().nonnegative(),
|
|||
|
|
<span class=hljs-attr>additionalFlags</span>: z.record(z.any()).nullable().optional(),
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>export</span> <span class=hljs-keyword>async</span> <span class=hljs-function><span class=hljs-keyword>function</span> <span class=hljs-title>generateServerCommand</span>(<span class=hljs-params>configName: <span class=hljs-built_in>string</span></span>): <span class=hljs-title>Promise</span><<span class=hljs-title>string</span>> </span>{
|
|||
|
|
<span class=hljs-keyword>const</span> dbConfig = <span class=hljs-keyword>await</span> prisma.llamaConfig.findUnique({
|
|||
|
|
<span class=hljs-attr>where</span>: { <span class=hljs-attr>name</span>: configName }
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>if</span> (!dbConfig) <span class=hljs-keyword>throw</span> <span class=hljs-keyword>new</span> <span class=hljs-built_in>Error</span>(<span class=hljs-string>`Model configuration "<span class=hljs-subst>${configName}</span>" not found.`</span>);
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// 2. Structural parsing with Zod</span>
|
|||
|
|
<span class=hljs-keyword>const</span> clean = LlamaConfigValidator.parse(dbConfig);
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// 3. Systematically construct the execution string</span>
|
|||
|
|
<span class=hljs-keyword>let</span> command = <span class=hljs-string>`./llama-server -m <span class=hljs-subst>${clean.modelPath}</span> --port <span class=hljs-subst>${clean.port}</span> -c <span class=hljs-subst>${clean.contextSize}</span> --n-gpu-layers <span class=hljs-subst>${clean.gpuLayers}</span>`</span>;
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// 4. Safely parse optional/variable arguments stored in the JSON block</span>
|
|||
|
|
<span class=hljs-keyword>if</span> (clean.additionalFlags) {
|
|||
|
|
<span class=hljs-keyword>for</span> (<span class=hljs-keyword>const</span> [flag, value] <span class=hljs-keyword>of</span> <span class=hljs-built_in>Object</span>.entries(clean.additionalFlags)) {
|
|||
|
|
<span class=hljs-keyword>if</span> (<span class=hljs-keyword>typeof</span> value === <span class=hljs-string>'boolean'</span> && value === <span class=hljs-literal>true</span>) {
|
|||
|
|
command += <span class=hljs-string>` <span class=hljs-subst>${flag}</span>`</span>;
|
|||
|
|
} <span class=hljs-keyword>else</span> <span class=hljs-keyword>if</span> (value !== <span class=hljs-literal>null</span> && value !== <span class=hljs-literal>false</span>) {
|
|||
|
|
command += <span class=hljs-string>` <span class=hljs-subst>${flag}</span> <span class=hljs-subst>${value}</span>`</span>;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>return</span> command;
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=12>3. What This Architecture Unlocks in Your CMS</h3><p data-path-to-node=13>By moving the runtime architecture out of an isolated <code data-path-to-node=13 data-index-in-node=54>.sh</code> file and into Prisma, your Next.js application gains massive functional capabilities:<h4 data-path-to-node=14>A. Smart Context Switching</h4><p data-path-to-node=15>When processing short text strings for basic vocabulary matching, your backend can pull a database row targeting <code data-path-to-node=15 data-index-in-node=113>contextSize: 2048</code> to preserve system memory. When an expansive <code data-path-to-node=15 data-index-in-node=176>.epub</code> file is loaded into the viewer, the system can instantly reload the execution call targeting a <code data-path-to-node=15 data-index-in-node=277>contextSize: 32768</code> configuration row to absorb the document depth.<h4 data-path-to-node=16>B. Centralized Dashboard Control</h4><p data-path-to-node=17>Because configuration strings live inside PostgreSQL, you can easily build a hidden settings pane directly inside your Next.js CMS frontend. You can adjust the context footprint, throttle threads, or switch models cleanly from a browser window without touching code or logging into an SSH window.<h4 data-path-to-node=18>C. Health Checks and Failovers</h4><p data-path-to-node=19>Your Next.js API server can keep tabs on which server configuration is marked <code data-path-to-node=19 data-index-in-node=78>isActive: true</code>. If an execution instance hits a memory wall or a port clash occurs, an automated routine can flip the active bit in PostgreSQL to transition background traffic cleanly to an alternative <code data-path-to-node=19 data-index-in-node=280>.gguf</code> file profile.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-576 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-577 hide-action-bar ng-tns-c3135600295-576 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-577"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-577 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-577 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_68bead323aac4f4e","c_4089c2ad7c8dde16",null,"rc_618023398f493abe",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-
|
|||
|
|
{
|
|||
|
|
<span class=hljs-attr>"system_prompt"</span>: <span class=hljs-string>"You are a precise engineering assistant. Respond using strict markdown format."</span>,
|
|||
|
|
<span class=hljs-attr>"parameters"</span>: {
|
|||
|
|
<span class=hljs-attr>"temperature"</span>: <span class=hljs-number>0.2</span>,
|
|||
|
|
<span class=hljs-attr>"top_k"</span>: <span class=hljs-number>40</span>,
|
|||
|
|
<span class=hljs-attr>"top_p"</span>: <span class=hljs-number>0.95</span>,
|
|||
|
|
<span class=hljs-attr>"repeat_penalty"</span>: <span class=hljs-number>1.1</span>,
|
|||
|
|
<span class=hljs-attr>"ctx_size"</span>: <span class=hljs-number>8192</span>
|
|||
|
|
},
|
|||
|
|
<span class=hljs-attr>"engine_options"</span>: {
|
|||
|
|
<span class=hljs-attr>"num_predict"</span>: <span class=hljs-number>-1</span>,
|
|||
|
|
<span class=hljs-attr>"flash_attn"</span>: <span class=hljs-literal>true</span>
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-678 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-678 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_68ad8e815626a28d","c_4089c2ad7c8dde16",null,"rc_26696448fd4bb2e7",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=8><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-678"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-678"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-678 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-678>JSON</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-678 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-678 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-678 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_68ad8e815626a28d","c_4089c2ad7c8dde16",null,"rc_26696448fd4bb2e7",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=copy data-mat-icon-namespace=lumi-symbols fonticon=copy></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></div></div><pre _ngcontent-ng-c2960786434
|
|||
|
|
{
|
|||
|
|
<span class=hljs-attr>"system_prompt"</span>: <span class=hljs-string>"You are a creative copywriter."</span>,
|
|||
|
|
<span class=hljs-attr>"parameters"</span>: {
|
|||
|
|
<span class=hljs-attr>"temperature"</span>: <span class=hljs-number>0.7</span>,
|
|||
|
|
<span class=hljs-attr>"presence_penalty"</span>: <span class=hljs-number>0.5</span>,
|
|||
|
|
<span class=hljs-attr>"max_tokens"</span>: <span class=hljs-number>2048</span>
|
|||
|
|
},
|
|||
|
|
<span class=hljs-attr>"api_options"</span>: {
|
|||
|
|
<span class=hljs-attr>"frequency_penalty"</span>: <span class=hljs-number>0.0</span>
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h2 data-path-to-node=18>Technical Trade-offs to Keep in Mind</h2><p data-path-to-node=19>Before committing to this, look at both sides of the coin:<h3 data-path-to-node=20>🟩 The Good</h3><ul data-path-to-node=21><li><p data-path-to-node=21,0,0><b data-path-to-node=21,0,0 data-index-in-node=0>Zero Migrations:</b> You can add, remove, or modify parameters on the fly without database downtime.</p><li><p data-path-to-node=21,1,0><b data-path-to-node=21,1,0 data-index-in-node=0>Polymorphic UI:</b> Your application frontend can read the <code data-path-to-node=21,1,0 data-index-in-node=55>config</code> object and dynamically render sliders only for the parameters that actually exist for that specific model.</p><li><p data-path-to-node=21,2,0><b data-path-to-node=21,2,0 data-index-in-node=0>Clean Context Isolation:</b> It keeps the database clean of dead columns that only apply to deprecated models.</p></ul><h3 data-path-to-node=22>🟥 The Gotchas</h3><ul data-path-to-node=23><li><p data-path-to-node=23,0,0><b data-path-to-node=23,0,0 data-index-in-node=0>Loss of Database-Level Validation:</b> The database won't natively stop you from writing a typo like <code data-path-to-node=23,0,0 data-index-in-node=97>"tempreture": 0.7</code>.<ul data-path-to-node=23,0,1><li><p data-path-to-node=23,0,1,0,0><i data-path-to-node=23,0,1,0,0 data-index-in-node=0>The Fix:</i> Validate the incoming payload in your application layer using a runtime validation library (like Zod if you are using TypeScript/Next.js, or Pydantic if you are in Python) before saving it to the JSON column.</p></ul><li><p data-path-to-node=23,1,0><b data-path-to-node=23,1,0 data-index-in-node=0>Harder to Aggregate:</b> If you ever want to run a dashboard query like <i data-path-to-node=23,1,0 data-index-in-node=68>"What is the average temperature across all my configurations?"</i>, querying inside JSON fields is slightly more verbose than standard SQL columns (though modern PostgreSQL handles this quite well using operators like <code data-path-to-node=23,1,0 data-index-in-node=283>config->'parameters'->>'temperature'</code>).</p></ul><p data-path-to-node=24>Using a JSON configuration block gives your local LLM testing or production setups the exact elasticity they need to keep pace with changing model architectures.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-578 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-579 hide-action-bar ng-tns-c3135600295-578 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-579"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-579 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-579 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-b
|
|||
|
|
git <span class=hljs-built_in>clone</span> https://github.com/ggml-org/llama.cpp
|
|||
|
|
|
|||
|
|
<span class=hljs-comment># New infrastructure workflow isolated on your local network:</span>
|
|||
|
|
git <span class=hljs-built_in>clone</span> http://192.168.1.XX:3000/your-username/llama.cpp
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=21>This ensures that even if your external internet gateway drops out entirely, your internal engineering pipeline can still fetch, rebuild, and configure new local model instances directly from your Forgejo server.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete has-thoughts gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-580 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-581 hide-action-bar ng-tns-c3135600295-580 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-581"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-581 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-581 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_db98274a69459086","c_4089c2ad7c8dde16",null,"rc_a8d18420a4fe1554",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-581 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_db98274a69459086","c_4089c2ad7c8dde16",null,"rc_a8d18420a4fe1554",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _ngho
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=10>A native system window will pop up asking for permission to install the <b data-path-to-node=10 data-index-in-node=72>Command Line Developer Tools</b>. This lightweight package (around 500MB) instantly installs <code data-path-to-node=10 data-index-in-node=161>clang</code>, <code data-path-to-node=10 data-index-in-node=168>make</code>, <code data-path-to-node=10 data-index-in-node=174>git</code>, and all the essential C system header files right into your environment path.<h3 data-path-to-node=11>4. What if you need a "Real" GNU GCC?</h3><p data-path-to-node=12>If you are compiling legacy infrastructure packages or academic code that strictly relies on GNU-specific compiler extensions or strict OpenMP parallelism structures that Clang doesn't mirror identically:<ol start=1 data-path-to-node=13><li><p data-path-to-node=13,0,0>Do not use the default <code data-path-to-node=13,0,0 data-index-in-node=23>gcc</code> command (as it is just Clang in disguise).</p><li><p data-path-to-node=13,1,0>Install the true GNU compiler via <b data-path-to-node=13,1,0 data-index-in-node=34>Homebrew</b> by running:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-681 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-681 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_c1a21037126e547f","c_4089c2ad7c8dde16",null,"rc_f3993079cedaafe3",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-681"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-681"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-681 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-681>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-681 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-681 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-681 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><li><p data-path-to-node=13,2,0>This will install the actual GNU compiler binaries, explicitly named by their version number (e.g., <code data-path-to-node=13,2,0 data-index-in-node=100>gcc-13</code> or <code data-path-to-node=13,2,0 data-index-in-node=110>gcc-14</code>), allowing you to preserve both environments safely without creating configuration collisions on your local machine.</p></ol><p data-path-to-node=15 data-sf-nesting-track-id=1.10.1.3.2.2.2.1.3.2.1.1.2.1.10.2.1.1.1.2.1.1.3.1.1.1.15><response-element ng-version=0.0.0-PLACEHOLDER><link-block _nghost-ng-c227220131 class=ng-star-inserted><a _ngcontent-ng-c227220131 target=_blank rel=noopener externallink _nghost-ng-c1003445734 jslog='197247;track:generic_click,impression,attention;BardVeMetadataKey:[["r_c1a21037126e547f","c_4089c2ad7c8dde16",null,"rc_f3993079cedaafe3",null,null,"en",null,1,null,null,1,0]]' href="https://www.youtube.com/watch?v=Nx-eFv5mePE" class=ng-star-inserted data-hveid=9>This guide on installing GCC and running C programs on macOS</a></link-block></response-element> walks step-by-step through setting up and verifying code execution across Apple Silicon platforms.<div class="attachment-container youtube" data-sf-nesting-track-id=1.10.1.3.2.2.2.1.3.2.1.1.2.1.10.2.1.1.1.2.1.1.3.1.1.1.15.2><response-element ng-version=0.0.0-PLACEHOLDER><youtube-block _nghost-ng-c3084883328 class=ng-star-inserted><attribution-container _ngcontent-ng-c3084883328 _nghost-ng-c1760471056><div _ngcontent-ng-c1760471056 hide-from-message-actions class="tool-attribution show-header hide-from-message-actions"><a _ngcontent-ng-c1760471056 target=_blank rel=noopener class=link href="https://www.youtube.com/watch?v=Nx-eFv5mePE"><div _ngcontent-ng-c1760471056 class="tool-attribution-content ng-star-inserted"><div _ngcontent-ng-c1760471056 class=tool-attribution-header-container><div _ngcontent-ng-c1760471056 class="tool-attribution-header gds-title-s"><img _ngcontent-ng-c1760471056 alt class="tool-logo ng-star-inserted" src="data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGVuYWJsZS1iYWNrZ3JvdW5kPSJuZXcgMCAwIDE5MiAxOTIiIGhlaWdodD0iMTkyIiB2aWV3Qm94PSIwIDAgMTkyIDE5MiIgd2lkdGg9IjE5MiI+PHJlY3QgZmlsbD0ibm9uZSIgaGVpZ2h0PSIxOTIiIHdpZHRoPSIxOTIiLz48Zz48Zz48cGF0aCBkPSJNMTc3LjQ0LDU1LjA5Yy0xLjk2LTcuMzEtNy43Mi0xMy4wOC0xNS4wMy0xNS4wM0MxNDkuMTcsMzYuNSw5NiwzNi41LDk2LDM2LjVzLTUzLjE3LDAtNjYuNDEsMy41NiBjLTcuMzEsMS45Ni0xMy4wOCw3LjcyLTE1LjAzLDE1LjAzQzExLDY4LjMzLDExLDk2LDExLDk2czAsMjcuNjcsMy41Niw0MC45MWMxLjk2LDcuMzEsNy43MiwxMy4wOCwxNS4wMywxNS4wMyBDNDIuODMsMTU1LjUsOTYsMTU1LjUsOTYsMTU1LjVzNTMuMTcsMCw2Ni40MS0zLjU2YzcuMzEtMS45NiwxMy4wOC03LjcyLDE1LjAzLTE1LjAzQzE4MSwxMjMuNjcsMTgxLDk2LDE4MSw5NiBTMTgxLDY4LjMzLDE3Ny40NCw1NS4wOXoiIGZpbGw9IiNGRjAwMDAiLz48cG9seWdvbiBmaWxsPSIjRkZGRkZGIiBwb2ludHM9Ijc5LDEyMS41IDEyMy4xNyw5NiA3OSw3MC41Ii8+PC9nPjwvZz48L3N2Zz4="><div _ngcontent-ng-c1760471056 class="tool-attribution-title ng-star-inserted"><span _ngcontent-ng-c1760471056 class=tool-attribution-title-text>C Programming Installation in Mac OS | (M1/M2/M3/M4/M5) | GCC - YouTube</span></div></div><div _ngcontent-ng-c1760471056 class="tool-attribution-label gds-body-m ng-star-inserted">Saddam Tech · 695 views</div></div></div></a><div _ngcontent-ng-c3084883328 role=presentation class="youtube-block ng-star-inserted"><single-video _ngcontent-ng-c3084883328 class="youtube-item last-item ng-star-inserted" _nghost-ng-c2408101700><default-player _ngcontent-ng-c2408101700 _nghost-ng-c801375417 class=ng-star-inserted><div _ngcontent-ng-c801375417 role=presentation class=single-view-container jslog='270365;track:impression;BardVeMetadataKey:[["r_c1a21037126e547f","c_4089c2ad7c8dde16",null,null,null,null,null,null,1,null,null,null,0]]'><div _ngcontent-ng-c801375417 role=presentation class=single-video-container><div _ngcontent-ng-c801375417 role=presentation class="single-video-thumbnail ng-star-inserted"><img _ngcontent-ng-c801375417 aria-hidden=true class=thumbnail src="data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAMCAg
|
|||
|
|
git <span class=hljs-built_in>clone</span> https://github.com/ggml-org/llama.cpp
|
|||
|
|
<span class=hljs-built_in>cd</span> llama.cpp
|
|||
|
|
|
|||
|
|
<span class=hljs-comment># 2. Compile the binaries using 'make'</span>
|
|||
|
|
<span class=hljs-comment># By default, llama.cpp automatically detects macOS and enables Metal acceleration.</span>
|
|||
|
|
make
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=8>During the <code data-path-to-node=8 data-index-in-node=11>make</code> execution, you will see hundreds of lines of compilation logs pass by. Clang is compiling the core C/C++ files and linking them directly to your Mac's internal frameworks.<p data-path-to-node=9>Once it finishes, you will have a fresh, optimized <code data-path-to-node=9 data-index-in-node=51>./llama-server</code> binary sitting right in that folder, built specifically by your local compiler to squeeze every bit of performance out of your hardware.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-584 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-585 hide-action-bar ng-tns-c3135600295-584 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-585"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-585 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-585 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_9ce1c0bbfbb5be63","c_4089c2ad7c8dde16",null,"rc_74ae65e855ed9550",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-585 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='1
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=21>Setting <code data-path-to-node=21 data-index-in-node=8>--n-gpu-layers 99</code> guarantees Apple's Metal API hooks every matrix dimension straight into the M3's GPU cores, yielding conversational generation speeds around <b data-path-to-node=21 data-index-in-node=167>25 to 35 tokens per second</b> while leaving plenty of memory headroom for the rest of your system.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response complete has-thoughts gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-586 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-587 hide-action-bar ng-tns-c3135600295-586 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-587"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-587 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-587 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_3ea520f2acdb9a9c","c_4089c2ad7c8dde16",null,"rc_6ab33338d9946cc8",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-587 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_3ea520f2acdb9a9c","c_4089c2ad7c8dde16",null,"rc_6ab
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=8><i data-path-to-node=8 data-index-in-node=0>(Alternatively, if you use Python/pip in your workflow, you can run: <code data-path-to-node=8 data-index-in-node=69>pip install huggingface_hub</code>)</i><h4 data-path-to-node=9>2. Execute the Targeted Download Command</h4><p data-path-to-node=10>Navigate to your compiled <code data-path-to-node=10 data-index-in-node=26>llama.cpp</code> directory, create a <code data-path-to-node=10 data-index-in-node=56>models</code> folder if you haven't already, and run the targeted download command to pull the specific Q5 GGUF file:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-685 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-685 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_d6dd441e92aa7ffa","c_4089c2ad7c8dde16",null,"rc_67593761fa7b39d5",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-685"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-685"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-685 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-685>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-685 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-685 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-685 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_d6dd441e92aa7ffa","c_4089c2ad7c8dde16",null,"rc_67593761fa7b39d5",null,null,"en",null,1,null,null,1,0]];mutable
|
|||
|
|
<span class=hljs-built_in>cd</span> llama.cpp/models
|
|||
|
|
|
|||
|
|
<span class=hljs-comment># Download only the specific 5-bit quantized GGUF file</span>
|
|||
|
|
huggingface-cli download google/gemma-4-12b-it-GGUF gemma-4-12b-it-Q5_K_M.gguf --local-dir . --local-dir-use-symlinks False
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><ul data-path-to-node=12><li><p data-path-to-node=12,0,0><b data-path-to-node=12,0,0 data-index-in-node=0><code data-path-to-node=12,0,0 data-index-in-node=0>google/gemma-4-12b-it-GGUF</code></b>: Specifies the official Google repository hosting the GGUF conversions.</p><li><p data-path-to-node=12,1,0><b data-path-to-node=12,1,0 data-index-in-node=0><code data-path-to-node=12,1,0 data-index-in-node=0>gemma-4-12b-it-Q5_K_M.gguf</code></b>: The specific filename for the 5-bit medium quantization, optimizing performance on your 18GB hardware layout.</p><li><p data-path-to-node=12,2,0><b data-path-to-node=12,2,0 data-index-in-node=0><code data-path-to-node=12,2,0 data-index-in-node=0>--local-dir .</code></b>: Tells the CLI to place the downloaded file straight into your current folder instead of hiding it deep within a global cache directory.</p></ul><h3 data-path-to-node=14>Method 2: Direct Web Browser Download</h3><p data-path-to-node=15>If you prefer to download the asset through your browser and move it manually, you can pull it straight from the repository interface.<ol start=1 data-path-to-node=16><li><p data-path-to-node=16,0,0>Open your web browser and navigate directly to the model repository:
|
|||
|
|
<code data-path-to-node=16,0,0 data-index-in-node=69>https://huggingface.co/google/gemma-4-12b-it-GGUF</code></p><li><p data-path-to-node=16,1,0>Click on the <b data-path-to-node=16,1,0 data-index-in-node=13>Files and versions</b> tab near the top of the page.</p><li><p data-path-to-node=16,2,0>Scroll through the file list to locate <b data-path-to-node=16,2,0 data-index-in-node=39><code data-path-to-node=16,2,0 data-index-in-node=39>gemma-4-12b-it-Q5_K_M.gguf</code></b>.</p><li><p data-path-to-node=16,3,0>Click the small <b data-path-to-node=16,3,0 data-index-in-node=16>download icon</b> (the downward-pointing arrow) on the right side of the file row.</p><li><p data-path-to-node=16,4,0>Once the download completes, move the file from your <code data-path-to-node=16,4,0 data-index-in-node=53>~/Downloads</code> directory into your active project workspace:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-686 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-686 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_d6dd441e92aa7ffa","c_4089c2ad7c8dde16",null,"rc_67593761fa7b39d5",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=9><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-686"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-686"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-686 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-686>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-686 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-686 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-686 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-ce
|
|||
|
|
</code></pre></div></div></div></code-block></response-element></ol><h3 data-path-to-node=18>Verifying and Launching the Downloaded Asset</h3><p data-path-to-node=19>Once the file transfer finishes, confirm that the binary size is roughly <b data-path-to-node=19 data-index-in-node=73>9GB to 10GB</b> to ensure it didn't truncate mid-download:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-687 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-687 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_d6dd441e92aa7ffa","c_4089c2ad7c8dde16",null,"rc_67593761fa7b39d5",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=12><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-687"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-687"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-687 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-687>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-687 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-687 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-687 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_d6dd441e92aa7ffa","c_4089c2ad7c8dde16",null,"rc_67593761fa7b39d5",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true dat
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=21>With the file safely in place, verify the end-to-end setup by spinning up your local open standard standard API server using the compiled <code data-path-to-node=21 data-index-in-node=138>llama-server</code> binary:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-688 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-688 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_d6dd441e92aa7ffa","c_4089c2ad7c8dde16",null,"rc_67593761fa7b39d5",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=15><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-688"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-688"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-688 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-688>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-688 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-688 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-688 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_d6dd441e92aa7ffa","c_4089c2ad7c8dde16",null,"rc_67593761fa7b39d5",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=c
|
|||
|
|
./llama-server -m models/gemma-4-12b-it-Q5_K_M.gguf --port 8080 -c 8192 --n-gpu-layers 99
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=23>Your local background service will initialize the weights directly inside your Mac's unified memory cache, mapping everything via the Metal API layer. It is now completely listening and ready to accept structured JSON payloads from your Next.js application route handler.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-588 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-589 hide-action-bar ng-tns-c3135600295-588 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-589"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-589 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-589 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_d6dd441e92aa7ffa","c_4089c2ad7c8dde16",null,"rc_67593761fa7b39d5",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-589 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_d6dd441e92aa7ffa","c_4089c2ad7c8dde16",null,"rc_67593761fa7b39d5",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"><
|
|||
|
|
-H <span class=hljs-string>"Content-Type: application/json"</span> \
|
|||
|
|
-H <span class=hljs-string>"Authorization: Bearer local-testing"</span> \
|
|||
|
|
-d <span class=hljs-string>'{
|
|||
|
|
"model": "gemma-4-12b",
|
|||
|
|
"messages": [
|
|||
|
|
{
|
|||
|
|
"role": "user",
|
|||
|
|
"content": "¿Cómo se dice '</span>\<span class=hljs-string>''</span>the dog runs fast<span class=hljs-string>'\'</span><span class=hljs-string>' en español?"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"temperature": 0.3
|
|||
|
|
}'</span>
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=5>Key Details to Notice in the Command</h3><ul data-path-to-node=6><li><p data-path-to-node=6,0,0><b data-path-to-node=6,0,0 data-index-in-node=0>The Endpoint (<code data-path-to-node=6,0,0 data-index-in-node=14>/v1/chat/completions</code>):</b> This is the exact open standard path used by the official OpenAI libraries. Your Next.js backend will target this exact structure.</p><li><p data-path-to-node=6,1,0><b data-path-to-node=6,1,0 data-index-in-node=0>The Authorization Header:</b> Because <code data-path-to-node=6,1,0 data-index-in-node=34>llama-server</code> is running locally on your machine without access keys, you can pass any string here (like <code data-path-to-node=6,1,0 data-index-in-node=138>"Bearer local-testing"</code> or <code data-path-to-node=6,1,0 data-index-in-node=164>"Bearer ollama"</code>). It acts as a valid placeholder so standard HTTP clients don't complain.</p><li><p data-path-to-node=6,2,0><b data-path-to-node=6,2,0 data-index-in-node=0>Escaping Quotes in Bash:</b> Notice how the Spanish test string wraps the English words inside <code data-path-to-node=6,2,0 data-index-in-node=91>'\''</code>. This ensures Bash parses the nested quotes properly inside the JSON payload data block (<code data-path-to-node=6,2,0 data-index-in-node=185>-d</code>).</p></ul><h3 data-path-to-node=7>What a Successful Response Looks Like</h3><p data-path-to-node=8>If your M3 is processing the data cleanly via the Metal API, your terminal will instantly receive a structured JSON object back from the server:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-690 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-690 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_2a879aa08abdd171","c_4089c2ad7c8dde16",null,"rc_b21fc468b60fee1d",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-690"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-690"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-690 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-690>JSON</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-690 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-690 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1
|
|||
|
|
<span class=hljs-attr>"id"</span>: <span class=hljs-string>"chatcmpl-12345..."</span>,
|
|||
|
|
<span class=hljs-attr>"object"</span>: <span class=hljs-string>"chat.completion"</span>,
|
|||
|
|
<span class=hljs-attr>"created"</span>: <span class=hljs-number>1717612945</span>,
|
|||
|
|
<span class=hljs-attr>"model"</span>: <span class=hljs-string>"gemma-4-12b"</span>,
|
|||
|
|
<span class=hljs-attr>"choices"</span>: [
|
|||
|
|
{
|
|||
|
|
<span class=hljs-attr>"index"</span>: <span class=hljs-number>0</span>,
|
|||
|
|
<span class=hljs-attr>"message"</span>: {
|
|||
|
|
<span class=hljs-attr>"role"</span>: <span class=hljs-string>"assistant"</span>,
|
|||
|
|
<span class=hljs-attr>"content"</span>: <span class=hljs-string>"Se dice: \"El perro corre rápido.\""</span>
|
|||
|
|
},
|
|||
|
|
<span class=hljs-attr>"finish_reason"</span>: <span class=hljs-string>"stop"</span>
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
<span class=hljs-attr>"usage"</span>: {
|
|||
|
|
<span class=hljs-attr>"prompt_tokens"</span>: <span class=hljs-number>17</span>,
|
|||
|
|
<span class=hljs-attr>"completion_tokens"</span>: <span class=hljs-number>9</span>,
|
|||
|
|
<span class=hljs-attr>"total_tokens"</span>: <span class=hljs-number>26</span>
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=10>If you get this JSON payload back in your terminal, your local infrastructure layer is 100% functional. Your local server is ready, optimized for your hardware, and waiting for you to plug it into your Next.js application routes.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-590 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-591 hide-action-bar ng-tns-c3135600295-590 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-591"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-591 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-591 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_2a879aa08abdd171","c_4089c2ad7c8dde16",null,"rc_b21fc468b60fee1d",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-591 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_2a879aa08abdd171","c_4089c2ad7c8dde16",null,"rc_b21fc468b60fee1d",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _
|
|||
|
|
<span class=hljs-comment># or</span>
|
|||
|
|
./llama-server --version
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=3>What the Output Tells You</h3><p data-path-to-node=4>When you run it, <code data-path-to-node=4 data-index-in-node=17>llama.cpp</code> doesn't just print a simple number like <code data-path-to-node=4 data-index-in-node=67>v1.0.0</code> (since the project updates almost daily via rolling GitHub commits). Instead, it outputs highly detailed compilation metadata:<ol start=1 data-path-to-node=5><li><p data-path-to-node=5,0,0><b data-path-to-node=5,0,0 data-index-in-node=0>The Build Number:</b> A unique incremental release count showing exactly how many updates have been pushed to the main codebase since inception.</p><li><p data-path-to-node=5,1,0><b data-path-to-node=5,1,0 data-index-in-node=0>The Git Commit Hash:</b> The specific alphanumeric string (e.g., <code data-path-to-node=5,1,0 data-index-in-node=61>b345aef</code>) representing the exact snapshot of the source code you cloned onto your machine.</p><li><p data-path-to-node=5,2,0><b data-path-to-node=5,2,0 data-index-in-node=0>The Local Compiler Stack:</b> It explicitly logs the compiler engine used to build it (<b data-path-to-node=5,2,0 data-index-in-node=83>Apple Clang</b> along with your specific Xcode toolchain version).</p><li><p data-path-to-node=5,3,0><b data-path-to-node=5,3,0 data-index-in-node=0>The Hardware Optimization Flag:</b> It will indicate whether hardware acceleration features like <b data-path-to-node=5,3,0 data-index-in-node=93>Metal</b> (for your M3's GPU/VRAM matrix operations) are actively linked into the binary execution layer.</p></ol><p data-path-to-node=6>This output provides a quick way to audit your deployment pipeline, verifying that your binaries are fully updated and built with optimization flags before launching your Next.js network routes.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response complete has-thoughts gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-592 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-593 hide-action-bar ng-tns-c3135600295-592 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-593"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-593 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-593 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_7f6164d98406f15f","c_4089c2ad7c8dde16",null,"rc_398546b5cd1c7b72",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat
|
|||
|
|
-H <span class=hljs-string>"Content-Type: application/json"</span> \
|
|||
|
|
-H <span class=hljs-string>"Authorization: Bearer local-testing"</span> \
|
|||
|
|
-d <span class=hljs-string>'{
|
|||
|
|
"model": "gemma-4-12b",
|
|||
|
|
"messages": [
|
|||
|
|
{
|
|||
|
|
"role": "system",
|
|||
|
|
"content": "You are a Spanish text analyzer. You must output raw JSON matching the requested keys."
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"role": "user",
|
|||
|
|
"content": "Analyze the sentence: '</span>\<span class=hljs-string>''</span>El tren es rápido.<span class=hljs-string>'\'</span><span class=hljs-string>' Output keys: '</span>\<span class=hljs-string>''</span>word_count<span class=hljs-string>'\'</span><span class=hljs-string>' and '</span>\<span class=hljs-string>''</span>has_accent<span class=hljs-string>'\'</span><span class=hljs-string>'."
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"max_tokens": 256,
|
|||
|
|
"temperature": 0.0,
|
|||
|
|
"top_p": 0.9,
|
|||
|
|
"min_p": 0.05,
|
|||
|
|
"presence_penalty": 0.0,
|
|||
|
|
"frequency_penalty": 0.0,
|
|||
|
|
"response_format": { "type": "json_object" },
|
|||
|
|
"stop": ["User:", "\n\n"]
|
|||
|
|
}'</span>
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=22>Because <code data-path-to-node=22 data-index-in-node=8>temperature</code> is clamped to <code data-path-to-node=22 data-index-in-node=34>0.0</code> and the <code data-path-to-node=22 data-index-in-node=46>response_format</code> is designated as <code data-path-to-node=22 data-index-in-node=79>json_object</code>, Gemma 4 will skip conversational introductions and write a clean, minified string object. This payload format is ideal for ingestion by your Next.js route handlers and verification via your Zod validation rules.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response complete has-thoughts gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-594 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-595 hide-action-bar ng-tns-c3135600295-594 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-595"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-595 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-595 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_64676beaefdeb423","c_4089c2ad7c8dde16",null,"rc_afa8314af7699264",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-595 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-l
|
|||
|
|
│ Human Voice ├─────────────────────►│ navigator.media... │
|
|||
|
|
│ (Acoustic Wave) │ Granted │ .getUserMedia() │
|
|||
|
|
└─────────────────┘ └──────────┬───────────┘
|
|||
|
|
│
|
|||
|
|
▼
|
|||
|
|
┌──────────────────────┐
|
|||
|
|
│ MediaStream │
|
|||
|
|
│ (Raw Audio Track) │
|
|||
|
|
└──────────────────────┘
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=11>When permission is granted, this method returns a <b data-path-to-node=11 data-index-in-node=50><code data-path-to-node=11 data-index-in-node=50>MediaStream</code></b> object. This stream contains a raw, live <code data-path-to-node=11 data-index-in-node=103>MediaStreamTrack</code> representing the audio buffer coming off the microphone capsule at a default sample rate (typically 44.1kHz or 16kHz depending on system settings).</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-694 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-694 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_b6ffadb45ed4dae5","c_4089c2ad7c8dde16",null,"rc_5d09e8458fbb30bb",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=4><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-694"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-694"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-694 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-694>TypeScript</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-694 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-694 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-694 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_b6ffadb45ed4dae5","c_4089c2ad7c8dde16",null,"rc_5d09e8458fbb30bb",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c40
|
|||
|
|
<span class=hljs-keyword>async</span> <span class=hljs-function><span class=hljs-keyword>function</span> <span class=hljs-title>startMicrophone</span>(<span class=hljs-params></span>) </span>{
|
|||
|
|
<span class=hljs-keyword>try</span> {
|
|||
|
|
<span class=hljs-keyword>const</span> stream = <span class=hljs-keyword>await</span> navigator.mediaDevices.getUserMedia({
|
|||
|
|
<span class=hljs-attr>audio</span>: {
|
|||
|
|
<span class=hljs-attr>echoCancellation</span>: <span class=hljs-literal>true</span>, <span class=hljs-comment>// Crucial if playing audio via speakers simultaneously</span>
|
|||
|
|
<span class=hljs-attr>noiseSuppression</span>: <span class=hljs-literal>true</span>,
|
|||
|
|
<span class=hljs-attr>autoGainControl</span>: <span class=hljs-literal>false</span> <span class=hljs-comment>// Keep false if analyzing raw pronunciation amplitude</span>
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
<span class=hljs-built_in>console</span>.log(<span class=hljs-string>"Microphone linked successfully:"</span>, stream.id);
|
|||
|
|
<span class=hljs-keyword>return</span> stream;
|
|||
|
|
} <span class=hljs-keyword>catch</span> (err) {
|
|||
|
|
<span class=hljs-built_in>console</span>.error(<span class=hljs-string>"Microphone access denied or unavailable:"</span>, err);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h2 data-path-to-node=14>3. Accessing the Speakers (Output)</h2><p data-path-to-node=15>Playing sound back to the user is managed via the <b data-path-to-node=15 data-index-in-node=50>Web Audio API</b>. This system operates as a <b data-path-to-node=15 data-index-in-node=91>directed audio graph</b>, where modules (nodes) are chained together like virtual guitar pedals.</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-695 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-695 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_b6ffadb45ed4dae5","c_4089c2ad7c8dde16",null,"rc_5d09e8458fbb30bb",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=7><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-695"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-695"><pre _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-695><code _ngcontent-ng-c2960786434 role=text data-test-id=code-content class="code-container formatted ng-tns-c2960786434-695 no-decoration-radius">┌───────────────────────┐ ┌─────────────────┐ ┌─────────────────────────┐
|
|||
|
|
│ Audio Source │─────►│ GainNode │─────►│ AudioDestinationNode │
|
|||
|
|
│ (Oscillator / Buffer) │ │ (Volume Slider) │ │ (System Default Speaker)│
|
|||
|
|
└───────────────────────┘ └─────────────────┘ └─────────────────────────┘
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=17>The absolute end point of any Web Audio graph is <b data-path-to-node=17 data-index-in-node=49><code data-path-to-node=17 data-index-in-node=49>audioContext.destination</code></b>. This is a special, built-in node that automatically maps directly to the operating system's default audio playback device (your laptop speakers or connected headphones).</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-696 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-696 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_b6ffadb45ed4dae5","c_4089c2ad7c8dde16",null,"rc_5d09e8458fbb30bb",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=8><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-696"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-696"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-696 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-696>TypeScript</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-696 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-696 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-696 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_b6ffadb45ed4dae5","c_4089c2ad7c8dde16",null,"rc_5d09e8458fbb30bb",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslat
|
|||
|
|
<span class=hljs-keyword>const</span> audioContext = <span class=hljs-keyword>new</span> AudioContext();
|
|||
|
|
|
|||
|
|
<span class=hljs-function><span class=hljs-keyword>function</span> <span class=hljs-title>playSynthesizedTone</span>(<span class=hljs-params></span>) </span>{
|
|||
|
|
<span class=hljs-comment>// 1. Create a sound generator node (Oscillator)</span>
|
|||
|
|
<span class=hljs-keyword>const</span> oscillator = audioContext.createOscillator();
|
|||
|
|
oscillator.type = <span class=hljs-string>'sine'</span>;
|
|||
|
|
oscillator.frequency.setValueAtTime(<span class=hljs-number>440</span>, audioContext.currentTime); <span class=hljs-comment>// 440Hz standard A note</span>
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// 2. Connect the generator node to the physical speaker node</span>
|
|||
|
|
oscillator.connect(audioContext.destination);
|
|||
|
|
|
|||
|
|
<span class=hljs-comment>// 3. Trigger playback</span>
|
|||
|
|
oscillator.start();
|
|||
|
|
oscillator.stop(audioContext.currentTime + <span class=hljs-number>1.0</span>); <span class=hljs-comment>// Stop after 1 second</span>
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h2 data-path-to-node=20>4. Advanced Control: Targeted Routing (<code data-path-to-node=20 data-index-in-node=39>setSinkId</code>)</h2><p data-path-to-node=21>By default, <code data-path-to-node=21 data-index-in-node=12>audioContext.destination</code> blindly targets whatever output device the operating system dictates. However, if you are building an interactive coaching or communications system, you may want to let the user select a <i data-path-to-node=21 data-index-in-node=224>specific</i> speaker (e.g., routing pronunciation feedback to headphones while keeping system alerts on the laptop chassis).<p data-path-to-node=22>This is achieved via the <b data-path-to-node=22 data-index-in-node=25>Audio Output Devices API</b> using the <code data-path-to-node=22 data-index-in-node=60>setSinkId()</code> method.<ol start=1 data-path-to-node=23><li><p data-path-to-node=23,0,0><b data-path-to-node=23,0,0 data-index-in-node=0>Enumerate Devices:</b> The app requests a structural map of the physical hardware chips connected to the system.</p><li><p data-path-to-node=23,1,0><b data-path-to-node=23,1,0 data-index-in-node=0>Filter Out Outputs:</b> It targets entries flagged specifically as <code data-path-to-node=23,1,0 data-index-in-node=63>kind: 'audiooutput'</code>.</p><li><p data-path-to-node=23,2,0><b data-path-to-node=23,2,0 data-index-in-node=0>Route the Stream:</b> It locks the context output directly to that hardware hash identifier.</p></ol><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-697 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-697 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_b6ffadb45ed4dae5","c_4089c2ad7c8dde16",null,"rc_5d09e8458fbb30bb",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=11><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-697"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-697"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-697 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-697>TypeScript</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-697 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-697 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-te
|
|||
|
|
<span class=hljs-comment>// Request permission to examine output devices</span>
|
|||
|
|
<span class=hljs-keyword>const</span> devices = <span class=hljs-keyword>await</span> navigator.mediaDevices.enumerateDevices();
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>const</span> targetSpeaker = devices.find(
|
|||
|
|
<span class=hljs-function><span class=hljs-params>device</span> =></span> device.kind === <span class=hljs-string>'audiooutput'</span> && device.label.includes(targetDeviceName)
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
<span class=hljs-keyword>if</span> (targetSpeaker && <span class=hljs-string>'setSinkId'</span> <span class=hljs-keyword>in</span> audioContext) {
|
|||
|
|
<span class=hljs-comment>// Force the entire Web Audio API graph destination to emit from this specific hardware node</span>
|
|||
|
|
<span class=hljs-keyword>await</span> (audioContext <span class=hljs-keyword>as</span> <span class=hljs-built_in>any</span>).setSinkId(targetSpeaker.deviceId);
|
|||
|
|
<span class=hljs-built_in>console</span>.log(<span class=hljs-string>`Audio successfully routed directly to: <span class=hljs-subst>${targetSpeaker.label}</span>`</span>);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=25>The Acoustic Conundrum: Echo Control</h3><p data-path-to-node=26>When a single web page opens both channels simultaneously (capturing from the mic while screaming audio out of the speakers), it can create an instantaneous acoustic loopback.<p data-path-to-node=27>To prevent loud feedback squeals or microphone bleed-through from ruining pronunciation data analytics, browsers use an underlying C++ DSP architecture that executes <b data-path-to-node=27 data-index-in-node=166>Hardware Echo Cancellation</b>. By passing <code data-path-to-node=27 data-index-in-node=205>{ echoCancellation: true }</code> inside your microphone request configuration, the browser automatically samples the output graph, matches its waveform frequency, and subtracts it out of the incoming microphone buffer wave before handing the array data over to your JavaScript environment.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response complete has-thoughts gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-598 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-599 hide-action-bar ng-tns-c3135600295-598 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-599"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-599 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-599 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_b6ffadb45ed4dae5","c_4089c2ad7c8dde16",null,"rc_5d09e8458fbb30bb",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-599 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 matic
|
|||
|
|
│ State 1: The Gateway Gatekeeper (Landing / Activation) │
|
|||
|
|
├────────────────────────────────────────────────────────┤
|
|||
|
|
│ State 2: Hardware Configuration (Device Selection) │
|
|||
|
|
├────────────────────────────────────────────────────────┤
|
|||
|
|
│ State 3: Core Audio Session Loop (The Workspace) │
|
|||
|
|
└────────────────────────────────────────────────────────┘
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=8>Section 1: The Activation Gatekeeper (State 1)</h3><p data-path-to-node=9>Browsers strictly block access to audio chips until a explicit user gesture occurs. Your page must load with a clean, un-intrusive initialization view.<ul data-path-to-node=10><li><p data-path-to-node=10,0,0><b data-path-to-node=10,0,0 data-index-in-node=0>UI Element:</b> A prominent <b data-path-to-node=10,0,0 data-index-in-node=24>"Initialize Audio Studio"</b> or <b data-path-to-node=10,0,0 data-index-in-node=53>"Start Lesson"</b> button.</p><li><p data-path-to-node=10,1,0><b data-path-to-node=10,1,0 data-index-in-node=0>Underlying Logic:</b> Clicking this button executes the initial <code data-path-to-node=10,1,0 data-index-in-node=60>navigator.mediaDevices.getUserMedia()</code> request. This forces the browser's native permissions popup to drop down. Until they click this button, the rest of your application's audio processing machinery stays dormant.</p></ul><h3 data-path-to-node=11>Section 2: Hardware Verification & Calibration (State 2)</h3><p data-path-to-node=12>Once permissions are granted, you must ensure the computer is routing audio from the correct microphone and out to the correct speakers (e.g., forcing audio through bone-conduction earbuds rather than blasting the laptop chassis speakers).<ul data-path-to-node=13><li><p data-path-to-node=13,0,0><b data-path-to-node=13,0,0 data-index-in-node=0>UI Element - Dropdown Selectors:</b> Two html <code data-path-to-node=13,0,0 data-index-in-node=42><select></code> dropdown inputs that automatically populate with a list of the user's active hardware names.</p><li><p data-path-to-node=13,1,0><b data-path-to-node=13,1,0 data-index-in-node=0>UI Element - The Visual VU Meter:</b> A small horizontal audio level indicator bar that bounces dynamically when the user speaks.</p><li><p data-path-to-node=13,2,0><b data-path-to-node=13,2,0 data-index-in-node=0>Underlying Logic:</b> * The code runs <code data-path-to-node=13,2,0 data-index-in-node=34>navigator.mediaDevices.enumerateDevices()</code> to scan for available inputs (<code data-path-to-node=13,2,0 data-index-in-node=106>audioinput</code>) and outputs (<code data-path-to-node=13,2,0 data-index-in-node=131>audiooutput</code>).<ul data-path-to-node=13,2,1><li><p data-path-to-node=13,2,1,0,0>To drive the visual VU meter, the raw microphone buffer is funneled through an <code data-path-to-node=13,2,1,0,0 data-index-in-node=79>AnalyserNode</code> using the Web Audio API, calculating the volume amplitude in real-time. This provides instant visual confirmation that the mic is receiving data before they start a Spanish drill.</p></ul></ul><h3 data-path-to-node=14>Section 3: The Active Training Loop Workspace (State 3)</h3><p data-path-to-node=15>Once calibrated, the UI shifts to display your primary training environment. For your pronunciation application, this section layout requires three clear control interfaces:<ul data-path-to-node=16><li><p data-path-to-node=16,0,0><b data-path-to-node=16,0,0 data-index-in-node=0>The System Player Trigger:</b> A button (often stylized as a speaker icon) that triggers <b data-path-to-node=16,0,0 data-index-in-node=85>Kokoro</b> or <b data-path-to-node=16,0,0 data-index-in-node=95>Piper</b> via your Next.js server route to stream the target Spanish sentence out through the designated speaker path.</p><li><p data-path-to-node=16,1,0><b data-path-to-node=16,1,0 data-index-in-node=0>The Record/Stop Interactive Toggle:</b> A toggle button that controls audio stream serialization.<ul data-path-to-node=16,1,1><li><p data-path-to-node=16,1,1,0,0>When active, it changes color (e.g., a pulsing red ring) to signal that raw microphone PCM data chunks are buffering.</p><li><p data-path-to-node=16,1,1,1,0>When clicked again, it halts recording, packages the raw audio bytes into a clean <code data-path-to-node=16,1,1,1,0 data-index-in-node=82>Blob</code> container, and transmits it via a standard <code data-path-to-node=16,1,1,1,0 d
|
|||
|
|
│ MUI Card Container │
|
|||
|
|
│ │
|
|||
|
|
│ ┌──────────────────┐ ┌────────────────────┐ │
|
|||
|
|
│ │ Select Microphone│ │ Select Headset │ │
|
|||
|
|
│ └──────────────────┘ └────────────────────┘ │
|
|||
|
|
│ │
|
|||
|
|
│ ┌──────────────────────────────────────────────────┐ │
|
|||
|
|
│ │ [ vu-meter-react ] │ │
|
|||
|
|
│ │ Analog Needle Dial │ │
|
|||
|
|
│ └──────────────────────────────────────────────────┘ │
|
|||
|
|
│ │
|
|||
|
|
│ ┌────────────────┐ │
|
|||
|
|
│ │ Mic On/Off │ │
|
|||
|
|
│ └────────────────┘ │
|
|||
|
|
└────────────────────────────────────────────────────────┘
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=7>Core Integration Example (TypeScript)</h3><p data-path-to-node=8>Here is a structural implementation showing how to bind your application's low-level microphone stream processing loop natively to a custom Material UI practice widget card:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-700 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-700 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_9115c6b4cdb7e2be","c_4089c2ad7c8dde16",null,"rc_0ebbf37955229bce",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=4><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-700"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-700"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-700 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-700>TypeScript</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-700 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-700 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-700 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_9115c6b4cdb7e2be","c_4089c2ad7c8dde16",null,"rc_0ebbf37955229bce",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-ty
|
|||
|
|
|
|||
|
|
import React, { useState, useRef } from 'react';
|
|||
|
|
import { Card, CardContent, Button, Typography, Box, Stack } from '@mui/material';
|
|||
|
|
import MicIcon from '@mui/icons-material/Mic';
|
|||
|
|
import MicOffIcon from '@mui/icons-material/MicOff';
|
|||
|
|
import { VUMeter } from 'vu-meter-react';
|
|||
|
|
|
|||
|
|
export default function AudioInitializationCard() {
|
|||
|
|
const [isListening, setIsListening] = useState(false);
|
|||
|
|
|
|||
|
|
// Persistent references to the Web Audio pipeline objects
|
|||
|
|
const audioContextRef = useRef<AudioContext | null>(null);
|
|||
|
|
const sourceNodeRef = useRef<MediaStreamAudioSourceNode | null>(null);
|
|||
|
|
const streamRef = useRef<MediaStream | null>(null);
|
|||
|
|
|
|||
|
|
const startAudioPipeline = async () => {
|
|||
|
|
try {
|
|||
|
|
// 1. Initialize the shared local audio context core
|
|||
|
|
if (!audioContextRef.current) {
|
|||
|
|
audioContextRef.current = new (window.AudioContext || (window as any).webkitAudioContext)();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const ctx = audioContextRef.current;
|
|||
|
|
if (ctx.state === 'suspended') {
|
|||
|
|
await ctx.resume();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 2. Request local permission hooks for the microphone capture
|
|||
|
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|||
|
|
audio: { echoCancellation: true, noiseSuppression: true }
|
|||
|
|
});
|
|||
|
|
streamRef.current = stream;
|
|||
|
|
|
|||
|
|
// 3. Connect the hardware stream buffer into a Web Audio Node graph link
|
|||
|
|
sourceNodeRef.current = ctx.createMediaStreamSource(stream);
|
|||
|
|
|
|||
|
|
setIsListening(true);
|
|||
|
|
} catch (err) {
|
|||
|
|
console.error("Failed to initialize system audio devices:", err);
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const stopAudioPipeline = () => {
|
|||
|
|
// Teardown stream nodes systematically to avoid memory leaks or locked mics
|
|||
|
|
if (streamRef.current) {
|
|||
|
|
streamRef.current.getTracks().forEach(track => track.stop());
|
|||
|
|
}
|
|||
|
|
setIsListening(false);
|
|||
|
|
sourceNodeRef.current = null;
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
return (
|
|||
|
|
<Card sx={{ maxWidth: 400, mx: 'auto', mt: 4, boxShadow: 3, borderRadius: 3 }}>
|
|||
|
|
<CardContent>
|
|||
|
|
<Typography variant="h6" component="div" sx={{ fontWeight: 'bold', mb: 1 }}>
|
|||
|
|
Hardware Calibration
|
|||
|
|
</Typography>
|
|||
|
|
<Typography variant="body2" color="text.secondary" sx={{ mb: 3 }}>
|
|||
|
|
Speak into your microphone to calibrate input level before starting training.
|
|||
|
|
</Typography>
|
|||
|
|
|
|||
|
|
{/* The VU Meter Interface Container */}
|
|||
|
|
<Box sx={{
|
|||
|
|
display: 'flex',
|
|||
|
|
justifyContent: 'center',
|
|||
|
|
alignItems: 'center',
|
|||
|
|
p: 2,
|
|||
|
|
bgcolor: 'background.default',
|
|||
|
|
borderRadius: 2,
|
|||
|
|
minHeight: 140,
|
|||
|
|
border: '1px solid',
|
|||
|
|
borderColor: 'divider'
|
|||
|
|
}}>
|
|||
|
|
{isListening && audioContextRef.current && sourceNodeRef.current ? (
|
|||
|
|
<VUMeter
|
|||
|
|
audioContext={audioContextRef.current}
|
|||
|
|
sourceNode={sourceNodeRef.current}
|
|||
|
|
mono
|
|||
|
|
referenceLevel={-18} // Maps -18 dBFS straight to 0 VU alignment
|
|||
|
|
options={{
|
|||
|
|
width: 280,
|
|||
|
|
theme: 'light',
|
|||
|
|
fontFamily: 'Roboto, sans-serif'
|
|||
|
|
}}
|
|||
|
|
/>
|
|||
|
|
) : (
|
|||
|
|
<Typography variant="body2" color="text.disabled" italic>
|
|||
|
|
Audio hardware inactive
|
|||
|
|
</Typography>
|
|||
|
|
)}
|
|||
|
|
</Box>
|
|||
|
|
|
|||
|
|
<Stack direction="row" spacing={2} sx={{ mt: 3, justifyContent: 'flex-end' }}>
|
|||
|
|
{isListening ? (
|
|||
|
|
<Button
|
|||
|
|
variant="outlined"
|
|||
|
|
color="error"
|
|||
|
|
startIcon={<MicOffIcon />}
|
|||
|
|
onClick={stopAudioPipeline}
|
|||
|
|
>
|
|||
|
|
Disconnect
|
|||
|
|
</Button>
|
|||
|
|
) : (
|
|||
|
|
<Button
|
|||
|
|
variant="contained"
|
|||
|
|
color="primary"
|
|||
|
|
startIcon={<MicIcon />}
|
|||
|
|
onClick={startAudioPipeline}
|
|||
|
|
>
|
|||
|
|
Test Microphone
|
|||
|
|
</Button>
|
|||
|
|
)}
|
|||
|
|
</Stack>
|
|||
|
|
</CardContent>
|
|||
|
|
</Card>
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=10>Engineering Details with <code data-path-to-node=10 data-index-in-node=25>vu-meter-react</code></h3><ul data-path-to-node=11><li><p data-path-to-node=11,0,0><b data-path-to-node=11,0,0 data-index-in-node=0>Ballistics and Smoothing:</b> A great feature of this specific layout implementation is its authentic rendering of <b data-path-to-node=11,0,0 data-index-in-node=111>VU Ballistics (~300ms integration window)</b> natively. Because it computes RMS (Root Mean Square) mathematical bounds behind the scenes using <code data-path-to-node=11,0,0 data-index-in-node=250>getFloatTimeDomainData()</code>, the dial doesn't jitter rapidly like a simple peak visualizer. It moves smoothly with natural voice cadence.</p><li><p data-path-to-node=11,1,0><b data-path-to-node=11,1,0 data-index-in-node=0>Calibration Point:</b> Setting <code data-path-to-node=11,1,0 data-index-in-node=27>referenceLevel={-18}</code> is an industry standard mapping pattern. It translates to telling the canvas handler that when your microphone signal averages an input of <code data-path-to-node=11,1,0 data-index-in-node=187>-18 dBFS</code> inside the compiler stack, the mechanical needle layout lines up perfectly with the centered <b data-path-to-node=11,1,0 data-index-in-node=289>0 VU</b> target. This prevents standard speaking ranges from pinning the needle to the maximum right threshold continuously.</p><li><p data-path-to-node=11,2,0><b data-path-to-node=11,2,0 data-index-in-node=0>Cleanup Management:</b> Notice how the <code data-path-to-node=11,2,0 data-index-in-node=35>stopAudioPipeline</code> loop iterates explicitly across the track objects (<code data-path-to-node=11,2,0 data-index-in-node=104>track.stop()</code>). Failing to do this causes the browser to keep the hardware recording light active indefinitely, even if the React elements drop off the display screen entirely.</p></ul></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete has-thoughts gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-602 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-603 hide-action-bar ng-tns-c3135600295-602 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-603"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-603 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-603 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_9115c6b4cdb7e2be","c_4089c2ad7c8dde16",null,"rc_0ebbf37955229bce",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=6><i data-path-to-node=6 data-index-in-node=0>(Don't miss the space and the <b data-path-to-node=6 data-index-in-node=30><code data-path-to-node=6 data-index-in-node=30>.</code></b> at the end. This tells Git: "use the current directory instead of naming a new one.")</i><h3 data-path-to-node=8>Scenario B: If your directory already has files (e.g., your Next.js project)</h3><p data-path-to-node=9>If you try the command above in a directory that already contains your <code data-path-to-node=9 data-index-in-node=71>app/</code>, <code data-path-to-node=9 data-index-in-node=77>components/</code>, or <code data-path-to-node=9 data-index-in-node=93>package.json</code> files, Git will halt and throw this error:
|
|||
|
|
<code data-path-to-node=9 data-index-in-node=149>fatal: destination path '.' already exists and is not an empty directory.</code><p data-path-to-node=10>To bypass this without messing up your main project structure, you have two excellent choices:<h4 data-path-to-node=11>Option 1: The "Init and Pull" Strategy (Keeps the <code data-path-to-node=11 data-index-in-node=50>.git</code> history)</h4><p data-path-to-node=12>This initializes a Git connection inside your folder, hooks it to the remote repository, and pulls the files down natively:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-702 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-702 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_071805c065f67e2a","c_4089c2ad7c8dde16",null,"rc_e7a84082cb9bc839",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-702"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-702"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-702 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-702>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-702 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-702 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-702 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_071805c065f67e2a","c_4089c2ad7c8dde16",null,"rc_e7a84082cb9bc839",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c4966
|
|||
|
|
git remote add origin https://github.com/ggml-org/llama.cpp
|
|||
|
|
git pull origin main
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h4 data-path-to-node=14>Option 2: Download the Source Only (No Git history)</h4><p data-path-to-node=15>If you just want the C/C++ source files, Makefiles, and tools to compile <code data-path-to-node=15 data-index-in-node=73>./llama-server</code>, and you <b data-path-to-node=15 data-index-in-node=97>don't</b> want the massive history of the <code data-path-to-node=15 data-index-in-node=135>llama.cpp</code> repository bloating your project, you can use <code data-path-to-node=15 data-index-in-node=191>tar</code> to stream the codebase straight into your directory:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-703 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-703 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_071805c065f67e2a","c_4089c2ad7c8dde16",null,"rc_e7a84082cb9bc839",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=9><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-703"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-703"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-703 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-703>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-703 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-703 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-703 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_071805c065f67e2a","c_4089c2ad7c8dde16",null,"rc_e7a84082cb9bc839",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="ma
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><ul data-path-to-node=17><li><p data-path-to-node=17,0,0><b data-path-to-node=17,0,0 data-index-in-node=0>What this does:</b> It grabs the latest snapshot of the code as a compressed archive, extracts it instantly into your current directory, and drops the top-level folder prefix (<code data-path-to-node=17,0,0 data-index-in-node=172>--strip-components=1</code>) so everything sits cleanly at your root level without bringing a secondary <code data-path-to-node=17,0,0 data-index-in-node=269>.git</code> folder along for the ride.</p></ul></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-604 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-605 hide-action-bar ng-tns-c3135600295-604 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-605"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-605 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-605 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_071805c065f67e2a","c_4089c2ad7c8dde16",null,"rc_e7a84082cb9bc839",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-605 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;tr
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=6>Breaking Down the New Syntax</h3><ul data-path-to-node=7><li><p data-path-to-node=7,0,0><b data-path-to-node=7,0,0 data-index-in-node=0><code data-path-to-node=7,0,0 data-index-in-node=0>hf download google/gemma-4-12b-it-GGUF</code></b>: This tells the new core runner which repository layout to stream from.</p><li><p data-path-to-node=7,1,0><b data-path-to-node=7,1,0 data-index-in-node=0><code data-path-to-node=7,1,0 data-index-in-node=0>--include "gemma-4-12b-it-Q5_K_M.gguf"</code></b>: Instead of typing the file name as a loose argument string, the new CLI uses an explicit <code data-path-to-node=7,1,0 data-index-in-node=129>--include</code> filter block to pull down only that specific <code data-path-to-node=7,1,0 data-index-in-node=184>.gguf</code> slice (ignoring the other quantization sizes in the repository).</p><li><p data-path-to-node=7,2,0><b data-path-to-node=7,2,0 data-index-in-node=0><code data-path-to-node=7,2,0 data-index-in-node=0>--local-dir .</code></b>: Instructs the system to dump the file right into your active working directory instead of caching it inside a global system folder.</p></ul><p data-path-to-node=8>Once the download finishes, you can spin up your <code data-path-to-node=8 data-index-in-node=49>./llama-server</code> command exactly as before.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-606 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-607 hide-action-bar ng-tns-c3135600295-606 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-607"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-607 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-607 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_663fafdc7e54ef0e","c_4089c2ad7c8dde16",null,"rc_dee7649c23c20087",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-607 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=7>Option 2: Using Homebrew</h3><p data-path-to-node=8>If you prefer keeping your terminal utilities centralized through Mac package management, you can pull it directly from Homebrew:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-706 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-706 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_7e160660b9988c1a","c_4089c2ad7c8dde16",null,"rc_edb4237865ded67f",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-706"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-706"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-706 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-706>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-706 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-706 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-706 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_7e160660b9988c1a","c_4089c2ad7c8dde16",null,"rc_edb4237865ded67f",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=copy data-mat-icon-namespace=lumi-sy
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=10>Option 3: Python Pip Upgrade (Alternative)</h3><p data-path-to-node=11>If you manage your machine setups via Python environments, the binary also ships bundled natively inside the core developer SDK package. Simply updating your global package structure will drop the <code data-path-to-node=11 data-index-in-node=197>hf</code> binary alias straight into your path:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-707 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-707 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_7e160660b9988c1a","c_4089c2ad7c8dde16",null,"rc_edb4237865ded67f",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=9><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-707"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-707"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-707 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-707>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-707 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-707 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-707 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_7e160660b9988c1a","c_4089c2ad7c8dde16",null,"rc_edb4237865ded67f",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=14>Verifying and Updating the Toolchain</h3><p data-path-to-node=15>Once installed, verify that your terminal path links cleanly to the engine by requesting its software version state:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-708 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-708 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_7e160660b9988c1a","c_4089c2ad7c8dde16",null,"rc_edb4237865ded67f",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=12><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-708"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-708"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-708 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-708>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-708 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-708 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-708 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_7e160660b9988c1a","c_4089c2ad7c8dde16",null,"rc_edb4237865ded67f",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=copy data-mat-icon-namespace=lumi-
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=17>If it prints the active build release metadata, your local environment is fully configured.<blockquote data-path-to-node=18><p data-path-to-node=18,0>💡 <b data-path-to-node=18,0 data-index-in-node=3>Pro-Tip for the New CLI:</b> If you ever need to pull an upstream bug fix or optimization patch down later, you no longer have to guess your original installation method. You can simply execute:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-709 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-709 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_7e160660b9988c1a","c_4089c2ad7c8dde16",null,"rc_edb4237865ded67f",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=15><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-709"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-709"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-709 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-709>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-709 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-709 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-709 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_7e160660b9988c1a","c_4089c2ad7c8dde16",null,"rc_edb4237865ded67f",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c40723
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=18,2>The binary will automatically audit your machine's system footprint (detecting whether it came from the standalone script, Homebrew, or Pip) and auto-execute the correct underlying update sequence seamlessly.</p></blockquote></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-608 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-609 hide-action-bar ng-tns-c3135600295-608 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-609"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-609 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-609 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_7e160660b9988c1a","c_4089c2ad7c8dde16",null,"rc_edb4237865ded67f",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-609 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_7e160660b9988c1a","c_4089c2ad7c8dde16",null,"rc_edb4237865ded67f",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=8><i data-path-to-node=8 data-index-in-node=0>(If your terminal path defaults to a modern Python 3 layout, you may need to run <code data-path-to-node=8 data-index-in-node=81>pip3 uninstall huggingface_hub</code> instead).</i><h4 data-path-to-node=9>B. If you use Homebrew to manage your tooling:</h4><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-711 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-711 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_da59edab03aa62b8","c_4089c2ad7c8dde16",null,"rc_fa976f7c92797375",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-711"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-711"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-711 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-711>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-711 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-711 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-711 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_da59edab03aa62b8","c_4089c2ad7c8dde16",null,"rc_fa976f7c92797375",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color n
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h4 data-path-to-node=11>C. If you use the modern <code data-path-to-node=11 data-index-in-node=25>uv</code> python package runner:</h4><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-712 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-712 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_da59edab03aa62b8","c_4089c2ad7c8dde16",null,"rc_fa976f7c92797375",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=9><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-712"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-712"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-712 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-712>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-712 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-712 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-712 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_da59edab03aa62b8","c_4089c2ad7c8dde16",null,"rc_fa976f7c92797375",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=copy data-mat-icon-namespace=lumi-symbols fonticon=copy></mat-icon></gem-icon><span class=mat-focus-indicator
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=14>Step 2: Clean the Hidden Global Model Cache (Optional)</h3><p data-path-to-node=15>Uninstalling the CLI utility script only deletes the execution entry point; it <b data-path-to-node=15 data-index-in-node=79>does not</b> touch your physical hard drive cache directory.<p data-path-to-node=16>If you previously used <code data-path-to-node=16 data-index-in-node=23>huggingface-cli</code> to download large <code data-path-to-node=16 data-index-in-node=57>.safetensors</code> or <code data-path-to-node=16 data-index-in-node=73>.gguf</code> assets, they might still be taking up significant space deep inside your system path (<code data-path-to-node=16 data-index-in-node=165>~/.cache/huggingface/hub/</code>).<p data-path-to-node=17>If you want to completely erase the legacy file caches to free up space on your Mac's SSD, execute a manual removal string:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-713 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-713 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_da59edab03aa62b8","c_4089c2ad7c8dde16",null,"rc_fa976f7c92797375",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=12><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-713"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-713"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-713 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-713>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-713 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-713 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-713 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninit
|
|||
|
|
rm -rf ~/.cache/huggingface/hub/
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=19><i data-path-to-node=19 data-index-in-node=0>(Note: Your brand new native <code data-path-to-node=19 data-index-in-node=29>hf</code> utility downloads models cleanly using modern system layouts, meaning you can safely clear this older, deep-nested path without breaking your active <code data-path-to-node=19 data-index-in-node=181>./llama-server</code> links).</i></p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-610 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-611 hide-action-bar ng-tns-c3135600295-610 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-611"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-611 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-611 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_da59edab03aa62b8","c_4089c2ad7c8dde16",null,"rc_fa976f7c92797375",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-611 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_da59edab03aa62b8","c_4089c2ad7c8dde16",null,"rc_fa976f7c92797375",null,null,"en",null,1,null,nul
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=8>Step 2: Download Using the Valid Open Repository Path</h3><p data-path-to-node=9>Instead of trying to pass through Google's restricted endpoint, point your <code data-path-to-node=9 data-index-in-node=75>hf</code> downloader to <b data-path-to-node=9 data-index-in-node=92><code data-path-to-node=9 data-index-in-node=92>unsloth/gemma-4-12b-it-GGUF</code></b>.<p data-path-to-node=10>Execute this command to fetch the <b data-path-to-node=10 data-index-in-node=34>5-bit medium quantization (Q5_K_M)</b> directly into your active folder:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-715 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-715 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_98328a3954a0c9d0","c_4089c2ad7c8dde16",null,"rc_e95e2057cdf8f932",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-715"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-715"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-715 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-715>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-715 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-715 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-715 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_98328a3954a0c9d0","c_4089c2ad7c8dde16",null,"rc_e95e2057cdf8f932",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=12>Alternative Option (If you prefer the strict "google." filename mapping)</h3><p data-path-to-node=13>If you specifically want the file name to have the standard <code data-path-to-node=13 data-index-in-node=60>google.</code> naming convention inside your local filesystem, you can pull from the <b data-path-to-node=13 data-index-in-node=138>DevQuasar</b> mirror instead:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-716 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-716 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_98328a3954a0c9d0","c_4089c2ad7c8dde16",null,"rc_e95e2057cdf8f932",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=9><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-716"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-716"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-716 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-716>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-716 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-716 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-716 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_98328a3954a0c9d0","c_4089c2ad7c8dde16",null,"rc_e95e2057cdf8f932",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=16>Step 3: Run Your Verification Check</h3><p data-path-to-node=17>Once the progress bar completes, verify the integrity of your file download:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-717 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-717 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_98328a3954a0c9d0","c_4089c2ad7c8dde16",null,"rc_e95e2057cdf8f932",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=12><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-717"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-717"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-717 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-717>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-717 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-717 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-717 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_98328a3954a0c9d0","c_4089c2ad7c8dde16",null,"rc_e95e2057cdf8f932",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=copy data-mat-icon-namespace=lumi-symbols fonticon=copy></mat-icon></gem-ic
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=19>You should see a single, clean <code data-path-to-node=19 data-index-in-node=31>gemma-4-12b-it-Q5_K_M.gguf</code> binary file weighing in right around <b data-path-to-node=19 data-index-in-node=95>~9.5 GB</b>. Your compiled local server is now fully ready to spin up using your target parameters.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-612 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-613 hide-action-bar ng-tns-c3135600295-612 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-613"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-613 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-613 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_98328a3954a0c9d0","c_4089c2ad7c8dde16",null,"rc_e95e2057cdf8f932",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-613 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_98328a3954a0c9d0","c_4089c2ad7c8dde16",null,"rc_e95e2057cdf8f932",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple m
|
|||
|
|
<span class=hljs-built_in>cd</span> internal-weights
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=7>Step 2: Run the Open-Access Download Command</h3><p data-path-to-node=8>Now that you are in a completely empty, pristine folder (<code data-path-to-node=8 data-index-in-node=57>internal-weights</code>), invoke the <code data-path-to-node=8 data-index-in-node=87>hf</code> utility to fetch the exact 5-bit compressed model file.<p data-path-to-node=9><i data-path-to-node=9 data-index-in-node=0>(We will use the <b data-path-to-node=9 data-index-in-node=17>Unsloth</b> mirror setup, which doesn't require any private user keys or account registration forms).</i></p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-719 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-719 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_7fbed8a262779904","c_4089c2ad7c8dde16",null,"rc_7112293d740ae97a",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-719"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-719"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-719 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-719>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-719 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-719 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-719 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_7fbed8a262779904","c_4089c2ad7c8dde16",null,"rc_7112293d740ae97a",null,null,"en",null,1,null,null,1,0]];mutable:true'><s
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=11>Step 3: Verify the Binary Weight File</h3><p data-path-to-node=12>Once the download progress tracking clears out, check your local file list to confirm the asset landed safely:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-720 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-720 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_7fbed8a262779904","c_4089c2ad7c8dde16",null,"rc_7112293d740ae97a",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=9><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-720"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-720"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-720 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-720>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-720 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-720 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-720 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_7fbed8a262779904","c_4089c2ad7c8dde16",null,"rc_7112293d740ae97a",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=copy data-mat-icon-namespace=lumi-symbol
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=14>You should see exactly one large file listed:
|
|||
|
|
<code data-path-to-node=14 data-index-in-node=46>gemma-4-12b-it-Q5_K_M.gguf</code> (~9.5 GB)<h3 data-path-to-node=16>Step 4: Spin Up the Mac M3 Metal Backend Server</h3><p data-path-to-node=17>Now, step back out to the root directory where your compiled execution file resides, and spin up your local endpoint. We'll use your fresh path tracking adjustments:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-721 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-721 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_7fbed8a262779904","c_4089c2ad7c8dde16",null,"rc_7112293d740ae97a",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=12><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-721"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-721"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-721 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-721>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-721 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-721 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-721 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_7fbed8a262779904","c_4089c2ad7c8dde16",null,"rc_7112293d740ae97a",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-
|
|||
|
|
./llama-server -m models/internal-weights/gemma-4-12b-it-Q5_K_M.gguf --port 8080 -c 8192 --n-gpu-layers 99
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=19>Your terminal will instantly initialize, and you'll see your Mac's unified memory cache link directly to the hardware matrix layers. It is now fully ready to listen for your testing <code data-path-to-node=19 data-index-in-node=182>curl</code> scripts or Next.js app routes!</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-614 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-615 hide-action-bar ng-tns-c3135600295-614 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-615"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-615 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-615 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_7fbed8a262779904","c_4089c2ad7c8dde16",null,"rc_7112293d740ae97a",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-615 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_7fbed8a262779904","c_4089c2ad7c8dde16",null,"rc_7112293d740ae97a",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=9><i data-path-to-node=9 data-index-in-node=0>(The <code data-path-to-node=9 data-index-in-node=5>-j</code> flag tells your Mac to use all available CPU cores simultaneously, cutting compilation time down to under a minute).</i><h3 data-path-to-node=11>Step 2: Set Up Your Model Storage Directory</h3><p data-path-to-node=12>Now that you have a pristine, stock structure, let's create a dedicated folder inside the project to hold your heavy weight files, keeping it separated from the built-in vocabulary assets you saw earlier:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-723 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-723 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_443a7f43af095a67","c_4089c2ad7c8dde16",null,"rc_b9edf4f3ac790cde",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-723"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-723"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-723 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-723>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-723 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-723 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-723 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_443a7f43af095a67","c_4089c2ad7c8dde16",null,"rc_b9edf4f3ac790cde",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ri
|
|||
|
|
<span class=hljs-built_in>cd</span> models/production-weights
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=15>Step 3: Fetch the Gemma 4 Weights via the <code data-path-to-node=15 data-index-in-node=42>hf</code> Utility</h3><p data-path-to-node=16>Now, invoke your updated <code data-path-to-node=16 data-index-in-node=25>hf</code> tool inside this clean destination to download the 5-bit file slice from the open mirror repository:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-724 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-724 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_443a7f43af095a67","c_4089c2ad7c8dde16",null,"rc_b9edf4f3ac790cde",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=9><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-724"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-724"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-724 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-724>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-724 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-724 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-724 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_443a7f43af095a67","c_4089c2ad7c8dde16",null,"rc_b9edf4f3ac790cde",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=19>Step 4: Verify and Start Your Optimized API Backend</h3><p data-path-to-node=20>Once the transfer finishes, jump back up to your root directory (where your newly compiled <code data-path-to-node=20 data-index-in-node=91>llama-server</code> binary resides) and initiate your server background listener:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-725 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-725 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_443a7f43af095a67","c_4089c2ad7c8dde16",null,"rc_b9edf4f3ac790cde",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=12><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-725"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-725"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-725 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-725>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-725 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-725 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-725 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_443a7f43af095a67","c_4089c2ad7c8dde16",null,"rc_b9edf4f3ac790cde",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-co
|
|||
|
|
<span class=hljs-built_in>cd</span> ../..
|
|||
|
|
|
|||
|
|
<span class=hljs-comment># Launch the server utilizing full Metal layer mapping</span>
|
|||
|
|
./llama-server -m models/production-weights/gemma-4-12b-it-Q5_K_M.gguf --port 8080 -c 8192 --n-gpu-layers 99
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=22>When you see the terminal log show <code data-path-to-node=22 data-index-in-node=35>ggml_metal_init: allocating</code>, you are officially running a clean, natively compiled local engine. It's fully ready to process network payloads cleanly.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-616 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-617 hide-action-bar ng-tns-c3135600295-616 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-617"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-617 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-617 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_443a7f43af095a67","c_4089c2ad7c8dde16",null,"rc_b9edf4f3ac790cde",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-617 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_443a7f43af095a67","c_4089c2ad7c8dde16",null,"rc_b9edf4f3ac790cde",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=8>Alternatively, if you manually created a <code data-path-to-node=8 data-index-in-node=41>build</code> directory at the project root, you can safely delete <i data-path-to-node=8 data-index-in-node=100>that</i> folder and recreate it:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-727 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-727 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_ce34b85b7cb78357","c_4089c2ad7c8dde16",null,"rc_105022ad48645985",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-727"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-727"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-727 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-727>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-727 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-727 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-727 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_ce34b85b7cb78357","c_4089c2ad7c8dde16",null,"rc_105022ad48645985",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat
|
|||
|
|
mkdir build
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h4 data-path-to-node=10>For Next.js projects:</h4><p data-path-to-node=11>If you are trying to clear the development cache to fix a sticky build error, you should delete the hidden <code data-path-to-node=11 data-index-in-node=107>.next</code> folder in your project's root directory, not anything inside <code data-path-to-node=11 data-index-in-node=174>src</code>:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-728 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-728 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_ce34b85b7cb78357","c_4089c2ad7c8dde16",null,"rc_105022ad48645985",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=9><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-728"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-728"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-728 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-728>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-728 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-728 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-728 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_ce34b85b7cb78357","c_4089c2ad7c8dde16",null,"rc_105022ad48645985",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=13>If you actually want to delete everything in a folder</h3><p data-path-to-node=14>If you are entirely discarding a temporary folder and <i data-path-to-node=14 data-index-in-node=54>do</i> want it gone, it is safer to step out of the directory and delete the folder by name rather than using the <code data-path-to-node=14 data-index-in-node=164>*</code> wildcard. Using wildcards with <code data-path-to-node=14 data-index-in-node=197>rm -rf</code> is a common culprit for accidental data loss if you happen to be in the wrong terminal tab.</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-729 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-729 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_ce34b85b7cb78357","c_4089c2ad7c8dde16",null,"rc_105022ad48645985",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=12><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-729"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-729"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-729 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-729>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-729 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-729 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-729 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_ce34b85b7cb78357","c_4089c2ad7c8dde16",null,"rc_105022ad48645985",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-but
|
|||
|
|
rm -rf folder_name
|
|||
|
|
</code></pre></div></div></div></code-block></response-element></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-618 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-619 hide-action-bar ng-tns-c3135600295-618 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-619"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-619 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-619 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_ce34b85b7cb78357","c_4089c2ad7c8dde16",null,"rc_105022ad48645985",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-619 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_ce34b85b7cb78357","c_4089c2ad7c8dde16",null,"rc_105022ad48645985",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=7><i data-path-to-node=7 data-index-in-node=0>Alternatively, if you prefer the highly optimized quants provided by <code data-path-to-node=7 data-index-in-node=69>bartowski</code>, you can use:</i></p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-731 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-731 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_69c1058dd598d1fe","c_4089c2ad7c8dde16",null,"rc_90fd1e6d7bd027cb",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-731"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-731"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-731 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-731>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-731 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-731 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-731 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_69c1058dd598d1fe","c_4089c2ad7c8dde16",null,"rc_90fd1e6d7bd027cb",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=copy data-mat-icon-namespac
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=10>Option 2: Clean <code data-path-to-node=10 data-index-in-node=16>curl</code> or <code data-path-to-node=10 data-index-in-node=24>wget</code> download</h3><p data-path-to-node=11>If the <code data-path-to-node=11 data-index-in-node=7>huggingface-cli</code> wrapper continues to throw local directory context fits, you can completely bypass it and stream the file straight from the Hugging Face CDN using standard tools.<p data-path-to-node=12>Run either of these commands inside your <code data-path-to-node=12 data-index-in-node=41>models</code> folder:<p data-path-to-node=13><b data-path-to-node=13 data-index-in-node=0>Using curl:</b></p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-732 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-732 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_69c1058dd598d1fe","c_4089c2ad7c8dde16",null,"rc_90fd1e6d7bd027cb",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=9><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-732"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-732"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-732 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-732>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-732 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-732 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-732 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_69c1058dd598d1fe","c_4089c2ad7c8dde16",null,"
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=15><b data-path-to-node=15 data-index-in-node=0>Using wget:</b></p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-733 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-733 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_69c1058dd598d1fe","c_4089c2ad7c8dde16",null,"rc_90fd1e6d7bd027cb",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=12><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-733"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-733"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-733 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-733>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-733 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-733 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-733 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_69c1058dd598d1fe","c_4089c2ad7c8dde16",null,"rc_90fd1e6d7bd027cb",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=copy data-mat-icon-namespace=lumi-symbols fonticon=copy></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=17>Once the download finishes, you can check that it's sitting safely in your directory with a quick <code data-path-to-node=17 data-index-in-node=98>ls -lh</code>, and you'll be ready to compile and point <code data-path-to-node=17 data-index-in-node=147>llama-cli</code> or <code data-path-to-node=17 data-index-in-node=160>llama-server</code> right at it.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-620 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-621 hide-action-bar ng-tns-c3135600295-620 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-621"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-621 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-621 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_69c1058dd598d1fe","c_4089c2ad7c8dde16",null,"rc_90fd1e6d7bd027cb",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-621 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_69c1058dd598d1fe","c_4089c2ad7c8dde16",null,"rc_90fd1e6d7bd027cb",null,null,"",
|
|||
|
|
</code></pre></div></div></div></code-block></response-element></ul><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-735 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-735 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_8c60309aca666c31","c_4089c2ad7c8dde16",null,"rc_70a1db4576632c8c",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=8><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-735"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-735"><pre _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-735><code _ngcontent-ng-c2960786434 role=text data-test-id=code-content class="code-container formatted ng-tns-c2960786434-735 no-decoration-radius">* **Via Homebrew:**
|
|||
|
|
```bash
|
|||
|
|
brew install hf
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><ul data-path-to-node=10><li><p data-path-to-node=10,0,0><b data-path-to-node=10,0,0 data-index-in-node=0>Via Standalone Script:</b></p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-736 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-736 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_8c60309aca666c31","c_4089c2ad7c8dde16",null,"rc_70a1db4576632c8c",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=9><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-736"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-736"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-736 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-736>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-736 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-736 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-736 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_8c60309aca666c31","c_4089c2ad7c8dde16",null,"rc_70a1db4576632c8c",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=copy data-mat-icon-namespace=lumi-symbols fonticon=copy></mat-icon></gem-icon><span class=mat-focus-indicator></s
|
|||
|
|
</code></pre></div></div></div></code-block></response-element></ul><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-737 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-737 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_8c60309aca666c31","c_4089c2ad7c8dde16",null,"rc_70a1db4576632c8c",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=12><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-737"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-737"><pre _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-737><code _ngcontent-ng-c2960786434 role=text data-test-id=code-content class="code-container formatted ng-tns-c2960786434-737 no-decoration-radius">* **Via pip (Still bundled with the hub):**
|
|||
|
|
```bash
|
|||
|
|
pip install -U "huggingface_hub"
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=12>Common Download Syntax Examples</h3><p data-path-to-node=13>The <code data-path-to-node=13 data-index-in-node=4>hf download</code> command is much smarter about handling specific file targets or repository configurations natively:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-738 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-738 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_8c60309aca666c31","c_4089c2ad7c8dde16",null,"rc_70a1db4576632c8c",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=13><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-738"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-738"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-738 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-738>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-738 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-738 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-738 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_8c60309aca666c31","c_4089c2ad7c8dde16",null,"rc_70a1db4576632c8c",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat
|
|||
|
|
hf download stabilityai/stable-diffusion-2-1
|
|||
|
|
|
|||
|
|
<span class=hljs-comment># Download a specific file (like a GGUF or config)</span>
|
|||
|
|
hf download Qwen/Qwen2.5-7B-Instruct-GGUF qwen2.5-7b-instruct-q4_k_m.gguf
|
|||
|
|
|
|||
|
|
<span class=hljs-comment># Download a dataset or space using the explicit type flag</span>
|
|||
|
|
hf download karpathy/fineweb-edu-100b-shuffle --repo-type dataset
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=15>If you have any automated shell aliases, Makefiles, or deployment scripts pointing to the old binary, switching <code data-path-to-node=15 data-index-in-node=112>huggingface-cli download</code> to <code data-path-to-node=15 data-index-in-node=140>hf download</code> should clear up the deprecation bottlenecks.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-622 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-623 hide-action-bar ng-tns-c3135600295-622 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-623"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-623 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-623 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_8c60309aca666c31","c_4089c2ad7c8dde16",null,"rc_70a1db4576632c8c",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-623 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_8c60309aca666c31","c_4089c2ad7c8dde16",null,"rc_70a1db4576632c8c",null,null,"",null,1,null,null,1,0]];mutable:true'><span cl
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=15>Or, if you prefer to spin up a local, OpenAI-compatible server with a clean chat interface web UI in your browser:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-740 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-740 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_347310475384a60b","c_4089c2ad7c8dde16",null,"rc_1a39fc9680f2f847",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-740"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-740"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-740 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-740>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-740 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-740 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-740 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_347310475384a60b","c_4089c2ad7c8dde16",null,"rc_1a39fc9680f2f847",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=copy data-mat-icon-namespace=lumi-symbols fonticon=copy></mat-icon></gem-icon><span class=mat-focus-indicat
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=17>This will automatically pull the un-gated community quantum file into your cache and run it locally with zero accounts required.<p data-path-to-node=19><response-element ng-version=0.0.0-PLACEHOLDER><link-block _nghost-ng-c227220131 class=ng-star-inserted><a _ngcontent-ng-c227220131 target=_blank rel=noopener externallink _nghost-ng-c1003445734 jslog='197247;track:generic_click,impression,attention;BardVeMetadataKey:[["r_347310475384a60b","c_4089c2ad7c8dde16",null,"rc_1a39fc9680f2f847",null,null,"",null,1,null,null,1,0]]' href="https://www.youtube.com/watch?v=TTUD1B63Vq4" class=ng-star-inserted data-hveid=9>Gemma 4 12B Setup & Troubleshooting Guide</a></link-block></response-element> This video walk-through is highly relevant as it demonstrates how to properly route, configure, and troubleshoot runtime issues when pulling GGUF models locally onto your machine.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-624 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-625 hide-action-bar ng-tns-c3135600295-624 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-625"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-625 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-625 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_347310475384a60b","c_4089c2ad7c8dde16",null,"rc_1a39fc9680f2f847",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-625 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 matic
|
|||
|
|
</code></pre></div></div></div></code-block></structured-node-sequence></div></div></div><div class="sequence-event ng-star-inserted"><div hide-from-message-actions class="sequence-event-marker-container hide-from-message-actions"><div class="sequence-event-marker gds-body-s ng-star-inserted">2</div><div class="sequence-event-line ng-star-inserted"></div></div><div class=sequence-event-content><div hide-from-message-actions class=hide-from-message-actions><div class="sequence-event-title gds-emphasized-body-l">Generate the build configuration</div><div class="sequence-event-subtitle gds-extended-caption ng-star-inserted">Uses Apple Silicon Metal optimization</div></div><div class="sequence-event-description gds-body-l"><span only-show-to-message-actions data-test-id=sequence-export-header class=only-show-to-message-actions style=display:none>Uses Apple Silicon Metal optimization.</span><structured-node-sequence class=ng-star-inserted><structured-text class=ng-star-inserted><p class=ng-star-inserted>Use <code>cmake</code> to configure the build. Since you are on Apple Silicon, CMake automatically detects and configures it to use the Metal API, ensuring the model runs entirely on your Mac's integrated GPU.</p></structured-text><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-742 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-742 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_c0f16b74d2da6c59","c_4089c2ad7c8dde16",null,"rc_b42482ce46133ad3",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=7><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-742"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-742"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-742 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-742>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-742 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-742 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-742 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 m
|
|||
|
|
</code></pre></div></div></div></code-block></structured-node-sequence></div></div></div><div class="sequence-event ng-star-inserted"><div hide-from-message-actions class="sequence-event-marker-container hide-from-message-actions"><div class="sequence-event-marker gds-body-s ng-star-inserted">3</div><div class="sequence-event-line ng-star-inserted"></div></div><div class=sequence-event-content><div hide-from-message-actions class=hide-from-message-actions><div class="sequence-event-title gds-emphasized-body-l">Compile the binaries</div><div class="sequence-event-subtitle gds-extended-caption ng-star-inserted">Parallel compilation</div></div><div class="sequence-event-description gds-body-l"><span only-show-to-message-actions data-test-id=sequence-export-header class=only-show-to-message-actions style=display:none>Parallel compilation.</span><structured-node-sequence class=ng-star-inserted><structured-text class=ng-star-inserted><p class=ng-star-inserted>Compile the project. The <code>-j</code> flag tells it to use all available CPU cores to build it as fast as possible.</p></structured-text><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-743 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-743 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_c0f16b74d2da6c59","c_4089c2ad7c8dde16",null,"rc_b42482ce46133ad3",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=10><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-743"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-743"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-743 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-743>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-743 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-743 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-743 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-over
|
|||
|
|
</code></pre></div></div></div></code-block><structured-text class=ng-star-inserted><p class=ng-star-inserted>Once this finishes, your compiled executables will be waiting for you inside the <code>build/bin/</code> folder.</p></structured-text></structured-node-sequence></div></div></div><div class="sequence-event ng-star-inserted"><div hide-from-message-actions class="sequence-event-marker-container hide-from-message-actions"><div class="sequence-event-marker gds-body-s ng-star-inserted">4</div></div><div class=sequence-event-content><div hide-from-message-actions class=hide-from-message-actions><div class="sequence-event-title gds-emphasized-body-l">Execute and auto-download a model</div><div class="sequence-event-subtitle gds-extended-caption ng-star-inserted">First run</div></div><div class="sequence-event-description gds-body-l"><span only-show-to-message-actions data-test-id=sequence-export-header class=only-show-to-message-actions style=display:none>First run.</span><structured-node-sequence class=ng-star-inserted><structured-text class=ng-star-inserted><p class=ng-star-inserted>Instead of browsing Hugging Face manually, you can use the <code>-hf</code> (Hugging Face) flag. Pass it a repository and a specific quantization file, and <code>llama.cpp</code> will download it directly into a local cache and execute it immediately:</p></structured-text><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-744 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-744 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_c0f16b74d2da6c59","c_4089c2ad7c8dde16",null,"rc_b42482ce46133ad3",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=13><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-744"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-744"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-744 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-744>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-744 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-744 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-744 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted"
|
|||
|
|
</code></pre></div></div></div></code-block></structured-node-sequence></div></div></div></div></sequence></response-element></div><p><h3 data-path-to-node=6>If You Prefer Manual Downloads</h3><p data-path-to-node=7>If you still want to download a <code data-path-to-node=7 data-index-in-node=32>.gguf</code> file yourself through a browser or <code data-path-to-node=7 data-index-in-node=73>huggingface-cli</code>, you can store it anywhere on your drive and run it manually by pointing to its exact path using the <code data-path-to-node=7 data-index-in-node=190>-m</code> (model) flag:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-745 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-745 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_c0f16b74d2da6c59","c_4089c2ad7c8dde16",null,"rc_b42482ce46133ad3",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=16><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-745"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-745"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-745 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-745>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-745 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-745 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-745 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_c0f16b74d2da6c59","c_4089c2ad7c8dde16",null,"rc_b42482ce46133ad3",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ri
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=9>Running as a Local Server</h3><p data-path-to-node=10>If you're planning to interface this with code or external UI applications (like a Next.js project or local developer tools), you'll likely want to run the server binary instead of the CLI. This sets up a local OpenAI-compatible API endpoint:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-746 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-746 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_c0f16b74d2da6c59","c_4089c2ad7c8dde16",null,"rc_b42482ce46133ad3",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=19><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-746"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-746"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-746 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-746>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-746 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-746 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-746 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_c0f16b74d2da6c59","c_4089c2ad7c8dde16",null,"rc_b42482ce46133ad3",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng
|
|||
|
|
</code></pre></div></div></div></code-block></response-element></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-626 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-627 hide-action-bar ng-tns-c3135600295-626 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-627"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-627 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-627 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_c0f16b74d2da6c59","c_4089c2ad7c8dde16",null,"rc_b42482ce46133ad3",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-627 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_c0f16b74d2da6c59","c_4089c2ad7c8dde16",null,"rc_b42482ce46133ad3",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><ul data-path-to-node=6><li><p data-path-to-node=6,0,0><b data-path-to-node=6,0,0 data-index-in-node=0>What it does:</b> This doesn't compile any code. Instead, it looks at your system, detects your compiler, finds your hardware specifications (like Apple Silicon features or build configurations), and sets up a <code data-path-to-node=6,0,0 data-index-in-node=206>build/</code> directory with the necessary instructions (Makefiles or Xcode project structures).</p></ul><h3 data-path-to-node=7>Step 2: Compile the Targets</h3><p data-path-to-node=8>Once the first command finishes successfully, run the compilation command:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-748 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-748 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_1fae95ae0d841b73","c_4089c2ad7c8dde16",null,"rc_0c393a0ef7f17165",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-748"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-748"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-748 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-748>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-748 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-748 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-748 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_1fae95ae0d841b73","c_4089c2ad7c8dde16",null,"rc_0c393a0ef7f17165",null,null,"",null
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><ul data-path-to-node=10><li><p data-path-to-node=10,0,0><b data-path-to-node=10,0,0 data-index-in-node=0>What it does:</b> This is the command that does the heavy lifting. It reads the instructions generated in the <code data-path-to-node=10,0,0 data-index-in-node=106>build/</code> directory, fires up your CPU cores in parallel (<code data-path-to-node=10,0,0 data-index-in-node=161>-j</code>), and compiles the source code into the exact executables you requested: <code data-path-to-node=10,0,0 data-index-in-node=237>llama-cli</code> and <code data-path-to-node=10,0,0 data-index-in-node=251>llama-server</code>.</p></ul><p data-path-to-node=11>Once both finish, you'll find your compiled binaries ready to use inside the <code data-path-to-node=11 data-index-in-node=77>build/bin/</code> directory!</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-628 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-629 hide-action-bar ng-tns-c3135600295-628 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-629"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-629 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-629 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_1fae95ae0d841b73","c_4089c2ad7c8dde16",null,"rc_0c393a0ef7f17165",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-629 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><ul data-path-to-node=6><li><p data-path-to-node=6,0,0><b data-path-to-node=6,0,0 data-index-in-node=0>If it prints version details:</b> (e.g., <code data-path-to-node=6,0,0 data-index-in-node=37>Apple clang version...</code>), your compiler is active, fully functional, and ready to roll.</p><li><p data-path-to-node=6,1,0><b data-path-to-node=6,1,0 data-index-in-node=0>If a pop-up window appears:</b> macOS will ask you if you want to install "Command Line Developer Tools." Click <b data-path-to-node=6,1,0 data-index-in-node=108>Install</b> and let it finish. It takes about two minutes and installs <code data-path-to-node=6,1,0 data-index-in-node=175>make</code>, <code data-path-to-node=6,1,0 data-index-in-node=181>clang</code>, and all necessary hardware hooks automatically.</p></ul><h3 data-path-to-node=8>Step 2: Compile <code data-path-to-node=8 data-index-in-node=16>llama.cpp</code> using the native <code data-path-to-node=8 data-index-in-node=43>make</code> fallback</h3><p data-path-to-node=9>Since you don't have CMake, make sure you are inside your cloned <code data-path-to-node=9 data-index-in-node=65>src</code> directory where you pulled the repository files, and run the traditional compiler build wrapper:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-750 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-750 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_94b85a8b92726583","c_4089c2ad7c8dde16",null,"rc_bd8e18079c0074cc",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-750"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-750"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-750 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-750>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-750 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-750 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-750 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-desc
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><ul data-path-to-node=11><li><p data-path-to-node=11,0,0><b data-path-to-node=11,0,0 data-index-in-node=0>What this does:</b> It reads the stock <code data-path-to-node=11,0,0 data-index-in-node=35>Makefile</code> included in the root folder, automatically detects your Apple Silicon layout, couples the architecture to the <b data-path-to-node=11,0,0 data-index-in-node=154>Accelerate and Metal frameworks</b>, and runs parallel compilation using all of your Mac's internal CPU cores simultaneously.</p></ul><h3 data-path-to-node=13>Step 3: Run Your Backend Server</h3><p data-path-to-node=14>Once <code data-path-to-node=14 data-index-in-node=5>make -j</code> completes, your executables are generated directly in the <b data-path-to-node=14 data-index-in-node=71>root</b> folder (unlike CMake, which hides them inside a <code data-path-to-node=14 data-index-in-node=124>build/bin/</code> subdirectory).<p data-path-to-node=15>You can immediately spin up your engine and let it pull down your target model file automatically:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-751 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-751 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_94b85a8b92726583","c_4089c2ad7c8dde16",null,"rc_bd8e18079c0074cc",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=9><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-751"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-751"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-751 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-751>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-751 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-751 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-751 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-b
|
|||
|
|
</code></pre></div></div></div></code-block></response-element></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-630 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-631 hide-action-bar ng-tns-c3135600295-630 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-631"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-631 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-631 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_94b85a8b92726583","c_4089c2ad7c8dde16",null,"rc_bd8e18079c0074cc",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-631 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_94b85a8b92726583","c_4089c2ad7c8dde16",null,"rc_bd8e18079c0074cc",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=
|
|||
|
|
<span class=hljs-built_in>cd</span> 138_llama_cpp/src
|
|||
|
|
|
|||
|
|
<span class=hljs-comment># Build the binaries using all available CPU cores simultaneously</span>
|
|||
|
|
make -j
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=6><i data-path-to-node=6 data-index-in-node=0>(This will compile the engine code, link it directly to Apple's native <b data-path-to-node=6 data-index-in-node=71>Accelerate and Metal frameworks</b>, and generate your executable programs directly inside your active folder in under a minute).</i><h3 data-path-to-node=8>Step 2: Initialize the Server & Auto-Download Gemma 4</h3><p data-path-to-node=9>Once the compilation logs finish scroll-rolling, you can run the freshly minted <code data-path-to-node=9 data-index-in-node=80>llama-server</code> binary. We will use the <code data-path-to-node=9 data-index-in-node=117>-hf</code> flag so it connects to Hugging Face, downloads the correct un-gated <b data-path-to-node=9 data-index-in-node=189>Gemma 4 12B Q5_K_M</b> file slice right into your local cache directory, and boots it straight onto your Mac's integrated GPU layers:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-753 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-753 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_18335df77d7d2f82","c_4089c2ad7c8dde16",null,"rc_2cf917d9938d4fe4",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-753"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-753"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-753 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-753>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-753 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-753 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-753 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=11>What to Look For on Boot</h3><p data-path-to-node=12>As the text logs scroll by, look for these specific lines to verify your system is utilizing your optimal hardware pipeline:<ul data-path-to-node=13><li><p data-path-to-node=13,0,0><code data-path-to-node=13,0,0 data-index-in-node=0>ggml_metal_init: allocating pool...</code> — This confirms the server successfully hooked into the macOS Metal API to offload computation to your GPU.</p><li><p data-path-to-node=13,1,0><code data-path-to-node=13,1,0 data-index-in-node=0>HTTP server listening on [http://127.0.0.1:8080](http://127.0.0.1:8080)</code> — Your local OpenAI-compatible endpoint is live and fully ready to stream tokens into your background scripts or Next.js layout setups!</p></ul></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-632 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-633 hide-action-bar ng-tns-c3135600295-632 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-633"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-633 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-633 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_18335df77d7d2f82","c_4089c2ad7c8dde16",null,"rc_2cf917d9938d4fe4",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-633 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-b
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=7><i data-path-to-node=7 data-index-in-node=0>(This takes about 30 seconds to fetch and symlink into your global path).</i><h3 data-path-to-node=9>Step 2: Configure and Build <code data-path-to-node=9 data-index-in-node=28>llama.cpp</code></h3><p data-path-to-node=10>Now that your Mac has the modern compilation engine ready, run the two-step setup we looked at earlier. You are already in the correct folder (<code data-path-to-node=10 data-index-in-node=143>/src</code>), so just execute these directly:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-755 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-755 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_8c96d7d21453f383","c_4089c2ad7c8dde16",null,"rc_26984f4bd3688b78",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-755"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-755"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-755 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-755>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-755 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-755 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-755 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_8c96d7d21453f383","c_4089c2ad7c8dde16",null,"rc_26984f4bd3688b78",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__r
|
|||
|
|
cmake -B build
|
|||
|
|
|
|||
|
|
<span class=hljs-comment># 2. Compile the core binary executables in parallel using all available CPU threads</span>
|
|||
|
|
cmake --build build -j --target llama-cli llama-server
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=13>Step 3: Launch and Download Gemma 4</h3><p data-path-to-node=14>Once the compilation progress hits 100%, your executables will be waiting inside the newly created <code data-path-to-node=14 data-index-in-node=99>build/bin/</code> folder.<p data-path-to-node=15>Run this final execution command to boot up the server endpoint—it will natively handle connecting to Hugging Face, stream the <b data-path-to-node=15 data-index-in-node=127>Gemma 4 12B Q5_K_M</b> file right into your local system cache, and bind it straight to your Metal graphics layers:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-756 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-756 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_8c96d7d21453f383","c_4089c2ad7c8dde16",null,"rc_26984f4bd3688b78",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=9><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-756"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-756"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-756 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-756>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-756 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-756 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-756 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_8c96d7d21453f383","c_4089c2ad7c8dde16",null,"rc_26984f4bd3688b78",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="m
|
|||
|
|
</code></pre></div></div></div></code-block></response-element></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-634 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-635 hide-action-bar ng-tns-c3135600295-634 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-635"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-635 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-635 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_8c96d7d21453f383","c_4089c2ad7c8dde16",null,"rc_26984f4bd3688b78",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-635 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_8c96d7d21453f383","c_4089c2ad7c8dde16",null,"rc_26984f4bd3688b78",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=7>Step 2: Jump Back into your <code data-path-to-node=7 data-index-in-node=28>llama.cpp</code> Directory</h3><p data-path-to-node=8>Once <code data-path-to-node=8 data-index-in-node=5>brew</code> finishes installing CMake, navigate back into the source repository directory where you cloned the code:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-758 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-758 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_0b5419bac0c92c86","c_4089c2ad7c8dde16",null,"rc_c620ab28e5600411",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-758"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-758"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-758 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-758>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-758 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-758 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-758 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_0b5419bac0c92c86","c_4089c2ad7c8dde16",null,"rc_c620ab28e5600411",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=tru
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=11>Step 3: Run the Modern Build Stream</h3><p data-path-to-node=12>Now that your terminal is in the correct directory and CMake is ready, initialize and compile the project by running these two steps in sequence:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-759 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-759 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_0b5419bac0c92c86","c_4089c2ad7c8dde16",null,"rc_c620ab28e5600411",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=9><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-759"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-759"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-759 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-759>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-759 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-759 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-759 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_0b5419bac0c92c86","c_4089c2ad7c8dde16",null,"rc_c620ab28e5600411",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=copy data-mat-icon-namespace=lumi-symbols f
|
|||
|
|
cmake -B build
|
|||
|
|
|
|||
|
|
<span class=hljs-comment># 2. Compile the binaries using all your CPU cores</span>
|
|||
|
|
cmake --build build -j --target llama-cli llama-server
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=14>Once that hits 100%, your compiled binaries will be built and ready to go inside the <code data-path-to-node=14 data-index-in-node=85>build/bin</code> folder.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-636 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-637 hide-action-bar ng-tns-c3135600295-636 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-637"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-637 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-637 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_0b5419bac0c92c86","c_4089c2ad7c8dde16",null,"rc_c620ab28e5600411",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-637 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_0b5419bac0c92c86","c_4089c2ad7c8dde16",null,"rc_c620ab28e5600411",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="m
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=4>What happens next?</h3><ol start=1 data-path-to-node=5><li><p data-path-to-node=5,0,0><b data-path-to-node=5,0,0 data-index-in-node=0>The Download Begins:</b> <code data-path-to-node=5,0,0 data-index-in-node=21>llama-server</code> will start fetching the ~9.5 GB file directly from Hugging Face and show you a download progress bar right in the terminal window.</p><li><p data-path-to-node=5,1,0><b data-path-to-node=5,1,0 data-index-in-node=0>Metal Acceleration Boots:</b> Once downloaded, it will initialize the context (<code data-path-to-node=5,1,0 data-index-in-node=75>-c 8192</code> sets an 8k token window) and offload all 99 layers (<code data-path-to-node=5,1,0 data-index-in-node=135>--n-gpu-layers 99</code>) entirely onto your Mac's graphics pipeline.</p><li><p data-path-to-node=5,2,0><b data-path-to-node=5,2,0 data-index-in-node=0>The Web UI is Live:</b> Open up your browser and head to <b data-path-to-node=5,2,0 data-index-in-node=53><code data-path-to-node=5,2,0 data-index-in-node=53>http://localhost:8080</code></b>. You'll be greeted by the natively built chat interface where you can interact with Gemma 4 locally.</p></ol></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-638 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-639 hide-action-bar ng-tns-c3135600295-638 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-639"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-639 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-639 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_ba8a4b05475e3e2b","c_4089c2ad7c8dde16",null,"rc_2e918d640313b5f8",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-639 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button ge
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=7>Alternatively: Use the Q4 version from the official repo</h3><p data-path-to-node=8>If you just want to get up and running right now without changing repositories, you can pull the slightly smaller (but incredibly fast and efficient) 4-bit version that <code data-path-to-node=8 data-index-in-node=169>ggml-org</code> provides natively:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-762 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-762 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_a140a4a1cd07cbe4","c_4089c2ad7c8dde16",null,"rc_34379fd0eb38a8fc",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-762"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-762"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-762 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-762>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-762 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-762 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-762 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_a140a4a1cd07cbe4","c_4089c2ad7c8dde16",null,"rc_34379fd0eb38a8fc",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-sym
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=10>Once the download stream hits 100%, look for the <code data-path-to-node=10 data-index-in-node=49>HTTP server listening on http://127.0.0.1:8080</code> log message, open your web browser, and you can begin prompting.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-640 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-641 hide-action-bar ng-tns-c3135600295-640 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-641"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-641 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-641 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_a140a4a1cd07cbe4","c_4089c2ad7c8dde16",null,"rc_34379fd0eb38a8fc",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-641 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_a140a4a1cd07cbe4","c_4089c2ad7c8dde16",null,"rc_34379fd0eb38a8fc",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c40
|
|||
|
|
<code data-path-to-node=0 data-index-in-node=82>gemma-4-12B-it-Q5_K_M.gguf</code><p data-path-to-node=1>The reason it failed is due to a strict case-sensitivity mismatch in the file name itself. In the command, we used a lowercase <b data-path-to-node=1 data-index-in-node=127><code data-path-to-node=1 data-index-in-node=127>b</code></b> (<code data-path-to-node=1 data-index-in-node=130>gemma-4-12b...</code>), but <code data-path-to-node=1 data-index-in-node=151>bartowski</code> uploaded it with a capital <b data-path-to-node=1 data-index-in-node=188><code data-path-to-node=1 data-index-in-node=188>B</code></b> (<code data-path-to-node=1 data-index-in-node=191>gemma-4-12B...</code>).<p data-path-to-node=2>Because Git and Hugging Face look for exact string matches, that single lowercase letter caused the server to skip right past it.<h3 data-path-to-node=4>The Final, Corrected Command</h3><p data-path-to-node=5>Swap out that single character for a capital <b data-path-to-node=5 data-index-in-node=45><code data-path-to-node=5 data-index-in-node=45>B</code></b> to match the repository exactly. Run this inside your terminal:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-763 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-763 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_71bdadb62e06fb39","c_4089c2ad7c8dde16",null,"rc_798627d4f4f14acf",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=3><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-763"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-763"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-763 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-763>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-763 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-763 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-763 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-but
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=7>What to expect now:</h3><ul data-path-to-node=8><li><p data-path-to-node=8,0,0><b data-path-to-node=8,0,0 data-index-in-node=0>Progress Bar:</b> The server will immediately recognize the exact file, bypass the error block, and display a live download progress metric directly in your terminal workspace.</p><li><p data-path-to-node=8,1,0><b data-path-to-node=8,1,0 data-index-in-node=0>Server Verification:</b> As soon as it hits 100%, it will allocate your unified memory pool, hook into your Mac's Metal API, and boot up on port <code data-path-to-node=8,1,0 data-index-in-node=141>8080</code>.</p></ul></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-642 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-643 hide-action-bar ng-tns-c3135600295-642 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-643"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-643 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-643 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_71bdadb62e06fb39","c_4089c2ad7c8dde16",null,"rc_798627d4f4f14acf",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-643 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=6><i data-path-to-node=6 data-index-in-node=0>(You will see a standard progress bar. Let this run until it hits 100% and finishes downloading the ~9.5 GB file).</i><h3 data-path-to-node=8>Step 2: Launch the Server Region Locally</h3><p data-path-to-node=9>Once the download finishes, use the local model flag (<code data-path-to-node=9 data-index-in-node=54>-m</code>) instead of the buggy Hugging Face flag (<code data-path-to-node=9 data-index-in-node=98>-hf</code>). This tells <code data-path-to-node=9 data-index-in-node=115>llama-server</code> to read the file straight from your hard drive:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-765 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-765 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_81706dbdd3e25c61","c_4089c2ad7c8dde16",null,"rc_908dda5ca3c124f5",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=6><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-765"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-765"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-765 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-765>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-765 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-765 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-765 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_81706dbdd3e25c61","c_4089c2ad7c8dde16",null,"rc_908dda5ca3c124f5",null,null,"en",null,1
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=11>This will bypass the network lookup code entirely, instantly load the model into your Mac's unified memory via Metal, and open up your port listener at <code data-path-to-node=11 data-index-in-node=152>http://localhost:8080</code>.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-644 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-645 hide-action-bar ng-tns-c3135600295-644 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-645"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-645 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-645 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_81706dbdd3e25c61","c_4089c2ad7c8dde16",null,"rc_908dda5ca3c124f5",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-645 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_81706dbdd3e25c61","c_4089c2ad7c8dde16",null,"rc_908dda5ca3c124f5",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=4>What to Verify in Your Terminal Now:</h3><ol start=1 data-path-to-node=5><li><p data-path-to-node=5,0,0><b data-path-to-node=5,0,0 data-index-in-node=0><code data-path-to-node=5,0,0 data-index-in-node=0>ggml_metal_init</code></b>: Look for this line right at the start of the log scroll. It confirms <code data-path-to-node=5,0,0 data-index-in-node=86>llama.cpp</code> has successfully grabbed your Mac's graphics cores.</p><li><p data-path-to-node=5,1,0><b data-path-to-node=5,1,0 data-index-in-node=0><code data-path-to-node=5,1,0 data-index-in-node=0>llm_load_tensors</code></b>: You'll see a quick progress indicator mapping the layers directly into your unified memory cache.</p><li><p data-path-to-node=5,2,0><b data-path-to-node=5,2,0 data-index-in-node=0><code data-path-to-node=5,2,0 data-index-in-node=0>HTTP server listening</code></b>: The terminal will settle on a message stating it's listening on <code data-path-to-node=5,2,0 data-index-in-node=87>http://127.0.0.1:8080</code>.</p></ol><p data-path-to-node=6>Once that prints, open up your web browser of choice and head straight to <b data-path-to-node=6 data-index-in-node=74><code data-path-to-node=6 data-index-in-node=74>http://localhost:8080</code></b> to start prompting your fresh, fully private instance of Gemma 4!</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-646 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-647 hide-action-bar ng-tns-c3135600295-646 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-647"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-647 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-647 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_7fa9e11c38349edf","c_4089c2ad7c8dde16",null,"rc_0db166c24d605ead",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-647 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 the
|
|||
|
|
👉 <b data-path-to-node=4 data-index-in-node=71><code data-path-to-node=4 data-index-in-node=71>[http://127.0.0.1:8080](http://127.0.0.1:8080)</code></b> (or <code data-path-to-node=4 data-index-in-node=122>http://localhost:8080</code>)<p data-path-to-node=5>Open your favorite web browser, paste that URL into the address bar, and you'll be greeted by the built-in <code data-path-to-node=5 data-index-in-node=107>llama.cpp</code> chat interface. You can minimize this terminal window now—just leave it running in the background to act as your engine.<p data-path-to-node=6>Let me know how the initial response speeds feel on your Mac once you throw a few test prompts at it!</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-648 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-649 hide-action-bar ng-tns-c3135600295-648 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-649"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-649 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-649 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_20d6a3294eec80dc","c_4089c2ad7c8dde16",null,"rc_fe53c7e177006a88",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-649 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=4>Once Gemma 4 actually started writing the response, it ran at a perfectly respectable <b data-path-to-node=4 data-index-in-node=86>13.3 tokens per second</b>. For a 12-billion-parameter model running on a Q5 quantization layout locally, that's a great, highly readable pace.<h3 data-path-to-node=5>2. The Bad: Massive Context Over-Allocation</h3><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-772 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-772 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_40b7a1cd2b581c35","c_4089c2ad7c8dde16",null,"rc_614722bf9769adb3",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQpgk><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-772"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-772"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-772 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-772>Plaintext</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-772 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-772 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-772 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_40b7a1cd2b581c35","c_4089c2ad7c8dde16",null,"rc_614722bf9769adb3",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ng
|
|||
|
|
...
|
|||
|
|
eval time = 75.08 ms per token
|
|||
|
|
...
|
|||
|
|
prompt eval time = 2058.46 ms / 21 tokens (10.20 tokens per second)
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=7><b data-path-to-node=7 data-index-in-node=0>This is the bottleneck.</b> Because you didn't specify how many users or parallel streams you wanted, <code data-path-to-node=7 data-index-in-node=98>llama-server</code> guessed and auto-allocated <b data-path-to-node=7 data-index-in-node=138>4 parallel slots</b> (<code data-path-to-node=7 data-index-in-node=156>n_slots = 4</code>).<p data-path-to-node=8>Since your context length is set to <code data-path-to-node=8 data-index-in-node=36>-c 8192</code>, it divided your M3 Pro's memory cache by 4. This forced your prompt evaluation (the "thinking time" before it starts typing) down to a sluggish <b data-path-to-node=8 data-index-in-node=189>10 tokens per second</b>, introducing that ~2-second delay for a tiny 21-token question.<h3 data-path-to-node=9>How to drastically speed this up for a single user</h3><p data-path-to-node=10>Since you are the only person using this server, you don't need 4 parallel slots dragging down your system memory and throughput. You only need <b data-path-to-node=10 data-index-in-node=144>1</b>.<p data-path-to-node=11>If you restart the server with the <code data-path-to-node=11 data-index-in-node=35>--parallel 1</code> flag (or <code data-path-to-node=11 data-index-in-node=57>-np 1</code>), you will collapse all that overhead into a single dedicated channel. This will make your prompt processing feel instant and should bump your writing speeds even higher.<p data-path-to-node=12>When you're ready to test again, close the current terminal process with <b data-path-to-node=12 data-index-in-node=73><code data-path-to-node=12 data-index-in-node=73>Ctrl + C</code></b>, and restart it with this refined command:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-773 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-773 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_40b7a1cd2b581c35","c_4089c2ad7c8dde16",null,"rc_614722bf9769adb3",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQpwk><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-773"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-773"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-773 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-773>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-773 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-773 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=14>Give that a spin and throw another question at it—you should notice the "initial pause" before it answers almost entirely vanish.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer complete gap"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-654 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-655 ng-tns-c3135600295-654 ng-star-inserted hide-action-bar"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-655"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-655 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-655 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_40b7a1cd2b581c35","c_4089c2ad7c8dde16",null,"rc_614722bf9769adb3",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-655 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_40b7a1cd2b581c35","c_4089c2ad7c8dde16",null,"rc_614722bf9769adb3",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon
|
|||
|
|
<code data-path-to-node=10 data-index-in-node=200>EH-swehs kroo-SYAL</code> (or <code data-path-to-node=10 data-index-in-node=223>kroo-THYAL</code>)<p data-path-to-node=11>How is the server responding to these longer prompts now? Did adding the single-user flag flatten that initial pause?</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer gap complete"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-774 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-775 ng-star-inserted hide-action-bar"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-775"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-775 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-775 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_4c15153e7614d5fc","c_4089c2ad7c8dde16",null,"rc_f69a705b7abfbe08",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-775 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_4c15153e7614d5fc","c_4089c2ad7c8dde16",null,"rc_f69a705b7abfbe08",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=8><code data-path-to-node=8 data-index-in-node=0>llama.cpp</code> is aggressively caching your exact conversation history. When you send a new prompt, it doesn't re-evaluate the entire chat history from scratch. It instantly restores the saved state cache (the previous 808 tokens) from your unified memory, and <i data-path-to-node=8 data-index-in-node=256>only</i> spends time processing the brand-new text you just typed.<h3 data-path-to-node=9>3. Generation Throughput Stable</h3><p data-path-to-node=10>Your output text generation speed stayed rock-solid right around <b data-path-to-node=10 data-index-in-node=65>13.2 to 13.3 tokens per second</b>. On the final turn, as the total context pushed past 2,000 tokens, it dipped slightly to <code data-path-to-node=10 data-index-in-node=185>12.92 t/s</code>—which is completely normal as the internal attention matrices grow larger, but still highly readable and snappy.<p data-path-to-node=11>Your local environment is now running perfectly optimized for a single-user workflow. It's fast, private, and running entirely on the bare metal. Have you thrown any longer technical or reasoning tasks at it yet to see how the content holds up?</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer gap complete"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-777 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-780 ng-star-inserted hide-action-bar"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-780"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-780 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-780 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_427f3d55883fc0fa","c_4089c2ad7c8dde16",null,"rc_e57d8ada41d11437",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-780 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=8>The Trade-Offs to Keep in Mind:</h3><ul data-path-to-node=9><li><p data-path-to-node=9,0,0><b data-path-to-node=9,0,0 data-index-in-node=0>The Context Math:</b> Notice I bumped the total context (<code data-path-to-node=9,0,0 data-index-in-node=53>-c</code>) to <code data-path-to-node=9,0,0 data-index-in-node=60>16384</code>. With <code data-path-to-node=9,0,0 data-index-in-node=72>-np 3</code>, <code data-path-to-node=9,0,0 data-index-in-node=79>llama-server</code> will divide that space evenly, giving each of the 3 active users a dedicated <b data-path-to-node=9,0,0 data-index-in-node=169>5,461-token</b> history limit.</p><li><p data-path-to-node=9,1,0><b data-path-to-node=9,1,0 data-index-in-node=0>RAM Impact:</b> Gemma 4 (12B Q5) takes roughly 9.5 GB of VRAM. A single 8k context slot takes up a sliver of RAM, but multiplying slots and scaling context creates larger KV caches. Your M3 Pro has ~13.6 GB of free graphics memory, so pushing past 3 or 4 dense, simultaneous slots will hit a hard boundary and spill into standard system CPU RAM, dropping your speeds significantly.</p></ul><h2 data-path-to-node=11>2. How Remote Users Physically Connect</h2><p data-path-to-node=12>By default, <code data-path-to-node=12 data-index-in-node=12>llama-server</code> binds to <code data-path-to-node=12 data-index-in-node=34>127.0.0.1</code> (localhost), meaning it strictly listens to requests originating from your own machine. To let other people in, you have to decide whether they are on your local home network or connecting over the internet.<h3 data-path-to-node=13>Option A: Local Network Sharing (Friends in the same house/office)</h3><p data-path-to-node=14>If the other users are connected to your same Wi-Fi router, you simply tell <code data-path-to-node=14 data-index-in-node=76>llama-server</code> to listen on your local area network (LAN) by binding it to all interfaces using <code data-path-to-node=14 data-index-in-node=170>--host 0.0.0.0</code>:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-823 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-823 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_723a101a130d96ca","c_4089c2ad7c8dde16",null,"rc_9613b69a117891f2",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQ-Qk><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-823"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-823"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-823 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-823>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-823 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-823 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _n
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h4 data-path-to-node=16>How they connect:</h4><ol start=1 data-path-to-node=17><li><p data-path-to-node=17,0,0>Find your Mac's internal IP address (e.g., <code data-path-to-node=17,0,0 data-index-in-node=43>192.168.1.45</code>) via <b data-path-to-node=17,0,0 data-index-in-node=61>System Settings > Wi-Fi > Details</b>.</p><li><p data-path-to-node=17,1,0>Your friends simply open their web browsers on their own laptops and type: <code data-path-to-node=17,1,0 data-index-in-node=75>[http://192.168.1.45:8080](http://192.168.1.45:8080)</code></p></ol><h3 data-path-to-node=18>Option B: Remote Over the Internet (Safe & Encrypted)</h3><p data-path-to-node=19>If your users are sitting in different cities, <b data-path-to-node=19 data-index-in-node=47>do not</b> expose port 8080 directly to the raw internet via router port-forwarding; <code data-path-to-node=19 data-index-in-node=128>llama-server</code> doesn't have built-in user authentication or brute-force protections by default.<p data-path-to-node=20>Instead, use a secure, encrypted tunnel utility like <b data-path-to-node=20 data-index-in-node=53>ngrok</b> or <b data-path-to-node=20 data-index-in-node=62>Localtunnel</b> to expose just the port.<p data-path-to-node=21 data-sf-nesting-track-id=1.10.1.3.2.2.2.1.3.2.1.1.2.1.49.2.1.1.1.2.1.1.3.1.1.1.20><div class="attachment-container unknown" data-sf-nesting-track-id=1.10.1.3.2.2.2.1.3.2.1.1.2.1.49.2.1.1.1.2.1.1.3.1.1.1.20.1><response-element ng-version=0.0.0-PLACEHOLDER><sequence class="lm-enabled ng-star-inserted"><div class=sequence-container jslog=308913;track:impression,attention data-hveid=0 decode-data-ved=1 data-ved=0CAAQse0SahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQ-gk><div class="sequence-event ng-star-inserted"><div hide-from-message-actions class="sequence-event-marker-container hide-from-message-actions"><div class="sequence-event-marker gds-body-s ng-star-inserted">1</div><div class="sequence-event-line ng-star-inserted"></div></div><div class=sequence-event-content><div hide-from-message-actions class=hide-from-message-actions><div class="sequence-event-title gds-emphasized-body-l">Install a tunneling tool</div><div class="sequence-event-subtitle gds-extended-caption ng-star-inserted">1 minute</div></div><div class="sequence-event-description gds-body-l"><span only-show-to-message-actions data-test-id=sequence-export-header class=only-show-to-message-actions style=display:none>1 minute.</span><structured-node-sequence class=ng-star-inserted><structured-text class=ng-star-inserted><p class=ng-star-inserted>Open a separate terminal window and install ngrok via Homebrew:</p></structured-text><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-824 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-824 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_723a101a130d96ca","c_4089c2ad7c8dde16",null,"rc_9613b69a117891f2",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=1><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-824"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-824"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-824 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-824>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-824 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-824 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c4966679
|
|||
|
|
</code></pre></div></div></div></code-block></structured-node-sequence></div></div></div><div class="sequence-event ng-star-inserted"><div hide-from-message-actions class="sequence-event-marker-container hide-from-message-actions"><div class="sequence-event-marker gds-body-s ng-star-inserted">2</div><div class="sequence-event-line ng-star-inserted"></div></div><div class=sequence-event-content><div hide-from-message-actions class=hide-from-message-actions><div class="sequence-event-title gds-emphasized-body-l">Fire up your llama-server</div><div class="sequence-event-subtitle gds-extended-caption ng-star-inserted">Local engine setup</div></div><div class="sequence-event-description gds-body-l"><span only-show-to-message-actions data-test-id=sequence-export-header class=only-show-to-message-actions style=display:none>Local engine setup.</span><structured-node-sequence class=ng-star-inserted><structured-text class=ng-star-inserted><p class=ng-star-inserted>Keep your server running on your Mac normally:</p></structured-text><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-825 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-825 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_723a101a130d96ca","c_4089c2ad7c8dde16",null,"rc_9613b69a117891f2",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=4><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-825"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-825"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-825 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-825>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-825 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-825 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-825 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-
|
|||
|
|
</code></pre></div></div></div></code-block></structured-node-sequence></div></div></div><div class="sequence-event ng-star-inserted"><div hide-from-message-actions class="sequence-event-marker-container hide-from-message-actions"><div class="sequence-event-marker gds-body-s ng-star-inserted">3</div></div><div class=sequence-event-content><div hide-from-message-actions class=hide-from-message-actions><div class="sequence-event-title gds-emphasized-body-l">Create the public tunnel</div><div class="sequence-event-subtitle gds-extended-caption ng-star-inserted">Instant secure link</div></div><div class="sequence-event-description gds-body-l"><span only-show-to-message-actions data-test-id=sequence-export-header class=only-show-to-message-actions style=display:none>Instant secure link.</span><structured-node-sequence class=ng-star-inserted><structured-text class=ng-star-inserted><p class=ng-star-inserted>In your other terminal tab, tell ngrok to map your port 8080 out to their secure relay servers:</p></structured-text><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-826 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-826 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_723a101a130d96ca","c_4089c2ad7c8dde16",null,"rc_9613b69a117891f2",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=7><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-826"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-826"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-826 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-826>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-826 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-826 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-826 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-i
|
|||
|
|
</code></pre></div></div></div></code-block></structured-node-sequence></div></div></div></div></sequence></response-element></div><p><blockquote data-path-to-node=23><p data-path-to-node=23,0><b data-path-to-node=23,0 data-index-in-node=0>The Payload:</b> Ngrok will instantly spit out a unique, secure public URL (such as <code data-path-to-node=23,0 data-index-in-node=80>[https://a1b2-34-56-78.ngrok-free.app](https://a1b2-34-56-78.ngrok-free.app)</code>).</p></blockquote><p data-path-to-node=24>You can text or email that exact link to your friends anywhere in the world. When they click it, their browser will securely tunnel straight back to your Mac's graphics card, allowing them to experience local AI processing running out of your workspace.<p data-path-to-node=25>Are you looking to share the graphical browser interface with them, or are you planning to plug multiple users into it programmatically via an API endpoint?</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer gap complete"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-784 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-810 ng-star-inserted hide-action-bar"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-810"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-810 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-810 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_723a101a130d96ca","c_4089c2ad7c8dde16",null,"rc_9613b69a117891f2",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-810 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-md
|
|||
|
|
jq <span class=hljs-string>'.siblings[] | select(.rpath | endswith(".gguf")) | {file: .rpath, size_gb: (.size / 1024 / 1024 / 1024 | . * 100 | round / 100)}'</span>
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=14>Quick Sizing Heuristic for Your Engine</h3><p data-path-to-node=15>When choosing which file to download:<ul data-path-to-node=16><li><p data-path-to-node=16,0,0><b data-path-to-node=16,0,0 data-index-in-node=0>The 4-bit (<code data-path-to-node=16,0,0 data-index-in-node=11>Q4_K_M</code>) version</b> is the baseline standard for local testing; it provides massive memory savings with minimal quality loss.</p><li><p data-path-to-node=16,1,0><b data-path-to-node=16,1,0 data-index-in-node=0>The 5-bit (<code data-path-to-node=16,1,0 data-index-in-node=11>Q5_K_M</code>) version</b> is often considered the "sweet spot" for technical or code-heavy work, recovering almost all the loss of accuracy from quantization while staying well under the unquantized file size.</p></ul></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response gap complete"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-830 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-833 ng-star-inserted hide-action-bar"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-833"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-833 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-833 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_40876a826a295d9b","c_4089c2ad7c8dde16",null,"rc_d0910bafbaf80b6d",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-833 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-
|
|||
|
|
--threads 8 \
|
|||
|
|
-c 16384 \
|
|||
|
|
--jinja \
|
|||
|
|
--color \
|
|||
|
|
-p <span class=hljs-string>"Translate the following Python function to TypeScript..."</span>
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=9><i data-path-to-node=9 data-index-in-node=0>(The <code data-path-to-node=9 data-index-in-node=5>--jinja</code> flag instructs <code data-path-to-node=9 data-index-in-node=28>llama-cli</code> to automatically adopt the native chat template embedded in modern GGUF models for robust system/user formatting).</i><h2 data-path-to-node=11>2. <code data-path-to-node=11 data-index-in-node=3>llama-server</code> (The Persistent API Service)</h2><p data-path-to-node=12><code data-path-to-node=12 data-index-in-node=0>llama-server</code> turns the inference engine into a <b data-path-to-node=12 data-index-in-node=47>long-running HTTP API daemon</b>. It remains running in the background, listening for incoming JSON payloads, and handles concurrent requests or stateful web interface tokens.<h3 data-path-to-node=13>Core Uses</h3><ul data-path-to-node=14><li><p data-path-to-node=14,0,0><b data-path-to-node=14,0,0 data-index-in-node=0>OpenAI-Compatible Backend:</b> It serves endpoints matching OpenAI’s spec (e.g., <code data-path-to-node=14,0,0 data-index-in-node=77>/v1/chat/completions</code> or <code data-path-to-node=14,0,0 data-index-in-node=101>/v1/embeddings</code>). This makes it a drop-in replacement for any client, IDE extension (like Claude Code or Continue), or local script configured to point to an arbitrary port.</p><li><p data-path-to-node=14,1,0><b data-path-to-node=14,1,0 data-index-in-node=0>Built-in Web Dashboard:</b> Launching <code data-path-to-node=14,1,0 data-index-in-node=34>llama-server</code> initializes a native, browser-accessible user interface on your designated port, equipped with deep inspecting parameters, prompt tracking, and sampling diagnostics.</p><li><p data-path-to-node=14,2,0><b data-path-to-node=14,2,0 data-index-in-node=0>Multi-User Processing:</b> Unlike the single-user focus of <code data-path-to-node=14,2,0 data-index-in-node=55>llama-cli</code>, the server relies on context slot allocation and continuous batching parameters (<code data-path-to-node=14,2,0 data-index-in-node=147>--parallel</code>) to handle multiple discrete requests without wiping out the system cache.</p><li><p data-path-to-node=14,3,0><b data-path-to-node=14,3,0 data-index-in-node=0>Agent Operations:</b> Recent iterations support internal execution sandboxes and tool bindings directly through the interface (e.g., <code data-path-to-node=14,3,0 data-index-in-node=129>--tools all</code> for localized sandboxed shell or filesystem executions alongside reasoning configurations).</p></ul><h3 data-path-to-node=15>Key Example Arguments</h3><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-847 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-847 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_77296dbe5e3210fe","c_4089c2ad7c8dde16",null,"rc_4c467b32a0c211d9",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQ0Qo><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-847"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-847"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-847 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-847>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-847 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-847 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-
|
|||
|
|
--host 127.0.0.1 \
|
|||
|
|
--port 8080 \
|
|||
|
|
-c 32768 \
|
|||
|
|
--parallel 2 \
|
|||
|
|
--api-key <span class=hljs-string>"your-secure-local-token"</span>
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h2 data-path-to-node=18>Architectural and Operational Differences</h2><div class=horizontal-scroll-wrapper><div class=table-block-component><response-element ng-version=0.0.0-PLACEHOLDER><table-block _nghost-ng-c909611229 class="enable-luminous-table-block ng-star-inserted"><div _ngcontent-ng-c909611229 class="table-block has-export-button new-table-style is-at-scroll-start is-at-scroll-end"><div _ngcontent-ng-c909611229 not-end-of-paragraph class=table-content jslog=275421;track:impression,attention data-hveid=0 decode-data-ved=1 data-ved=0CAAQ3ecQahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQ0go><table data-path-to-node=19><thead><tr><th><span data-path-to-node=19,0,0,0>Characteristic</span><th><span data-path-to-node=19,0,1,0><code data-path-to-node=19,0,1,0 data-index-in-node=0>llama-cli</code></span><th><span data-path-to-node=19,0,2,0><code data-path-to-node=19,0,2,0 data-index-in-node=0>llama-server</code></span><tbody><tr><td><span data-path-to-node=19,1,0,0><b data-path-to-node=19,1,0,0 data-index-in-node=0>Lifecycle</b></span><td><span data-path-to-node=19,1,1,0>Ephemeral. Spawns, infers, terminates.</span><td><span data-path-to-node=19,1,2,0>Persistent background daemon process.</span><tr><td><span data-path-to-node=19,2,0,0><b data-path-to-node=19,2,0,0 data-index-in-node=0>Interface</b></span><td><span data-path-to-node=19,2,1,0>Standard I/O (stdin/stdout) and terminal chat.</span><td><span data-path-to-node=19,2,2,0>HTTP API Endpoints and a built-in Web UI.</span><tr><td><span data-path-to-node=19,3,0,0><b data-path-to-node=19,3,0,0 data-index-in-node=0>Concurrency</b></span><td><span data-path-to-node=19,3,1,0><b data-path-to-node=19,3,1,0 data-index-in-node=0>Single-threaded/Single-task:</b> Processes one sequence at a time.</span><td><span data-path-to-node=19,3,2,0><b data-path-to-node=19,3,2,0 data-index-in-node=0>Multi-slot concurrency:</b> Uses <code data-path-to-node=19,3,2,0 data-index-in-node=29>--parallel</code> to handle multiple independent requests.</span><tr><td><span data-path-to-node=19,4,0,0><b data-path-to-node=19,4,0,0 data-index-in-node=0>Parameter Precedence</b></span><td><span data-path-to-node=19,4,1,0>Locked at invocation via flags (e.g., <code data-path-to-node=19,4,1,0 data-index-in-node=38>--temp</code>, <code data-path-to-node=19,4,1,0 data-index-in-node=46>--top-p</code>).</span><td><span data-path-to-node=19,4,2,0>Overridable dynamically per request via the payload JSON.</span><tr><td><span data-path-to-node=19,5,0,0><b data-path-to-node=19,5,0,0 data-index-in-node=0>Integration Pattern</b></span><td><span data-path-to-node=19,5,1,0>Shell pipes, native binaries, Cron tasks, or local shell scripts.</span><td><span data-path-to-node=19,5,2,0>System services, developer tools, custom UI frontends, or local IDE plug-ins.</span></table></div><div _ngcontent-ng-c909611229 hide-from-message-actions class="table-footer hide-from-message-actions"><gem-icon-button _ngcontent-ng-c909611229 tabindex=0 gemmenutrigger cdkoverlayorigin theme=lm type=tonal arialabel="More options" gemtooltip="More options" class="mat-mdc-tooltip-trigger overflow-menu-button gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-tonal lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-disabled=false aria-haspopup=true aria-expanded=false aria-controls=ng-menu-a42457-175 aria-describedby=cdk-describedby-message-ng-1-226 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="More options" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c407
|
|||
|
|
-m models/qwen2.5-7b-instruct-q8_0.gguf \
|
|||
|
|
--host 127.0.0.1 \
|
|||
|
|
--port 8080 \
|
|||
|
|
-c 32768 \
|
|||
|
|
--api-key <span class=hljs-string>"sk-local-dev-precision-token-77x"</span> \
|
|||
|
|
--<span class=hljs-built_in>alias</span> <span class=hljs-string>"qwen2.5-7b"</span> \
|
|||
|
|
--parallel 2
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><blockquote data-path-to-node=8><p data-path-to-node=8,0>⚠️ <b data-path-to-node=8,0 data-index-in-node=3>Flag Conflict Warning:</b> Avoid combining <code data-path-to-node=8,0 data-index-in-node=42>--api-key</code> with <code data-path-to-node=8,0 data-index-in-node=57>--api-prefix</code> in current versions of <code data-path-to-node=8,0 data-index-in-node=93>llama-cpp</code>. A known configuration routing bug causes internal path resolution to throw a <code data-path-to-node=8,0 data-index-in-node=181>401 Invalid API Key</code> on valid requests when both parameters are explicitly declared simultaneously.</p></blockquote><h2 data-path-to-node=10>2. Testing the Boundary with cURL</h2><p data-path-to-node=11>Before pointing your development environments to the endpoint, verify that the validation layer is operating correctly by testing both unauthenticated and authenticated requests.<h3 data-path-to-node=12>Check Blocked Status (Should return 401 Unauthorized)</h3><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-870 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-870 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_8642f819b9cbc081","c_4089c2ad7c8dde16",null,"rc_6633a4efb6fe4323",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQ_go><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-870"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-870"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-870 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-870>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-870 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-870 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-870 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button
|
|||
|
|
-H <span class=hljs-string>"Content-Type: application/json"</span> \
|
|||
|
|
-d <span class=hljs-string>'{
|
|||
|
|
"model": "qwen2.5-7b",
|
|||
|
|
"messages": [{"role": "user", "content": "Ping"}]
|
|||
|
|
}'</span>
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=14>Check Verified Status (Should stream or complete successfully)</h3><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-871 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-871 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_8642f819b9cbc081","c_4089c2ad7c8dde16",null,"rc_6633a4efb6fe4323",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQ_wo><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-871"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-871"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-871 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-871>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-871 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-871 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-871 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_8642f819b9cbc081","c_4089c2ad7c8dde16",null,"rc_6633a4efb6fe4323",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=copy data-mat-icon-namespace=lumi-symbols fonticon=copy></mat-icon></gem-icon><span c
|
|||
|
|
-H <span class=hljs-string>"Content-Type: application/json"</span> \
|
|||
|
|
-H <span class=hljs-string>"Authorization: Bearer sk-local-dev-precision-token-77x"</span> \
|
|||
|
|
-d <span class=hljs-string>'{
|
|||
|
|
"model": "qwen2.5-7b",
|
|||
|
|
"messages": [{"role": "user", "content": "Respond with the word Pong."}]
|
|||
|
|
}'</span>
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h2 data-path-to-node=17>3. Connecting Local Development Tools</h2><p data-path-to-node=18>Most contemporary development extensions expect standard OpenAI request structures. Once the base URL and API key are passed into their respective config files, they communicate seamlessly with the local GGUF engine.<h3 data-path-to-node=19>Configuration for Claude Code</h3><p data-path-to-node=20>When configuring Anthropic's <code data-path-to-node=20 data-index-in-node=29>claude-code</code> CLI tool to leverage a local fallback or custom testing engine, update your local shell profile or configuration parameters to pass the local target:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-872 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-872 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_8642f819b9cbc081","c_4089c2ad7c8dde16",null,"rc_6633a4efb6fe4323",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQgAs><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-872"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-872"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-872 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-872>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-872 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-872 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-872 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r
|
|||
|
|
<span class=hljs-built_in>export</span> CLAUDE_API_KEY=<span class=hljs-string>"sk-local-dev-precision-token-77x"</span>
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=22>Configuration for Continue (VS Code / JetBrains)</h3><p data-path-to-node=23>For the <code data-path-to-node=23 data-index-in-node=8>Continue</code> extension, adjust your local <code data-path-to-node=23 data-index-in-node=46>~/.continue/config.json</code> file. Ensure that the <code data-path-to-node=23 data-index-in-node=92>provider</code> attribute is specified explicitly as <code data-path-to-node=23 data-index-in-node=138>"openai"</code> so it appends the correct bearer token formatting:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-873 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-873 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_8642f819b9cbc081","c_4089c2ad7c8dde16",null,"rc_6633a4efb6fe4323",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQgQs><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-873"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-873"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-873 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-873>JSON</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-873 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-873 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-873 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_8642f819b9cbc081","c_4089c2ad7c8dde16",null,"rc_6633a4efb6fe4323",null,null,"",null,1,null,null,1,0]];mutable:tr
|
|||
|
|
<span class=hljs-attr>"models"</span>: [
|
|||
|
|
{
|
|||
|
|
<span class=hljs-attr>"title"</span>: <span class=hljs-string>"Local Qwen 7B (Authenticated)"</span>,
|
|||
|
|
<span class=hljs-attr>"provider"</span>: <span class=hljs-string>"openai"</span>,
|
|||
|
|
<span class=hljs-attr>"model"</span>: <span class=hljs-string>"qwen2.5-7b"</span>,
|
|||
|
|
<span class=hljs-attr>"apiBase"</span>: <span class=hljs-string>"http://127.0.0.1:8080/v1"</span>,
|
|||
|
|
<span class=hljs-attr>"apiKey"</span>: <span class=hljs-string>"sk-local-dev-precision-token-77x"</span>
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
<span class=hljs-attr>"tabAutocompleteModel"</span>: {
|
|||
|
|
<span class=hljs-attr>"title"</span>: <span class=hljs-string>"Local Autocomplete"</span>,
|
|||
|
|
<span class=hljs-attr>"provider"</span>: <span class=hljs-string>"openai"</span>,
|
|||
|
|
<span class=hljs-attr>"model"</span>: <span class=hljs-string>"qwen2.5-7b"</span>,
|
|||
|
|
<span class=hljs-attr>"apiBase"</span>: <span class=hljs-string>"http://127.0.0.1:8080/v1"</span>,
|
|||
|
|
<span class=hljs-attr>"apiKey"</span>: <span class=hljs-string>"sk-local-dev-precision-token-77x"</span>
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=25>Configuration for Python Automation Scripts</h3><p data-path-to-node=26>If executing local code evaluations using the official <code data-path-to-node=26 data-index-in-node=55>openai</code> Python SDK, initialize your client connection directly against the loopback daemon:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-874 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-874 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_8642f819b9cbc081","c_4089c2ad7c8dde16",null,"rc_6633a4efb6fe4323",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQggs><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-874"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-874"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-874 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-874>Python</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-874 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-874 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-874 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_8642f819b9cbc081","c_4089c2ad7c8dde16",null,"rc_6633a4efb6fe4323",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font m
|
|||
|
|
<span class=hljs-keyword>from</span> openai <span class=hljs-keyword>import</span> OpenAI
|
|||
|
|
|
|||
|
|
client = OpenAI(
|
|||
|
|
base_url=<span class=hljs-string>"http://127.0.0.1:8080/v1"</span>,
|
|||
|
|
api_key=<span class=hljs-string>"sk-local-dev-precision-token-77x"</span>
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
response = client.chat.completions.create(
|
|||
|
|
model=<span class=hljs-string>"qwen2.5-7b"</span>,
|
|||
|
|
messages=[{<span class=hljs-string>"role"</span>: <span class=hljs-string>"user"</span>, <span class=hljs-string>"content"</span>: <span class=hljs-string>"Optimise this SQL query..."</span>}]
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
print(response.choices[<span class=hljs-number>0</span>].message.content)
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=28>Using this architecture completely avoids the memory overhead of intermediate wrapper applications, ensuring direct interaction with your compiled <code data-path-to-node=28 data-index-in-node=147>llama.cpp</code> pipeline.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response gap complete"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-848 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-861 ng-star-inserted hide-action-bar"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-861"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-861 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-861 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_8642f819b9cbc081","c_4089c2ad7c8dde16",null,"rc_6633a4efb6fe4323",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-861 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_8642f819b9cbc081","c_4089c2ad7c8dde16",null,"rc_6633a4efb6fe4323",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _ngho
|
|||
|
|
│ Static Model Weights │
|
|||
|
|
│ (Gemma 4 Loaded in Memory) │
|
|||
|
|
└───────────────┬───────────────┘
|
|||
|
|
│
|
|||
|
|
┌────────────────────────┼────────────────────────┐
|
|||
|
|
▼ ▼ ▼
|
|||
|
|
┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐
|
|||
|
|
│ Slot 0 │ │ Slot 1 │ │ Slot 2 │
|
|||
|
|
│ (User A's Cache) │ │ (User B's Cache) │ │ (User C's Cache) │
|
|||
|
|
└──────────────────┘ └──────────────────┘ └──────────────────┘
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=9>When you specify <code data-path-to-node=9 data-index-in-node=17>-np 3</code> (3 parallel slots), <code data-path-to-node=9 data-index-in-node=43>llama-server</code> carves up your available graphics memory to support three independent notebooks simultaneously.<h3 data-path-to-node=11>How Multiple Requests Are Processed: Continuous Batching</h3><p data-path-to-node=12>If User A, User B, and User C are all connected to your server, slots allow them to interact at the exact same time without locking each other out. This is handled through a mechanism called <b data-path-to-node=12 data-index-in-node=191>Continuous Batching</b>.<ul data-path-to-node=13><li><p data-path-to-node=13,0,0><b data-path-to-node=13,0,0 data-index-in-node=0>The Slot Mapping:</b> When User A sends a prompt, the server assigns them to <code data-path-to-node=13,0,0 data-index-in-node=73>Slot 0</code>. When User B types a second later, they are routed to <code data-path-to-node=13,0,0 data-index-in-node=134>Slot 1</code>.</p><li><p data-path-to-node=13,1,0><b data-path-to-node=13,1,0 data-index-in-node=0>The Token Matrix:</b> The GPU doesn't process User A's entire answer, freeze, and then move to User B. Instead, on every single clock cycle (or "forward pass"), the GPU evaluates one token for <code data-path-to-node=13,1,0 data-index-in-node=189>Slot 0</code> <i data-path-to-node=13,1,0 data-index-in-node=196>and</i> one token for <code data-path-to-node=13,1,0 data-index-in-node=214>Slot 1</code> simultaneously, combining them into a single mathematical batch.</p><li><p data-path-to-node=13,2,0><b data-path-to-node=13,2,0 data-index-in-node=0>Complete Isolation:</b> Because <code data-path-to-node=13,2,0 data-index-in-node=28>Slot 0</code> and <code data-path-to-node=13,2,0 data-index-in-node=39>Slot 1</code> are physically separate blocks of memory, User A’s context never leaks into User B’s conversation.</p></ul><h3 data-path-to-node=15>Monitoring Slots in Your Terminal Logs</h3><p data-path-to-node=16>Because you've been looking at your terminal logs, you can actually see <code data-path-to-node=16 data-index-in-node=72>llama-server</code> actively managing these slots.<p data-path-to-node=17>When you first open a connection, you will see the server hunting for an empty desk:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-888 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-888 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_ce0e06b2a01ed7e4","c_4089c2ad7c8dde16",null,"rc_66a0a70433f81077",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQnws><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-888"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-888"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-888 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-888>Plaintext</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-888 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-888 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badg
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><ul data-path-to-node=19><li><p data-path-to-node=19,0,0><b data-path-to-node=19,0,0 data-index-in-node=0><code data-path-to-node=19,0,0 data-index-in-node=0>id 0</code></b>: This tells you the server picked Slot 0 to handle the request.</p><li><p data-path-to-node=19,1,0><b data-path-to-node=19,1,0 data-index-in-node=0><code data-path-to-node=19,1,0 data-index-in-node=0>selected slot by LCP similarity</code></b>: "LCP" stands for <i data-path-to-node=19,1,0 data-index-in-node=50>Longest Common Prefix</i>. The server is intelligently checking if this user has chatted before. If the beginning of their new prompt matches the history already sitting inside Slot 0's cache, it will reuse it, skipping the processing time for that old text.</p></ul><p data-path-to-node=20>When the generation finishes, you see the slot clear its desk but keep its notes:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-889 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-889 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_ce0e06b2a01ed7e4","c_4089c2ad7c8dde16",null,"rc_66a0a70433f81077",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQoAs><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-889"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-889"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-889 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-889>Plaintext</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-889 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-889 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-889 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge
|
|||
|
|
I srv update_slots: all slots are idle
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=22>The slot goes back into an "idle" state. The memory isn't wiped; it holds onto that user's history until a new task forces it to clear or until that same user sends a follow-up message.<h3 data-path-to-node=24>Why Slots Matter to Your Mac's Hardware</h3><p data-path-to-node=25>Slots are the ultimate balancing act between <b data-path-to-node=25 data-index-in-node=45>compute parallelism</b> and <b data-path-to-node=25 data-index-in-node=69>memory consumption</b>.<p data-path-to-node=26>Every slot you open requires a fixed reservation of unified memory to hold its tokens. If you configure a server with 4 slots and an 8,192 context window each, your Mac has to hold space for up to 32,768 total tokens in active RAM.<p data-path-to-node=27>This is exactly why your single-user setup (<code data-path-to-node=27 data-index-in-node=44>--parallel 1</code>) felt so fast—by reducing the slots down to one, you forced the server to put 100% of its matrix processing power and memory bandwidth into a single, un-batched execution line.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer gap complete"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-875 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-882 ng-star-inserted hide-action-bar"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-882"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-882 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-882 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_ce0e06b2a01ed7e4","c_4089c2ad7c8dde16",null,"rc_66a0a70433f81077",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-882 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-d
|
|||
|
|
-H <span class=hljs-string>"Content-Type: application/json"</span> \
|
|||
|
|
-d <span class=hljs-string>'{
|
|||
|
|
"id_slot": 0,
|
|||
|
|
"prompt": "Extract the names and roles from this log: [10:23] Admin Dave changed system settings. [10:24] Dev Sarah deployed a hotfix.",
|
|||
|
|
"temperature": 0.0,
|
|||
|
|
"n_predict": 128
|
|||
|
|
}'</span>
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=6>Slot 1: Running a Code Review Task</h3><p data-path-to-node=7>This command routes a code optimization request strictly to <b data-path-to-node=7 data-index-in-node=60>Slot 1</b>, ensuring its cache is entirely isolated from the data processing task.</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-906 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-906 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_2f2a0fc185581747","c_4089c2ad7c8dde16",null,"rc_0f36388758b09e1b",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQwws><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-906"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-906"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-906 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-906>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-906 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-906 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-906 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_2f2a0fc185581747","c_4089c2ad7c8dde16",null,"rc_0f36388758b09e1b",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-f
|
|||
|
|
-H <span class=hljs-string>"Content-Type: application/json"</span> \
|
|||
|
|
-d <span class=hljs-string>'{
|
|||
|
|
"id_slot": 1,
|
|||
|
|
"prompt": "Optimize this Python function for memory efficiency:\ndef process_data(lines):\n return [line.strip() for line in lines if line != \"\"]",
|
|||
|
|
"temperature": 0.2,
|
|||
|
|
"n_predict": 256
|
|||
|
|
}'</span>
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=9>Slot 2: Running a Creative Draft Task</h3><p data-path-to-node=10>This command targets <b data-path-to-node=10 data-index-in-node=21>Slot 2</b> to draft an email update, running completely in parallel with the other two operations if they hit the server at the same time.</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-907 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-907 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_2f2a0fc185581747","c_4089c2ad7c8dde16",null,"rc_0f36388758b09e1b",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQxAs><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-907"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-907"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-907 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-907>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-907 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-907 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-907 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_2f2a0fc185581747","c_4089c2ad7c8dde16",null,"rc_0f36388758b09e1b",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-
|
|||
|
|
-H <span class=hljs-string>"Content-Type: application/json"</span> \
|
|||
|
|
-d <span class=hljs-string>'{
|
|||
|
|
"id_slot": 2,
|
|||
|
|
"prompt": "Draft a concise status report email to engineering stakeholders stating that the Potato server migration to Ubuntu 26.04 is complete.",
|
|||
|
|
"temperature": 0.7,
|
|||
|
|
"n_predict": 2048
|
|||
|
|
}'</span>
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=13>What to Look for in Your Terminal Logs</h3><p data-path-to-node=14>When you fire these off (especially if you open three terminal windows and run them simultaneously), keep an eye on your <code data-path-to-node=14 data-index-in-node=121>llama-server</code> log output. You will see the server bypass its typical Longest Common Prefix (LCP) similarity matching and explicitly respect your routing commands:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-908 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-908 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_2f2a0fc185581747","c_4089c2ad7c8dde16",null,"rc_0f36388758b09e1b",null,null,"",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQxQs><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-908"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-908"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-908 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-908>Plaintext</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-908 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-908 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-908 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_2f2a0fc185581747","c_4089c2ad7c8dde16",null,"rc_0f36388758b09e1b",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-
|
|||
|
|
I slot processing: id 1 | task 1202 | processing 42 tokens...
|
|||
|
|
I slot processing: id 2 | task 1203 | processing 29 tokens...
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=16>If you send a follow-up request to <code data-path-to-node=16 data-index-in-node=35>id_slot: 1</code> with an appended prompt, the server will leverage the existing KV cache sitting in Slot 1's memory block, skipping the processing time for the Python code you sent it previously.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer contains-extensions-response gap complete"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-890 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-899 ng-star-inserted hide-action-bar"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-899"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-899 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-899 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_2f2a0fc185581747","c_4089c2ad7c8dde16",null,"rc_0f36388758b09e1b",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-899 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_2f2a0fc185581747","c_4089c2ad7c8dde16",null,"rc_0f36388758b09e1b",null,null,"",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><ul data-path-to-node=8><li><p data-path-to-node=8,0,0><b data-path-to-node=8,0,0 data-index-in-node=0>Dynamic Loading:</b> The server starts up instantly without loading anything into memory. The moment you ask for a specific model, it initializes it on demand.</p><li><p data-path-to-node=8,1,0><b data-path-to-node=8,1,0 data-index-in-node=0><code data-path-to-node=8,1,0 data-index-in-node=0>--models-max 3</code></b>: This sets an LRU (Least Recently Used) cache ceiling. If your Mac's memory fills up and you request a 4th model, the server automatically unloads the oldest idle model to free up VRAM, then drops the new one in.</p></ul><h3 data-path-to-node=9>2. The Configuration Preset (<code data-path-to-node=9 data-index-in-node=29>--models-preset</code>)</h3><p data-path-to-node=10>If you want granular control—like running a coding model with a large <code data-path-to-node=10 data-index-in-node=70>32k</code> context window while restricting a smaller conversational model to an agile <code data-path-to-node=10 data-index-in-node=150>4k</code> context window—you can map them out using an <code data-path-to-node=10 data-index-in-node=198>.ini</code> configuration file.<p data-path-to-node=11>Create a file named <code data-path-to-node=11 data-index-in-node=20>models.ini</code>:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-925 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-925 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_333b4057f09d1c3d","c_4089c2ad7c8dde16",null,"rc_1bf5051187508514",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQ5gs><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-925"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-925"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-925 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-925>Ini, TOML</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-925 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-925 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-92
|
|||
|
|
|
|||
|
|
<span class=hljs-comment># [*] defines default options inherited by all models</span>
|
|||
|
|
<span class=hljs-section>[*]</span>
|
|||
|
|
<span class=hljs-attr>n-gpu-layers</span> = <span class=hljs-number>99</span>
|
|||
|
|
<span class=hljs-attr>flash-attn</span> = <span class=hljs-literal>on</span>
|
|||
|
|
|
|||
|
|
<span class=hljs-section>[gemma4:12b]</span>
|
|||
|
|
<span class=hljs-attr>model</span> = models/gemma-<span class=hljs-number>4</span>-<span class=hljs-number>12</span>B-it-Q5_K_M.gguf
|
|||
|
|
<span class=hljs-attr>ctx-size</span> = <span class=hljs-number>8192</span>
|
|||
|
|
|
|||
|
|
<span class=hljs-section>[qwen2.5:7b-coder]</span>
|
|||
|
|
<span class=hljs-attr>model</span> = models/qwen2.<span class=hljs-number>5</span>-<span class=hljs-number>7</span>b-instruct-q4_k_m.gguf
|
|||
|
|
<span class=hljs-attr>ctx-size</span> = <span class=hljs-number>32768</span>
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=13>Then launch your backend using that configuration layout:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-926 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-926 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_333b4057f09d1c3d","c_4089c2ad7c8dde16",null,"rc_1bf5051187508514",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQ5ws><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-926"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-926"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-926 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-926>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-926 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-926 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-926 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_333b4057f09d1c3d","c_4089c2ad7c8dde16",null,"rc_1bf5051187508514",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=copy data-mat-icon-namespace=lumi-symbols fonticon=copy></mat-icon></gem-icon><span clas
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h2 data-path-to-node=16>How You Query It</h2><p data-path-to-node=17>Once the router engine is online, your orchestration layer or terminal commands just need to declare the target model inside the standard OpenAI formatting structure:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-927 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-927 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_333b4057f09d1c3d","c_4089c2ad7c8dde16",null,"rc_1bf5051187508514",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQ6As><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-927"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-927"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-927 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-927>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-927 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-927 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-927 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_333b4057f09d1c3d","c_4089c2ad7c8dde16",null,"rc_1bf5051187508514",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-ins
|
|||
|
|
-H <span class=hljs-string>"Content-Type: application/json"</span> \
|
|||
|
|
-d <span class=hljs-string>'{
|
|||
|
|
"model": "qwen2.5:7b-coder",
|
|||
|
|
"messages": [{"role": "user", "content": "Write a clean TypeScript interface..."}]
|
|||
|
|
}'</span>
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h2 data-path-to-node=20>Under the Hood: Multi-Process Isolation</h2><p data-path-to-node=21>The architecture <code data-path-to-node=21 data-index-in-node=17>llama-server</code> uses for this is incredibly elegant. Instead of cramming multiple models into a single thread where a memory corruption bug could crash the whole stack, the router relies on a <b data-path-to-node=21 data-index-in-node=206>multi-process architecture</b>.<p data-path-to-node=22>The main <code data-path-to-node=22 data-index-in-node=9>llama-server</code> process acts strictly as a lightweight traffic controller. When a request hits, it spawns or signals a dedicated worker subprocess specifically tasked with executing that chosen GGUF. If a single experimental model hitches or throws an exception, it crashes silently in its own isolated container—the main gateway server and your other active models remain completely unaffected.<h3 data-path-to-node=23>The Apple Silicon VRAM Reality Check</h3><p data-path-to-node=24>While this is incredibly convenient, the physical laws of your hardware still apply. If you configure your preset to keep multiple models resident in memory simultaneously without unloading them, their combined size (including their active context KV caches) cannot exceed your Mac's available unified memory space. If it overflows, the OS will start swap-paging to the SSD, dragging your inference speeds right down.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer gap complete"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-909 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-918 ng-star-inserted hide-action-bar"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-918"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-918 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-918 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_333b4057f09d1c3d","c_4089c2ad7c8dde16",null,"rc_1bf5051187508514",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-918 lumino
|
|||
|
|
Hello<turn|>
|
|||
|
|
<|turn>model
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=17>Every model provider creates a unique formatting structure. The <code data-path-to-node=17 data-index-in-node=64>.jinja</code> templates tell <code data-path-to-node=17 data-index-in-node=86>llama-server</code> exactly how to wrap your text for dozens of different industry models (like DeepSeek-R1, Mistral, and Llama 3.3). If a brand-new model launches, <code data-path-to-node=17 data-index-in-node=244>llama.cpp</code> developers can simply drop a new <code data-path-to-node=17 data-index-in-node=287>.jinja</code> file here to support its custom chat wrapper instantly without rewriting the core C++ engine.<h3 data-path-to-node=19>Summary Checklist</h3><ul data-path-to-node=20><li><p data-path-to-node=20,0,0><b data-path-to-node=20,0,0 data-index-in-node=0>Your Actual Models:</b> <code data-path-to-node=20,0,0 data-index-in-node=20>gemma-4-12B-it-Q5_K_M.gguf</code> (The only real brain in the room right now).</p><li><p data-path-to-node=20,1,0><b data-path-to-node=20,1,0 data-index-in-node=0><code data-path-to-node=20,1,0 data-index-in-node=0>ggml-vocab-*</code>:</b> Low-level developer calibration charts for word-chopping validation.</p><li><p data-path-to-node=20,2,0><b data-path-to-node=20,2,0 data-index-in-node=0><code data-path-to-node=20,2,0 data-index-in-node=0>templates/*.jinja</code>:</b> Formatting recipes used to structure raw user text into clear dialogue turns for different AI architectures.</p></ul><p data-path-to-node=21>You can safely leave them right where they are; they occupy very little disk space and serve as the blueprint dictionary that allows your <code data-path-to-node=21 data-index-in-node=138>llama-server</code> binary to be so highly compatible with whatever <i data-path-to-node=21 data-index-in-node=199>actual</i> large GGUF files you decide to download via <code data-path-to-node=21 data-index-in-node=250>curl</code> later!</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer gap complete"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-928 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-931 ng-star-inserted hide-action-bar"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-931"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-931 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-931 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_9f9367739f3c08fc","c_4089c2ad7c8dde16",null,"rc_f6a227929589c006",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-
|
|||
|
|
-m models/gemma-4-12B-it-Q5_K_M.gguf \
|
|||
|
|
--port 8080 \
|
|||
|
|
-c 262144 \
|
|||
|
|
--cache-type-k q4_0 \
|
|||
|
|
--cache-type-v q4_0 \
|
|||
|
|
--flash-attn \
|
|||
|
|
--parallel 1
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=9>Once that server is running with that allocation, your <code data-path-to-node=9 data-index-in-node=55>curl</code> statements interact with it completely normally. The client doesn't need to specify context space; it can simply write queries back-to-back, knowing the engine has the runway to hold 256,000 tokens of conversation history.<h2 data-path-to-node=11>2. Managing Your Project Requirements Document</h2><p data-path-to-node=12>You <b data-path-to-node=12 data-index-in-node=4>do not</b> have to re-feed your Requirements Document into <code data-path-to-node=12 data-index-in-node=59>curl</code> with every single prompt manually. <code data-path-to-node=12 data-index-in-node=99>llama-server</code> features an extremely efficient optimization pattern designed exactly for this use case: <b data-path-to-node=12 data-index-in-node=201>Smart Prefix Caching</b>.<p data-path-to-node=13>Because the text of your software requirements document is static and won't change from turn to turn, <code data-path-to-node=13 data-index-in-node=102>llama-server</code> evaluates it <b data-path-to-node=13 data-index-in-node=128>exactly once</b> on your first prompt, creates a cryptographic checkpoint of it in your Mac's RAM, and locks it down.<p data-path-to-node=14>On every subsequent prompt, the server matches the text, detects the pre-computed math block, skips processing entirely, and jumps straight to generating code or answering your questions instantly.<h3 data-path-to-node=15>How to Guarantee Your Document is Always Used</h3><p data-path-to-node=16>There are two primary strategies to lock this document down into the server context layout:<h3 data-path-to-node=17>Strategy A: The System Prompt File (Highly Recommended)</h3><p data-path-to-node=18>If you want this requirements document to serve as the absolute baseline foundation for <i data-path-to-node=18 data-index-in-node=88>every single interaction</i> hit on that server, save your markdown text to a file named <code data-path-to-node=18 data-index-in-node=173>requirements.md</code> and pass it directly to the server binary at startup using the <code data-path-to-node=18 data-index-in-node=252>--system-prompt-file</code> flag:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-951 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-951 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_8bec0995437eb1b6","c_4089c2ad7c8dde16",null,"rc_78953126a2cdccf9",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQpQw><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-951"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-951"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-951 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-951>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-951 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-951 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=ma
|
|||
|
|
-m models/gemma-4-12B-it-Q5_K_M.gguf \
|
|||
|
|
--port 8080 \
|
|||
|
|
-c 262144 \
|
|||
|
|
--cache-type-k q4_0 \
|
|||
|
|
--cache-type-v q4_0 \
|
|||
|
|
--flash-attn \
|
|||
|
|
--parallel 1 \
|
|||
|
|
--system-prompt-file requirements.md
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><ul data-path-to-node=20><li><p data-path-to-node=20,0,0><b data-path-to-node=20,0,0 data-index-in-node=0>How it behaves:</b> When the backend loads, it processes your markdown project rules immediately into Slot 0.</p><li><p data-path-to-node=20,1,0><b data-path-to-node=20,1,0 data-index-in-node=0>Your cURL command:</b> Your API calls can now remain completely clean and short. You don't need to pass the file text in the JSON array; the model already "knows" it as its core operating reality.</p></ul><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-952 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-952 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_8bec0995437eb1b6","c_4089c2ad7c8dde16",null,"rc_78953126a2cdccf9",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQpgw><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-952"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-952"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-952 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-952>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-952 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-952 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-952 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_8bec0995437eb1b6","c_4089c2ad7c8dde16",null,"rc_78953126a2cdccf9",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="m
|
|||
|
|
-H <span class=hljs-string>"Content-Type: application/json"</span> \
|
|||
|
|
-d <span class=hljs-string>'{
|
|||
|
|
"messages": [{"role": "user", "content": "Write the TypeScript interface for the authentication module described in the document."}]
|
|||
|
|
}'</span>
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><h3 data-path-to-node=22>Strategy B: Standard Conversation History (The Ordered Array)</h3><p data-path-to-node=23>If you prefer not to lock the server boot sequence to a single document, you can send the document text inside your very first API transaction. To ensure it caches perfectly and remains persistent, you must place the static file text at the <b data-path-to-node=23 data-index-in-node=241>very beginning</b> of the conversation structure, wrapped inside the <code data-path-to-node=23 data-index-in-node=306>system</code> role block:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-953 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-953 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_8bec0995437eb1b6","c_4089c2ad7c8dde16",null,"rc_78953126a2cdccf9",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQpww><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-953"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-953"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-953 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-953>Bash</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-953 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-953 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-953 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_8bec0995437eb1b6","c_4089c2ad7c8dde16",null,"rc_78953126a2cdccf9",null,null,"en",null,
|
|||
|
|
-H <span class=hljs-string>"Content-Type: application/json"</span> \
|
|||
|
|
-d <span class=hljs-string>'{
|
|||
|
|
"messages": [
|
|||
|
|
{
|
|||
|
|
"role": "system",
|
|||
|
|
"content": "# Software Project Requirements...\n[Insert your full Markdown Document Text Here]"
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"role": "user",
|
|||
|
|
"content": "Based on the specification above, map out our primary PostgreSQL database schema."
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
}'</span>
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=25>On your second turn, when you append your next query to that array, <code data-path-to-node=25 data-index-in-node=68>llama-server</code> evaluates your text, calculates the <b data-path-to-node=25 data-index-in-node=117>Longest Common Prefix</b>, notes that the massive <code data-path-to-node=25 data-index-in-node=163>system</code> string matches its internal cache checkpoint precisely, and retains it seamlessly without running a full document re-read.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer gap complete"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-935 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-944 ng-star-inserted hide-action-bar"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-944"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-944 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-944 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_8bec0995437eb1b6","c_4089c2ad7c8dde16",null,"rc_78953126a2cdccf9",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-m lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=thumb_up data-mat-icon-namespace=lumi-symbols fonticon=thumb_up></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button></thumb-up-button><thumb-down-button _ngcontent-ng-c3132230974 _nghost-ng-c3072525816 class="ng-tns-c3132230974-944 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c3072525816 tabindex=-1 theme=lm type=onSurface arialabel="Bad response" gemtooltip="Bad response" data-test-id=thumb-down-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-217 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Bad response" aria-pressed=false jslog='173904;track:generic_click,impression;BardVeMetadataKey:[["r_8bec0995437eb1b6","c_4089c2a
|
|||
|
|
With Docling's JSON output, every single piece of text is explicitly boxed with parent metadata:</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-960 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-960 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_a63966294963063b","c_4089c2ad7c8dde16",null,"rc_24fc3c95a3de25d5",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQtww><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-960"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-960"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-960 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-960>JSON</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-960 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-960 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=arrow_circle_down data-mat-icon-namespace=lumi-symbols fonticon=arrow_circle_down></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-mdc-button-touch-target sf-hidden"></span></button></gem-icon-button><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Copy code" gemtooltip="Copy code" data-test-id=gem-copy-button class="mat-mdc-tooltip-trigger copy-button ng-tns-c2960786434-960 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-225 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Copy code" jslog='179062;track:generic_click,impression;BardVeMetadataKey:[["r_a63966294963063b","c_4089c2ad7c8dde16",null,"rc_24fc3c95a3de25d5",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notranslate lm-icon-l lumi-symbols mat-ligature-font mat-icon-no-color ng-star-inserted" aria-hidden=true data-mat-icon-type=font data-mat-icon-name=copy data-mat-icon-namespace=lumi-symbols fonticon=copy></mat-icon></gem-icon><span class=mat-focus-indicator></span><span class="mat-md
|
|||
|
|
<span class=hljs-attr>"chunk_id"</span>: <span class=hljs-string>"sec_04_auth"</span>,
|
|||
|
|
<span class=hljs-attr>"heading"</span>: <span class=hljs-string>"Authentication Module"</span>,
|
|||
|
|
<span class=hljs-attr>"content"</span>: <span class=hljs-string>"The system must use SHA-256 tokens..."</span>,
|
|||
|
|
<span class=hljs-attr>"page_number"</span>: <span class=hljs-number>12</span>,
|
|||
|
|
<span class=hljs-attr>"file_source"</span>: <span class=hljs-string>"spec_v2.md"</span>
|
|||
|
|
}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=9>If you pass this JSON object to the LLM, it inherits the absolute context of where that text came from, preventing it from getting "lost" in a long thread.<h3 data-path-to-node=10>Programmatic Pre-Filtering</h3><p data-path-to-node=11>If your local TypeScript or Python script wants to read your requirements document and selectively feed <i data-path-to-node=11 data-index-in-node=104>only</i> the database requirements to the LLM, parsing JSON is deterministic and trivial (<code data-path-to-node=11 data-index-in-node=190>JSON.parse()</code>). Parsing and slicing Markdown files requires writing error-prone regular expressions or string splits.<h2 data-path-to-node=13>2. Why Markdown Wins for LLM Reasoning (The Sweet Spot)</h2><p data-path-to-node=14>While JSON is perfect for your database or pipeline scripts, <b data-path-to-node=14 data-index-in-node=61>LLMs inherently "think" better in Markdown.</b><h3 data-path-to-node=15>Native Training Alignment</h3><p data-path-to-node=16>Large Language Models (including Gemma 4 and Qwen) are trained on vast crawls of open-source code repositories, technical documentation, and Wikipedia pages. The vast majority of structured text they ingest during training is written natively in <b data-path-to-node=16 data-index-in-node=246>Markdown</b>. The model natively understands that a <code data-path-to-node=16 data-index-in-node=294>#</code> means a major topic change, and <code data-path-to-node=16 data-index-in-node=328>-</code> represents an itemized array of logic constraints.<h3 data-path-to-node=17>The Token Overhead Tax</h3><p data-path-to-node=18>JSON is syntax-heavy. The constant repetition of quotation marks, curly braces, colons, and escaped characters (<code data-path-to-node=18 data-index-in-node=112>\n</code>, <code data-path-to-node=18 data-index-in-node=116>\"</code>) eats up your context window rapidly.</p><response-element ng-version=0.0.0-PLACEHOLDER><code-block _nghost-ng-c2960786434 class="ng-tns-c2960786434-961 enable-luminous-code-block ng-star-inserted"><div _ngcontent-ng-c2960786434 class="code-block ng-tns-c2960786434-961 ng-animate-disabled ng-trigger ng-trigger-codeBlockRevealAnimation" jslog='223238;track:impression,attention;BardVeMetadataKey:[["r_a63966294963063b","c_4089c2ad7c8dde16",null,"rc_24fc3c95a3de25d5",null,null,"en",null,1,null,null,1,0]]' style=display:block data-hveid=0 decode-data-ved=1 data-ved=0CAAQhtANahgKEwiPhs6--O-UAxUAAAAAHQAAAAAQuAw><div _ngcontent-ng-c2960786434 class="formatted-code-block-internal-container ng-tns-c2960786434-961"><div _ngcontent-ng-c2960786434 class="animated-opacity ng-tns-c2960786434-961"><div _ngcontent-ng-c2960786434 class="code-block-decoration header-formatted gds-emphasized-body-m ng-tns-c2960786434-961 ng-star-inserted"><span _ngcontent-ng-c2960786434 class=ng-tns-c2960786434-961>Plaintext</span><div _ngcontent-ng-c2960786434 class="buttons ng-tns-c2960786434-961 ng-star-inserted"><gem-icon-button _ngcontent-ng-c2960786434 tabindex=-1 type=onSurface size=small theme=lm arialabel="Download code" gemtooltip="Download code" class="mat-mdc-tooltip-trigger download-button ng-tns-c2960786434-961 gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-224 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Download code" jslog=305704;track:generic_click,impression><span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><gem-icon _ngcontent-ng-c496667975 _nghost-ng-c4072398418><mat-icon _ngcontent-ng-c4072398418 role=img class="mat-icon notr
|
|||
|
|
# Auth
|
|||
|
|
- Must use SHA-256.
|
|||
|
|
|
|||
|
|
JSON:
|
|||
|
|
{"section": "Auth", "rules": ["Must use SHA-256."]}
|
|||
|
|
</code></pre></div></div></div></code-block></response-element><p data-path-to-node=20>Over a 100-page requirements specification, JSON formatting can introduce <b data-path-to-node=20 data-index-in-node=74>15% to 25% token bloat</b> compared to clean Markdown. On your local M3 Pro setup, that bloat directly translates to slower prompt processing speeds and a smaller effective memory window.<h3 data-path-to-node=21>Syntax Fragmentation Risk</h3><p data-path-to-node=22>If you pass a massive JSON structure and it gets cut off because you hit a token ceiling, the trailing brackets (<code data-path-to-node=22 data-index-in-node=113>}</code>) will be missing. An invalid, broken JSON string frequently confuses local LLMs, causing them to hallucinate or throw formatting errors. If a Markdown document gets cut off, the LLM treats it naturally as an incomplete sentence and continues working with what it has.<h2 data-path-to-node=24>The Industry Standard Architecture: A Hybrid Pipeline</h2><p data-path-to-node=25>Because you are using Docling, you don't actually have to choose one over the other. The standard, robust architecture for local software engineering agents utilizes <b data-path-to-node=25 data-index-in-node=166>both</b>:<ol start=1 data-path-to-node=26><li><p data-path-to-node=26,0,0><b data-path-to-node=26,0,0 data-index-in-node=0>Docling processes the document</b> and outputs the rich structural layout.</p><li><p data-path-to-node=26,1,0><b data-path-to-node=26,1,0 data-index-in-node=0>Your application uses JSON</b> to track the metadata, chunk IDs, and document sections cleanly behind the scenes.</p><li><p data-path-to-node=26,2,0><b data-path-to-node=26,2,0 data-index-in-node=0>Your application strips the JSON wrapping</b> and injects just the raw, clean <b data-path-to-node=26,2,0 data-index-in-node=74>Markdown content chunk</b> into the LLM system prompt window for the model to read and reason against.</p></ol><p data-path-to-node=27>This hybrid approach gives your backend code strict programmatic control via JSON, while feeding your local Gemma 4 model the clean, token-efficient Markdown it needs to generate accurate code.</p></div></message-content></div></structured-content-container><div _ngcontent-ng-c557908647 class="response-footer gap complete"><sources-list _ngcontent-ng-c557908647 class="sources-list ng-star-inserted"></sources-list></div></div><sensitive-memories-banner _ngcontent-ng-c557908647 _nghost-ng-c3271088061 class=ng-star-inserted></sensitive-memories-banner></div></div><div _ngcontent-ng-c3135600295 class="response-container-footer ng-tns-c3135600295-954 ng-star-inserted"><message-actions _ngcontent-ng-c557908647 footer _nghost-ng-c3132230974 class="ng-tns-c3132230974-959 ng-star-inserted"><div _ngcontent-ng-c3132230974 class="actions-container-v2 ng-tns-c3132230974-959"><div _ngcontent-ng-c3132230974 class="buttons-container-v2 ng-tns-c3132230974-959 ng-star-inserted"><thumb-up-button _ngcontent-ng-c3132230974 _nghost-ng-c410953545 class="ng-tns-c3132230974-959 luminous-theme ng-star-inserted"><gem-icon-button _ngcontent-ng-c410953545 tabindex=-1 theme=lm type=onSurface arialabel="Good response" gemtooltip="Good response" data-test-id=thumb-up-button class="mat-mdc-tooltip-trigger gem-button gem-button-badge-size-small gem-button-size-small gem-button-type-on-surface lm-enabled ng-star-inserted" _nghost-ng-c496667975 aria-describedby=cdk-describedby-message-ng-1-216 cdk-describedby-host=ng-1><button _ngcontent-ng-c496667975 maticonbutton matbadgeposition=after class="mdc-icon-button mat-mdc-icon-button mat-mdc-button-base mat-badge mat-unthemed mat-badge-overlap mat-badge-above mat-badge-after mat-badge-small mat-badge-hidden ng-star-inserted" mat-ripple-loader-uninitialized mat-ripple-loader-class-name=mat-mdc-button-ripple mat-ripple-loader-centered aria-label="Good response" aria-pressed=false jslog='173913;track:generic_click,impression;BardVeMetadataKey:[["r_a63966294963063b","c_4089c2ad7c8dde16",null,"rc_24fc3c95a3de25d5",null,null,"en",null,1,null,null,1,0]];mutable:true'><span class=
|