Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Implementarea Optimizată a Inferenței","local":"implementarea-optimizată-a-inferenței","sections":[{"title":"Ghid de Selecție a Framework-ului","local":"ghid-de-selecție-a-framework-ului","sections":[{"title":"Gestionarea Memoriei și Performanța","local":"gestionarea-memoriei-și-performanța","sections":[],"depth":3},{"title":"Implementarea și Integrarea","local":"implementarea-și-integrarea","sections":[],"depth":3}],"depth":2},{"title":"Începerea","local":"începerea","sections":[{"title":"Instalarea și Configurarea de Bază","local":"instalarea-și-configurarea-de-bază","sections":[],"depth":3},{"title":"Generarea de Bază a Textului","local":"generarea-de-bază-a-textului","sections":[],"depth":3}],"depth":2},{"title":"Controlul Avansat al Generării","local":"controlul-avansat-al-generării","sections":[{"title":"Selecția Token-urilor și Eșantionarea","local":"selecția-token-urilor-și-eșantionarea","sections":[],"depth":3},{"title":"Controlul Repetării","local":"controlul-repetării","sections":[],"depth":3},{"title":"Controlul Lungimii și Secvențele de Oprire","local":"controlul-lungimii-și-secvențele-de-oprire","sections":[],"depth":3}],"depth":2},{"title":"Gestionarea Memoriei","local":"gestionarea-memoriei","sections":[],"depth":2},{"title":"Resurse","local":"resurse","sections":[],"depth":2}],"depth":1}"> | |
| <link href="/docs/course/pr_1069/rum/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> | |
| <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/entry/start.1de7c3d2.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/scheduler.37c15a92.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/singletons.e13b7dfd.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/index.18351ede.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/paths.e130b7b0.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/entry/app.1f82014c.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/index.2bf4358c.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/nodes/0.3c83e1ab.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/each.e59479a4.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/nodes/42.5af8cdc1.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/Tip.363c041f.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/CodeBlock.4e987730.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/getInferenceSnippets.24b50994.js"> | |
| <link rel="modulepreload" href="/docs/course/pr_1069/rum/_app/immutable/chunks/stores.cb4752a8.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Implementarea Optimizată a Inferenței","local":"implementarea-optimizată-a-inferenței","sections":[{"title":"Ghid de Selecție a Framework-ului","local":"ghid-de-selecție-a-framework-ului","sections":[{"title":"Gestionarea Memoriei și Performanța","local":"gestionarea-memoriei-și-performanța","sections":[],"depth":3},{"title":"Implementarea și Integrarea","local":"implementarea-și-integrarea","sections":[],"depth":3}],"depth":2},{"title":"Începerea","local":"începerea","sections":[{"title":"Instalarea și Configurarea de Bază","local":"instalarea-și-configurarea-de-bază","sections":[],"depth":3},{"title":"Generarea de Bază a Textului","local":"generarea-de-bază-a-textului","sections":[],"depth":3}],"depth":2},{"title":"Controlul Avansat al Generării","local":"controlul-avansat-al-generării","sections":[{"title":"Selecția Token-urilor și Eșantionarea","local":"selecția-token-urilor-și-eșantionarea","sections":[],"depth":3},{"title":"Controlul Repetării","local":"controlul-repetării","sections":[],"depth":3},{"title":"Controlul Lungimii și Secvențele de Oprire","local":"controlul-lungimii-și-secvențele-de-oprire","sections":[],"depth":3}],"depth":2},{"title":"Gestionarea Memoriei","local":"gestionarea-memoriei","sections":[],"depth":2},{"title":"Resurse","local":"resurse","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="implementarea-optimizată-a-inferenței" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#implementarea-optimizată-a-inferenței"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Implementarea Optimizată a Inferenței</span></h1> <p data-svelte-h="svelte-l25pl2">În această secțiune, vom explora framework-uri avansate pentru optimizarea implementărilor LLM: Text Generation Inference (TGI), vLLM și llama.cpp. Aceste aplicații sunt utilizate în principal în medii de producție pentru a servi LLM-uri utilizatorilor. Această secțiune se concentrează pe modul de implementare a acestor framework-uri în producție, mai degrabă decât pe modul de utilizare pentru inferență pe o singură mașină.</p> <p data-svelte-h="svelte-ieh7in">Vom acoperi modul în care aceste instrumente maximizează eficiența inferenței și simplifică implementările de producție ale Modelelor de Limbaj de Mari Dimensiuni.</p> <h2 class="relative group"><a id="ghid-de-selecție-a-framework-ului" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#ghid-de-selecție-a-framework-ului"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Ghid de Selecție a Framework-ului</span></h2> <p data-svelte-h="svelte-a0brur">TGI, vLLM și llama.cpp servesc scopuri similare, dar au caracteristici distincte care le fac mai potrivite pentru diferite cazuri de utilizare. Să ne uităm la diferențele cheie dintre ele, concentrându-ne pe performanță și integrare.</p> <h3 class="relative group"><a id="gestionarea-memoriei-și-performanța" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#gestionarea-memoriei-și-performanța"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Gestionarea Memoriei și Performanța</span></h3> <p data-svelte-h="svelte-bi6vas"><strong>TGI</strong> este proiectat să fie stabil și previzibil în producție, folosind lungimi fixe de secvență pentru a menține utilizarea memoriei consistentă. TGI gestionează memoria folosind Flash Attention 2 și tehnici de procesare continuă în loturi. Aceasta înseamnă că poate procesa calculele de atenție foarte eficient și poate menține GPU-ul ocupat prin alimentarea constantă cu lucru. Sistemul poate muta părți ale modelului între CPU și GPU când este necesar, ceea ce ajută la gestionarea modelelor mai mari.</p> <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/tgi/flash-attn.png" alt="Flash Attention"> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">Flash Attention este o tehnică care optimizează mecanismul de atenție în modelele transformer prin abordarea blocajelor de lățime de bandă a memoriei. Așa cum s-a discutat mai devreme în [Capitolul 1.8](/course/chapter1/8), mecanismul de atenție are complexitate și utilizare de memorie pătratică, făcându-l ineficient pentru secvențe lungi. | |
| <p data-svelte-h="svelte-bck4r0">Inovația cheie constă în modul în care gestionează transferurile de memorie între High Bandwidth Memory (HBM) și cache-ul SRAM mai rapid. Atenția tradițională transferă în mod repetat date între HBM și SRAM, creând blocaje prin lăsarea GPU-ului inactiv. Flash Attention încarcă datele o dată în SRAM și efectuează toate calculele acolo, minimizând transferurile costisitoare de memorie.</p> <p data-svelte-h="svelte-89j1zd">Deși beneficiile sunt cele mai semnificative în timpul antrenamentului, utilizarea redusă de VRAM și eficiența îmbunătățită a Flash Attention o fac valoroasă și pentru inferență, permițând servirea LLM mai rapidă și mai scalabilă.</p></div> <p data-svelte-h="svelte-zoy9o2"><strong>vLLM</strong> adoptă o abordare diferită prin utilizarea PagedAttention. La fel cum un computer își gestionează memoria în pagini, vLLM împarte memoria modelului în blocuri mai mici. Acest sistem inteligent înseamnă că poate gestiona cereri de dimensiuni diferite mai flexibil și nu risipește spațiul de memorie. Este deosebit de bun la partajarea memoriei între diferite cereri și reduce fragmentarea memoriei, ceea ce face întregul sistem mai eficient.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">PagedAttention este o tehnică care abordează un alt blocaj critic în inferența LLM: gestionarea memoriei cache KV. Așa cum s-a discutat în [Capitolul 1.8](/course/chapter1/8), în timpul generării de text, modelul stochează cheile și valorile de atenție (cache KV) pentru fiecare token generat pentru a reduce calculele redundante. Cache-ul KV poate deveni enorm, în special cu secvențe lungi sau multiple cereri concurente. | |
| <p data-svelte-h="svelte-1q7w70u">Inovația cheie a vLLM constă în modul în care gestionează acest cache:</p> <ol data-svelte-h="svelte-o4cwji"><li><strong>Paginarea Memoriei</strong>: În loc să trateze cache-ul KV ca un bloc mare, este împărțit în “pagini” de dimensiune fixă (similar cu memoria virtuală în sistemele de operare).</li> <li><strong>Stocare Non-Contigua</strong>: Paginile nu trebuie să fie stocate contiguu în memoria GPU, permițând o alocare mai flexibilă a memoriei.</li> <li><strong>Gestionarea Tabelului de Pagini</strong>: Un tabel de pagini urmărește care pagini aparțin cărei secvențe, permițând căutare și acces eficient.</li> <li><strong>Partajarea Memoriei</strong>: Pentru operații precum eșantionarea paralelă, paginile care stochează cache-ul KV pentru prompt pot fi partajate între multiple secvențe.</li></ol> <p data-svelte-h="svelte-1hypydk">Abordarea PagedAttention poate duce la un throughput de până la 24 de ori mai mare comparativ cu metodele tradiționale, făcând-o o schimbare de paradigmă pentru implementările LLM de producție. Dacă vrei să înțelegi cu adevărat în profunzime cum funcționează PagedAttention, poți citi <a href="https://docs.vllm.ai/en/latest/design/kernel/paged_attention.html" rel="nofollow">ghidul din documentația vLLM</a>.</p></div> <p data-svelte-h="svelte-bz5wux"><strong>llama.cpp</strong> este o implementare C/C++ extrem de optimizată proiectată inițial pentru rularea modelelor LLaMA pe hardware de consum. Se concentrează pe eficiența CPU cu accelerație GPU opțională și este ideal pentru medii cu resurse limitate. llama.cpp folosește tehnici de cuantificare pentru a reduce dimensiunea modelului și cerințele de memorie menținând în același timp performanță bună. Implementează kerneluri optimizate pentru diverse arhitecturi CPU și suportă gestionarea de bază a cache-ului KV pentru generarea eficientă de token-uri.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">Cuantificarea în llama.cpp reduce precizia ponderilor modelului de la puncte mobile pe 32-bit sau 16-bit la formate de precizie mai mică precum întregi pe 8-bit (INT8), 4-bit sau chiar mai mic. Aceasta reduce semnificativ utilizarea memoriei și îmbunătățește viteza de inferență cu pierderi minime de calitate. | |
| <p data-svelte-h="svelte-1wsf8af">Caracteristicile cheie de cuantificare în llama.cpp includ:</p> <ol data-svelte-h="svelte-6dlvtp"><li><strong>Multiple Niveluri de Cuantificare</strong>: Suportă cuantificare pe 8-bit, 4-bit, 3-bit și chiar 2-bit</li> <li><strong>Format GGML/GGUF</strong>: Folosește formate tensoriale personalizate optimizate pentru inferența cuantificată</li> <li><strong>Precizie Mixtă</strong>: Poate aplica diferite niveluri de cuantificare la diferite părți ale modelului</li> <li><strong>Optimizări Specifice Hardware</strong>: Include căi de cod optimizate pentru diverse arhitecturi CPU (AVX2, AVX-512, NEON)</li></ol> <p data-svelte-h="svelte-1fu05c7">Această abordare permite rularea modelelor cu miliarde de parametri pe hardware de consum cu memorie limitată, făcând-o perfectă pentru implementări locale și dispozitive de margine.</p></div> <h3 class="relative group"><a id="implementarea-și-integrarea" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#implementarea-și-integrarea"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Implementarea și Integrarea</span></h3> <p data-svelte-h="svelte-rjqdb8">Să trecem la diferențele de implementare și integrare între framework-uri.</p> <p data-svelte-h="svelte-vkzd9m"><strong>TGI</strong> excelează în implementarea la nivel enterprise cu caracteristicile sale gata pentru producție. Vine cu suport Kubernetes integrat și include tot ce ai nevoie pentru rularea în producție, precum monitorizarea prin Prometheus și Grafana, scalarea automată și caracteristici cuprinzătoare de siguranță. Sistemul include, de asemenea, logare de nivel enterprise și diverse măsuri de protecție precum filtrarea conținutului și limitarea ratei pentru a menține implementarea sigură și stabilă.</p> <p data-svelte-h="svelte-1gkp50b"><strong>vLLM</strong> adoptă o abordare mai flexibilă, prietenoasă cu dezvoltatorii pentru implementare. Este construit cu Python în nucleu și poate înlocui cu ușurință API-ul OpenAI în aplicațiile tale existente. Framework-ul se concentrează pe livrarea performanței brute și poate fi personalizat pentru a se potrivi nevoilor tale specifice. Funcționează deosebit de bine cu Ray pentru gestionarea clusterelor, făcându-l o alegere excelentă când ai nevoie de performanță înaltă și adaptabilitate.</p> <p data-svelte-h="svelte-ej2h50"><strong>llama.cpp</strong> prioritizează simplitatea și portabilitatea. Implementarea sa de server este ușoară și poate rula pe o gamă largă de hardware, de la servere puternice la laptopuri de consum și chiar unele dispozitive mobile de înaltă performanță. Cu dependențe minime și un nucleu C/C++ simplu, este ușor de implementat în medii unde instalarea framework-urilor Python ar fi provocatoare. Serverul oferă un API compatibil cu OpenAI menținând în același timp un amprentă de resurse mult mai mică decât alte soluții.</p> <h2 class="relative group"><a id="începerea" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#începerea"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Începerea</span></h2> <p data-svelte-h="svelte-k038du">Să explorăm cum să folosim aceste framework-uri pentru implementarea LLM-urilor, începând cu instalarea și configurarea de bază.</p> <h3 class="relative group"><a id="instalarea-și-configurarea-de-bază" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#instalarea-și-configurarea-de-bază"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Instalarea și Configurarea de Bază</span></h3> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"></div> <div class="language-select"><hfoption value="tgi" label="TGI"> | |
| <p data-svelte-h="svelte-d3ibl6">TGI este ușor de instalat și utilizat, cu integrare profundă în ecosistemul Hugging Face.</p> <p data-svelte-h="svelte-5he4zd">Mai întâi, lansează serverul TGI folosind Docker:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->docker run --gpus all \ | |
| --shm-size 1g \ | |
| -p 8080:80 \ | |
| -v ~/.cache/huggingface:/data \ | |
| ghcr.io/huggingface/text-generation-inference:latest \ | |
| --model-id HuggingFaceTB/SmolLM2-360M-Instruct<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-njbfao">Apoi interacționează cu acesta folosind InferenceClient de la Hugging Face:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> InferenceClient | |
| <span class="hljs-comment"># Inițializează clientul pointând către endpoint-ul TGI</span> | |
| client = InferenceClient( | |
| model=<span class="hljs-string">"http://localhost:8080"</span>, <span class="hljs-comment"># URL către serverul TGI</span> | |
| ) | |
| <span class="hljs-comment"># Generarea de text</span> | |
| response = client.text_generation( | |
| <span class="hljs-string">"Spune-mi o poveste"</span>, | |
| max_new_tokens=<span class="hljs-number">100</span>, | |
| temperature=<span class="hljs-number">0.7</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| details=<span class="hljs-literal">True</span>, | |
| stop_sequences=[], | |
| ) | |
| <span class="hljs-built_in">print</span>(response.generated_text) | |
| <span class="hljs-comment"># Pentru format de chat</span> | |
| response = client.chat_completion( | |
| messages=[ | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"system"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Ești un asistent util."</span>}, | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Spune-mi o poveste"</span>}, | |
| ], | |
| max_tokens=<span class="hljs-number">100</span>, | |
| temperature=<span class="hljs-number">0.7</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-dmbprn">Alternativ, poți folosi clientul OpenAI:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> openai <span class="hljs-keyword">import</span> OpenAI | |
| <span class="hljs-comment"># Inițializează clientul pointând către endpoint-ul TGI</span> | |
| client = OpenAI( | |
| base_url=<span class="hljs-string">"http://localhost:8080/v1"</span>, <span class="hljs-comment"># Asigură-te să incluzi /v1</span> | |
| api_key=<span class="hljs-string">"not-needed"</span>, <span class="hljs-comment"># TGI nu necesită o cheie API în mod implicit</span> | |
| ) | |
| <span class="hljs-comment"># Completarea de chat</span> | |
| response = client.chat.completions.create( | |
| model=<span class="hljs-string">"HuggingFaceTB/SmolLM2-360M-Instruct"</span>, | |
| messages=[ | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"system"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Ești un asistent util."</span>}, | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Spune-mi o poveste"</span>}, | |
| ], | |
| max_tokens=<span class="hljs-number">100</span>, | |
| temperature=<span class="hljs-number">0.7</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!-- HTML_TAG_END --></pre></div> | |
| </hfoption> | |
| <hfoption value="llama.cpp" label="llama.cpp"> | |
| <p data-svelte-h="svelte-910ccc">llama.cpp este ușor de instalat și utilizat, necesitând dependențe minime și suportând atât inferența CPU cât și GPU.</p> <p data-svelte-h="svelte-yvknv6">Mai întâi, instalează și construiește llama.cpp:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Clonează depozitul</span> | |
| git <span class="hljs-built_in">clone</span> https://github.com/ggerganov/llama.cpp | |
| <span class="hljs-built_in">cd</span> llama.cpp | |
| <span class="hljs-comment"># Construiește proiectul</span> | |
| make | |
| <span class="hljs-comment"># Descarcă modelul SmolLM2-1.7B-Instruct-GGUF</span> | |
| curl -L -O https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF/resolve/main/smollm2-1.7b-instruct.Q4_K_M.gguf<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1t3nfo6">Apoi, lansează serverul (cu compatibilitate API OpenAI):</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Pornește serverul</span> | |
| ./server \ | |
| -m smollm2-1.7b-instruct.Q4_K_M.gguf \ | |
| --host 0.0.0.0 \ | |
| --port 8080 \ | |
| -c 4096 \ | |
| --n-gpu-layers 0 <span class="hljs-comment"># Setează la un număr mai mare pentru a folosi GPU</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-15q65ck">Interacționează cu serverul folosind InferenceClient de la Hugging Face:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> InferenceClient | |
| <span class="hljs-comment"># Inițializează clientul pointând către serverul llama.cpp</span> | |
| client = InferenceClient( | |
| model=<span class="hljs-string">"http://localhost:8080/v1"</span>, <span class="hljs-comment"># URL către serverul llama.cpp</span> | |
| token=<span class="hljs-string">"sk-no-key-required"</span>, <span class="hljs-comment"># serverul llama.cpp necesită acest placeholder</span> | |
| ) | |
| <span class="hljs-comment"># Generarea de text</span> | |
| response = client.text_generation( | |
| <span class="hljs-string">"Spune-mi o poveste"</span>, | |
| max_new_tokens=<span class="hljs-number">100</span>, | |
| temperature=<span class="hljs-number">0.7</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| details=<span class="hljs-literal">True</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(response.generated_text) | |
| <span class="hljs-comment"># Pentru format de chat</span> | |
| response = client.chat_completion( | |
| messages=[ | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"system"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Ești un asistent util."</span>}, | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Spune-mi o poveste"</span>}, | |
| ], | |
| max_tokens=<span class="hljs-number">100</span>, | |
| temperature=<span class="hljs-number">0.7</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-dmbprn">Alternativ, poți folosi clientul OpenAI:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> openai <span class="hljs-keyword">import</span> OpenAI | |
| <span class="hljs-comment"># Inițializează clientul pointând către serverul llama.cpp</span> | |
| client = OpenAI( | |
| base_url=<span class="hljs-string">"http://localhost:8080/v1"</span>, | |
| api_key=<span class="hljs-string">"sk-no-key-required"</span>, <span class="hljs-comment"># serverul llama.cpp necesită acest placeholder</span> | |
| ) | |
| <span class="hljs-comment"># Completarea de chat</span> | |
| response = client.chat.completions.create( | |
| model=<span class="hljs-string">"smollm2-1.7b-instruct"</span>, <span class="hljs-comment"># Identificatorul modelului poate fi orice deoarece serverul încarcă doar un model</span> | |
| messages=[ | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"system"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Ești un asistent util."</span>}, | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Spune-mi o poveste"</span>}, | |
| ], | |
| max_tokens=<span class="hljs-number">100</span>, | |
| temperature=<span class="hljs-number">0.7</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!-- HTML_TAG_END --></pre></div> | |
| </hfoption> | |
| <hfoption value="vllm" label="vLLM"> | |
| <p data-svelte-h="svelte-1eczzi3">vLLM este ușor de instalat și utilizat, cu atât compatibilitatea API OpenAI cât și o interfață Python nativă.</p> <p data-svelte-h="svelte-sv1ope">Mai întâi, lansează serverul compatibil OpenAI vLLM:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->python -m vllm.entrypoints.openai.api_server \ | |
| --model HuggingFaceTB/SmolLM2-360M-Instruct \ | |
| --host 0.0.0.0 \ | |
| --port 8000<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-njbfao">Apoi interacționează cu acesta folosind InferenceClient de la Hugging Face:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> InferenceClient | |
| <span class="hljs-comment"># Inițializează clientul pointând către endpoint-ul vLLM</span> | |
| client = InferenceClient( | |
| model=<span class="hljs-string">"http://localhost:8000/v1"</span>, <span class="hljs-comment"># URL către serverul vLLM</span> | |
| ) | |
| <span class="hljs-comment"># Generarea de text</span> | |
| response = client.text_generation( | |
| <span class="hljs-string">"Spune-mi o poveste"</span>, | |
| max_new_tokens=<span class="hljs-number">100</span>, | |
| temperature=<span class="hljs-number">0.7</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| details=<span class="hljs-literal">True</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(response.generated_text) | |
| <span class="hljs-comment"># Pentru format de chat</span> | |
| response = client.chat_completion( | |
| messages=[ | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"system"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Ești un asistent util."</span>}, | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Spune-mi o poveste"</span>}, | |
| ], | |
| max_tokens=<span class="hljs-number">100</span>, | |
| temperature=<span class="hljs-number">0.7</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-dmbprn">Alternativ, poți folosi clientul OpenAI:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> openai <span class="hljs-keyword">import</span> OpenAI | |
| <span class="hljs-comment"># Inițializează clientul pointând către endpoint-ul vLLM</span> | |
| client = OpenAI( | |
| base_url=<span class="hljs-string">"http://localhost:8000/v1"</span>, | |
| api_key=<span class="hljs-string">"not-needed"</span>, <span class="hljs-comment"># vLLM nu necesită o cheie API în mod implicit</span> | |
| ) | |
| <span class="hljs-comment"># Completarea de chat</span> | |
| response = client.chat.completions.create( | |
| model=<span class="hljs-string">"HuggingFaceTB/SmolLM2-360M-Instruct"</span>, | |
| messages=[ | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"system"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Ești un asistent util."</span>}, | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Spune-mi o poveste"</span>}, | |
| ], | |
| max_tokens=<span class="hljs-number">100</span>, | |
| temperature=<span class="hljs-number">0.7</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!-- HTML_TAG_END --></pre></div> | |
| </hfoption></div> <h3 class="relative group"><a id="generarea-de-bază-a-textului" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#generarea-de-bază-a-textului"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Generarea de Bază a Textului</span></h3> <p data-svelte-h="svelte-1pfzauy">Să ne uităm la exemple de generare de text cu framework-urile:</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"></div> <div class="language-select"><hfoption value="tgi" label="TGI"> | |
| <p data-svelte-h="svelte-1vmaxnf">Mai întâi, implementează TGI cu parametri avansați:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->docker run --gpus all \ | |
| --shm-size 1g \ | |
| -p 8080:80 \ | |
| -v ~/.cache/huggingface:/data \ | |
| ghcr.io/huggingface/text-generation-inference:latest \ | |
| --model-id HuggingFaceTB/SmolLM2-360M-Instruct \ | |
| --max-total-tokens 4096 \ | |
| --max-input-length 3072 \ | |
| --max-batch-total-tokens 8192 \ | |
| --waiting-served-ratio 1.2<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1ajrx6o">Folosește InferenceClient pentru generarea flexibilă de text:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> InferenceClient | |
| client = InferenceClient(model=<span class="hljs-string">"http://localhost:8080"</span>) | |
| <span class="hljs-comment"># Exemplu de parametri avansați</span> | |
| response = client.chat_completion( | |
| messages=[ | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"system"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Ești un povestitor creativ."</span>}, | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Scrie o poveste creativă"</span>}, | |
| ], | |
| temperature=<span class="hljs-number">0.8</span>, | |
| max_tokens=<span class="hljs-number">200</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content) | |
| <span class="hljs-comment"># Generarea brută de text</span> | |
| response = client.text_generation( | |
| <span class="hljs-string">"Scrie o poveste creativă despre explorarea spațiului"</span>, | |
| max_new_tokens=<span class="hljs-number">200</span>, | |
| temperature=<span class="hljs-number">0.8</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| repetition_penalty=<span class="hljs-number">1.1</span>, | |
| do_sample=<span class="hljs-literal">True</span>, | |
| details=<span class="hljs-literal">True</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(response.generated_text)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-4b84cz">Sau folosește clientul OpenAI:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> openai <span class="hljs-keyword">import</span> OpenAI | |
| client = OpenAI(base_url=<span class="hljs-string">"http://localhost:8080/v1"</span>, api_key=<span class="hljs-string">"not-needed"</span>) | |
| <span class="hljs-comment"># Exemplu de parametri avansați</span> | |
| response = client.chat.completions.create( | |
| model=<span class="hljs-string">"HuggingFaceTB/SmolLM2-360M-Instruct"</span>, | |
| messages=[ | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"system"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Ești un povestitor creativ."</span>}, | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Scrie o poveste creativă"</span>}, | |
| ], | |
| temperature=<span class="hljs-number">0.8</span>, <span class="hljs-comment"># Mai mare pentru mai multă creativitate</span> | |
| ) | |
| <span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!-- HTML_TAG_END --></pre></div> | |
| </hfoption> | |
| <hfoption value="llama.cpp" label="llama.cpp"> | |
| <p data-svelte-h="svelte-gc9vgi">Pentru llama.cpp, poți seta parametri avansați când lansezi serverul:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->./server \ | |
| -m smollm2-1.7b-instruct.Q4_K_M.gguf \ | |
| --host 0.0.0.0 \ | |
| --port 8080 \ | |
| -c 4096 \ <span class="hljs-comment"># Dimensiunea contextului</span> | |
| --threads 8 \ <span class="hljs-comment"># Thread-uri CPU de folosit</span> | |
| --batch-size 512 \ <span class="hljs-comment"># Dimensiunea lotului pentru evaluarea prompt-ului</span> | |
| --n-gpu-layers 0 <span class="hljs-comment"># Straturi GPU (0 = doar CPU)</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1w27s98">Folosește InferenceClient:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> InferenceClient | |
| client = InferenceClient(model=<span class="hljs-string">"http://localhost:8080/v1"</span>, token=<span class="hljs-string">"sk-no-key-required"</span>) | |
| <span class="hljs-comment"># Exemplu de parametri avansați</span> | |
| response = client.chat_completion( | |
| messages=[ | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"system"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Ești un povestitor creativ."</span>}, | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Scrie o poveste creativă"</span>}, | |
| ], | |
| temperature=<span class="hljs-number">0.8</span>, | |
| max_tokens=<span class="hljs-number">200</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content) | |
| <span class="hljs-comment"># Pentru generarea directă de text</span> | |
| response = client.text_generation( | |
| <span class="hljs-string">"Scrie o poveste creativă despre explorarea spațiului"</span>, | |
| max_new_tokens=<span class="hljs-number">200</span>, | |
| temperature=<span class="hljs-number">0.8</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| repetition_penalty=<span class="hljs-number">1.1</span>, | |
| details=<span class="hljs-literal">True</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(response.generated_text)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-17sbnw3">Sau folosește clientul OpenAI pentru generare cu control asupra parametrilor de eșantionare:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> openai <span class="hljs-keyword">import</span> OpenAI | |
| client = OpenAI(base_url=<span class="hljs-string">"http://localhost:8080/v1"</span>, api_key=<span class="hljs-string">"sk-no-key-required"</span>) | |
| <span class="hljs-comment"># Exemplu de parametri avansați</span> | |
| response = client.chat.completions.create( | |
| model=<span class="hljs-string">"smollm2-1.7b-instruct"</span>, | |
| messages=[ | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"system"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Ești un povestitor creativ."</span>}, | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Scrie o poveste creativă"</span>}, | |
| ], | |
| temperature=<span class="hljs-number">0.8</span>, <span class="hljs-comment"># Mai mare pentru mai multă creativitate</span> | |
| top_p=<span class="hljs-number">0.95</span>, <span class="hljs-comment"># Probabilitatea eșantionării nucleus</span> | |
| frequency_penalty=<span class="hljs-number">0.5</span>, <span class="hljs-comment"># Reduce repetarea token-urilor frecvente</span> | |
| presence_penalty=<span class="hljs-number">0.5</span>, <span class="hljs-comment"># Reduce repetarea prin penalizarea token-urilor deja prezente</span> | |
| max_tokens=<span class="hljs-number">200</span>, <span class="hljs-comment"># Lungimea maximă de generare</span> | |
| ) | |
| <span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1k6lxxy">Poți folosi, de asemenea, biblioteca nativă llama.cpp pentru un control și mai mare:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Folosind pachetul llama-cpp-python pentru accesul direct la model</span> | |
| <span class="hljs-keyword">from</span> llama_cpp <span class="hljs-keyword">import</span> Llama | |
| <span class="hljs-comment"># Încarcă modelul</span> | |
| llm = Llama( | |
| model_path=<span class="hljs-string">"smollm2-1.7b-instruct.Q4_K_M.gguf"</span>, | |
| n_ctx=<span class="hljs-number">4096</span>, <span class="hljs-comment"># Dimensiunea ferestrei de context</span> | |
| n_threads=<span class="hljs-number">8</span>, <span class="hljs-comment"># Thread-uri CPU</span> | |
| n_gpu_layers=<span class="hljs-number">0</span>, <span class="hljs-comment"># Straturi GPU (0 = doar CPU)</span> | |
| ) | |
| <span class="hljs-comment"># Formatează prompt-ul conform formatului așteptat al modelului</span> | |
| prompt = <span class="hljs-string">"""<|im_start|>system | |
| Ești un povestitor creativ. | |
| <|im_end|> | |
| <|im_start|>user | |
| Scrie o poveste creativă | |
| <|im_end|> | |
| <|im_start|>assistant | |
| """</span> | |
| <span class="hljs-comment"># Generează răspunsul cu control precis al parametrilor</span> | |
| output = llm( | |
| prompt, | |
| max_tokens=<span class="hljs-number">200</span>, | |
| temperature=<span class="hljs-number">0.8</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| frequency_penalty=<span class="hljs-number">0.5</span>, | |
| presence_penalty=<span class="hljs-number">0.5</span>, | |
| stop=[<span class="hljs-string">"<|im_end|>"</span>], | |
| ) | |
| <span class="hljs-built_in">print</span>(output[<span class="hljs-string">"choices"</span>][<span class="hljs-number">0</span>][<span class="hljs-string">"text"</span>])<!-- HTML_TAG_END --></pre></div> | |
| </hfoption> | |
| <hfoption value="vllm" label="vLLM"> | |
| <p data-svelte-h="svelte-vd4z8t">Pentru utilizarea avansată cu vLLM, poți folosi InferenceClient:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> InferenceClient | |
| client = InferenceClient(model=<span class="hljs-string">"http://localhost:8000/v1"</span>) | |
| <span class="hljs-comment"># Exemplu de parametri avansați</span> | |
| response = client.chat_completion( | |
| messages=[ | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"system"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Ești un povestitor creativ."</span>}, | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Scrie o poveste creativă"</span>}, | |
| ], | |
| temperature=<span class="hljs-number">0.8</span>, | |
| max_tokens=<span class="hljs-number">200</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content) | |
| <span class="hljs-comment"># Pentru generarea directă de text</span> | |
| response = client.text_generation( | |
| <span class="hljs-string">"Scrie o poveste creativă despre explorarea spațiului"</span>, | |
| max_new_tokens=<span class="hljs-number">200</span>, | |
| temperature=<span class="hljs-number">0.8</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| details=<span class="hljs-literal">True</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(response.generated_text)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1ns2jx">Poți folosi, de asemenea, clientul OpenAI:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> openai <span class="hljs-keyword">import</span> OpenAI | |
| client = OpenAI(base_url=<span class="hljs-string">"http://localhost:8000/v1"</span>, api_key=<span class="hljs-string">"not-needed"</span>) | |
| <span class="hljs-comment"># Exemplu de parametri avansați</span> | |
| response = client.chat.completions.create( | |
| model=<span class="hljs-string">"HuggingFaceTB/SmolLM2-360M-Instruct"</span>, | |
| messages=[ | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"system"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Ești un povestitor creativ."</span>}, | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Scrie o poveste creativă"</span>}, | |
| ], | |
| temperature=<span class="hljs-number">0.8</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| max_tokens=<span class="hljs-number">200</span>, | |
| ) | |
| <span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1rrz7ab">vLLM oferă, de asemenea, o interfață Python nativă cu control fin:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> vllm <span class="hljs-keyword">import</span> LLM, SamplingParams | |
| <span class="hljs-comment"># Inițializează modelul cu parametri avansați</span> | |
| llm = LLM( | |
| model=<span class="hljs-string">"HuggingFaceTB/SmolLM2-360M-Instruct"</span>, | |
| gpu_memory_utilization=<span class="hljs-number">0.85</span>, | |
| max_num_batched_tokens=<span class="hljs-number">8192</span>, | |
| max_num_seqs=<span class="hljs-number">256</span>, | |
| block_size=<span class="hljs-number">16</span>, | |
| ) | |
| <span class="hljs-comment"># Configurează parametrii de eșantionare</span> | |
| sampling_params = SamplingParams( | |
| temperature=<span class="hljs-number">0.8</span>, <span class="hljs-comment"># Mai mare pentru mai multă creativitate</span> | |
| top_p=<span class="hljs-number">0.95</span>, <span class="hljs-comment"># Consideră masa de probabilitate de top 95%</span> | |
| max_tokens=<span class="hljs-number">100</span>, <span class="hljs-comment"># Lungimea maximă</span> | |
| presence_penalty=<span class="hljs-number">1.1</span>, <span class="hljs-comment"># Reduce repetarea</span> | |
| frequency_penalty=<span class="hljs-number">1.1</span>, <span class="hljs-comment"># Reduce repetarea</span> | |
| stop=[<span class="hljs-string">"\n\n"</span>, <span class="hljs-string">"###"</span>], <span class="hljs-comment"># Secvențe de oprire</span> | |
| ) | |
| <span class="hljs-comment"># Generează text</span> | |
| prompt = <span class="hljs-string">"Scrie o poveste creativă"</span> | |
| outputs = llm.generate(prompt, sampling_params) | |
| <span class="hljs-built_in">print</span>(outputs[<span class="hljs-number">0</span>].outputs[<span class="hljs-number">0</span>].text) | |
| <span class="hljs-comment"># Pentru interacțiuni în stil chat</span> | |
| chat_prompt = [ | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"system"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Ești un povestitor creativ."</span>}, | |
| {<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"Scrie o poveste creativă"</span>}, | |
| ] | |
| formatted_prompt = llm.get_chat_template()( | |
| chat_prompt | |
| ) <span class="hljs-comment"># Folosește șablonul de chat al modelului</span> | |
| outputs = llm.generate(formatted_prompt, sampling_params) | |
| <span class="hljs-built_in">print</span>(outputs[<span class="hljs-number">0</span>].outputs[<span class="hljs-number">0</span>].text)<!-- HTML_TAG_END --></pre></div> | |
| </hfoption></div> <h2 class="relative group"><a id="controlul-avansat-al-generării" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#controlul-avansat-al-generării"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Controlul Avansat al Generării</span></h2> <h3 class="relative group"><a id="selecția-token-urilor-și-eșantionarea" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#selecția-token-urilor-și-eșantionarea"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Selecția Token-urilor și Eșantionarea</span></h3> <p data-svelte-h="svelte-vv01ra">Procesul de generare a textului implică selectarea următorului token la fiecare pas. Acest proces de selecție poate fi controlat prin diverși parametri:</p> <ol data-svelte-h="svelte-tnbogk"><li><strong>Logit-uri Brute</strong>: Probabilitățile inițiale de ieșire pentru fiecare token</li> <li><strong>Temperatura</strong>: Controlează aleatoriul în selecție (mai mare = mai creativ)</li> <li><strong>Eșantionarea Top-p (Nucleus)</strong>: Filtrează la token-urile de top care alcătuiesc X% din masa de probabilitate</li> <li><strong>Filtrarea Top-k</strong>: Limitează selecția la k token-uri cele mai probabile</li></ol> <p data-svelte-h="svelte-1to0u51">Iată cum să configurezi acești parametri:</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"></div> <div class="language-select"><hfoption value="tgi" label="TGI"> | |
| <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->client.generate( | |
| <span class="hljs-string">"Scrie o poveste creativă"</span>, | |
| temperature=<span class="hljs-number">0.8</span>, <span class="hljs-comment"># Mai mare pentru mai multă creativitate</span> | |
| top_p=<span class="hljs-number">0.95</span>, <span class="hljs-comment"># Consideră masa de probabilitate de top 95%</span> | |
| top_k=<span class="hljs-number">50</span>, <span class="hljs-comment"># Consideră top 50 de token-uri</span> | |
| max_new_tokens=<span class="hljs-number">100</span>, <span class="hljs-comment"># Lungimea maximă</span> | |
| repetition_penalty=<span class="hljs-number">1.1</span>, <span class="hljs-comment"># Reduce repetarea</span> | |
| )<!-- HTML_TAG_END --></pre></div> | |
| </hfoption> | |
| <hfoption value="llama.cpp" label="llama.cpp"> | |
| <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Prin compatibilitatea API OpenAI</span> | |
| response = client.completions.create( | |
| model=<span class="hljs-string">"smollm2-1.7b-instruct"</span>, <span class="hljs-comment"># Numele modelului (poate fi orice string pentru serverul llama.cpp)</span> | |
| prompt=<span class="hljs-string">"Scrie o poveste creativă"</span>, | |
| temperature=<span class="hljs-number">0.8</span>, <span class="hljs-comment"># Mai mare pentru mai multă creativitate</span> | |
| top_p=<span class="hljs-number">0.95</span>, <span class="hljs-comment"># Consideră masa de probabilitate de top 95%</span> | |
| frequency_penalty=<span class="hljs-number">1.1</span>, <span class="hljs-comment"># Reduce repetarea</span> | |
| presence_penalty=<span class="hljs-number">0.1</span>, <span class="hljs-comment"># Reduce repetarea</span> | |
| max_tokens=<span class="hljs-number">100</span>, <span class="hljs-comment"># Lungimea maximă</span> | |
| ) | |
| <span class="hljs-comment"># Prin accesul direct llama-cpp-python</span> | |
| output = llm( | |
| <span class="hljs-string">"Scrie o poveste creativă"</span>, | |
| temperature=<span class="hljs-number">0.8</span>, | |
| top_p=<span class="hljs-number">0.95</span>, | |
| top_k=<span class="hljs-number">50</span>, | |
| max_tokens=<span class="hljs-number">100</span>, | |
| repeat_penalty=<span class="hljs-number">1.1</span>, | |
| )<!-- HTML_TAG_END --></pre></div> | |
| </hfoption> | |
| <hfoption value="vllm" label="vLLM"> | |
| <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->params = SamplingParams( | |
| temperature=<span class="hljs-number">0.8</span>, <span class="hljs-comment"># Mai mare pentru mai multă creativitate</span> | |
| top_p=<span class="hljs-number">0.95</span>, <span class="hljs-comment"># Consideră masa de probabilitate de top 95%</span> | |
| top_k=<span class="hljs-number">50</span>, <span class="hljs-comment"># Consideră top 50 de token-uri</span> | |
| max_tokens=<span class="hljs-number">100</span>, <span class="hljs-comment"># Lungimea maximă</span> | |
| presence_penalty=<span class="hljs-number">0.1</span>, <span class="hljs-comment"># Reduce repetarea</span> | |
| ) | |
| llm.generate(<span class="hljs-string">"Scrie o poveste creativă"</span>, sampling_params=params)<!-- HTML_TAG_END --></pre></div> | |
| </hfoption></div> <h3 class="relative group"><a id="controlul-repetării" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#controlul-repetării"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Controlul Repetării</span></h3> <p data-svelte-h="svelte-17cj06z">Ambele framework-uri oferă modalități de a preveni generarea repetitivă de text:</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"></div> <div class="language-select"><hfoption value="tgi" label="TGI"> | |
| ```python | |
| client.generate( | |
| "Scrie un text variat", | |
| repetition_penalty=1.1, # Penalizează token-urile repetate | |
| no_repeat_ngram_size=3, # Previne repetarea de 3-grame | |
| ) | |
| ``` | |
| </hfoption> | |
| <hfoption value="llama.cpp" label="llama.cpp"> | |
| <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Prin API OpenAI</span> | |
| response = client.completions.create( | |
| model=<span class="hljs-string">"smollm2-1.7b-instruct"</span>, | |
| prompt=<span class="hljs-string">"Scrie un text variat"</span>, | |
| frequency_penalty=<span class="hljs-number">1.1</span>, <span class="hljs-comment"># Penalizează token-urile frecvente</span> | |
| presence_penalty=<span class="hljs-number">0.8</span>, <span class="hljs-comment"># Penalizează token-urile deja prezente</span> | |
| ) | |
| <span class="hljs-comment"># Prin biblioteca directă</span> | |
| output = llm( | |
| <span class="hljs-string">"Scrie un text variat"</span>, | |
| repeat_penalty=<span class="hljs-number">1.1</span>, <span class="hljs-comment"># Penalizează token-urile repetate</span> | |
| frequency_penalty=<span class="hljs-number">0.5</span>, <span class="hljs-comment"># Penalitate de frecvență adițională</span> | |
| presence_penalty=<span class="hljs-number">0.5</span>, <span class="hljs-comment"># Penalitate de prezență adițională</span> | |
| )<!-- HTML_TAG_END --></pre></div> | |
| </hfoption> | |
| <hfoption value="vllm" label="vLLM"> | |
| <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->params = SamplingParams( | |
| presence_penalty=<span class="hljs-number">0.1</span>, <span class="hljs-comment"># Penalizează prezența token-urilor</span> | |
| frequency_penalty=<span class="hljs-number">0.1</span>, <span class="hljs-comment"># Penalizează frecvența token-urilor</span> | |
| )<!-- HTML_TAG_END --></pre></div> | |
| </hfoption></div> <h3 class="relative group"><a id="controlul-lungimii-și-secvențele-de-oprire" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#controlul-lungimii-și-secvențele-de-oprire"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Controlul Lungimii și Secvențele de Oprire</span></h3> <p data-svelte-h="svelte-3631at">Poți controla lungimea generării și specifica când să se oprească:</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"></div> <div class="language-select"><hfoption value="tgi" label="TGI"> | |
| ```python | |
| client.generate( | |
| "Generează un paragraf scurt", | |
| max_new_tokens=100, | |
| min_new_tokens=10, | |
| stop_sequences=["\n\n", "###"], | |
| ) | |
| ``` | |
| </hfoption> | |
| <hfoption value="llama.cpp" label="llama.cpp"> | |
| <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Prin API OpenAI</span> | |
| response = client.completions.create( | |
| model=<span class="hljs-string">"smollm2-1.7b-instruct"</span>, | |
| prompt=<span class="hljs-string">"Generează un paragraf scurt"</span>, | |
| max_tokens=<span class="hljs-number">100</span>, | |
| stop=[<span class="hljs-string">"\n\n"</span>, <span class="hljs-string">"###"</span>], | |
| ) | |
| <span class="hljs-comment"># Prin biblioteca directă</span> | |
| output = llm(<span class="hljs-string">"Generează un paragraf scurt"</span>, max_tokens=<span class="hljs-number">100</span>, stop=[<span class="hljs-string">"\n\n"</span>, <span class="hljs-string">"###"</span>])<!-- HTML_TAG_END --></pre></div> | |
| </hfoption> | |
| <hfoption value="vllm" label="vLLM"> | |
| <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->params = SamplingParams( | |
| max_tokens=<span class="hljs-number">100</span>, | |
| min_tokens=<span class="hljs-number">10</span>, | |
| stop=[<span class="hljs-string">"###"</span>, <span class="hljs-string">"\n\n"</span>], | |
| ignore_eos=<span class="hljs-literal">False</span>, | |
| skip_special_tokens=<span class="hljs-literal">True</span>, | |
| )<!-- HTML_TAG_END --></pre></div> | |
| </hfoption></div> <h2 class="relative group"><a id="gestionarea-memoriei" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#gestionarea-memoriei"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Gestionarea Memoriei</span></h2> <p data-svelte-h="svelte-m10jfl">Ambele framework-uri implementează tehnici avansate de gestionare a memoriei pentru inferența eficientă.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"></div> <div class="language-select"><hfoption value="tgi" label="TGI"> | |
| TGI folosește Flash Attention 2 și procesarea continuă în loturi: | |
| <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Implementarea Docker cu optimizarea memoriei</span> | |
| docker run --gpus all -p 8080:80 \ | |
| --shm-size 1g \ | |
| ghcr.io/huggingface/text-generation-inference:latest \ | |
| --model-id HuggingFaceTB/SmolLM2-1.7B-Instruct \ | |
| --max-batch-total-tokens 8192 \ | |
| --max-input-length 4096<!-- HTML_TAG_END --></pre></div> | |
| </hfoption> | |
| <hfoption value="llama.cpp" label="llama.cpp"> | |
| <p data-svelte-h="svelte-1sfqsd0">llama.cpp folosește cuantificarea și dispunerea optimată a memoriei:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Server cu optimizări de memorie</span> | |
| ./server \ | |
| -m smollm2-1.7b-instruct.Q4_K_M.gguf \ | |
| --host 0.0.0.0 \ | |
| --port 8080 \ | |
| -c 2048 \ <span class="hljs-comment"># Dimensiunea contextului</span> | |
| --threads 4 \ <span class="hljs-comment"># Thread-uri CPU</span> | |
| --n-gpu-layers 32 \ <span class="hljs-comment"># Folosește mai multe straturi GPU pentru modele mai mari</span> | |
| --mlock \ <span class="hljs-comment"># Blochează memoria pentru a preveni swapping-ul</span> | |
| --cont-batching <span class="hljs-comment"># Activează procesarea continuă în loturi</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1p18jlx">Pentru modele prea mari pentru GPU-ul tău, poți folosi offloading-ul CPU:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->./server \ | |
| -m smollm2-1.7b-instruct.Q4_K_M.gguf \ | |
| --n-gpu-layers 20 \ <span class="hljs-comment"># Păstrează primele 20 de straturi pe GPU</span> | |
| --threads 8 <span class="hljs-comment"># Folosește mai multe thread-uri CPU pentru straturile CPU</span><!-- HTML_TAG_END --></pre></div> | |
| </hfoption> | |
| <hfoption value="vllm" label="vLLM"> | |
| <p data-svelte-h="svelte-1p9u0qw">vLLM folosește PagedAttention pentru gestionarea optimă a memoriei:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> vllm.engine.arg_utils <span class="hljs-keyword">import</span> AsyncEngineArgs | |
| engine_args = AsyncEngineArgs( | |
| model=<span class="hljs-string">"HuggingFaceTB/SmolLM2-1.7B-Instruct"</span>, | |
| gpu_memory_utilization=<span class="hljs-number">0.85</span>, | |
| max_num_batched_tokens=<span class="hljs-number">8192</span>, | |
| block_size=<span class="hljs-number">16</span>, | |
| ) | |
| llm = LLM(engine_args=engine_args)<!-- HTML_TAG_END --></pre></div> | |
| </hfoption></div> <h2 class="relative group"><a id="resurse" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#resurse"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Resurse</span></h2> <ul data-svelte-h="svelte-1jjsuar"><li><a href="https://huggingface.co/docs/text-generation-inference" rel="nofollow">Documentația Text Generation Inference</a></li> <li><a href="https://github.com/huggingface/text-generation-inference" rel="nofollow">Depozitul GitHub TGI</a></li> <li><a href="https://vllm.readthedocs.io/" rel="nofollow">Documentația vLLM</a></li> <li><a href="https://github.com/vllm-project/vllm" rel="nofollow">Depozitul GitHub vLLM</a></li> <li><a href="https://arxiv.org/abs/2309.06180" rel="nofollow">Lucrarea PagedAttention</a></li> <li><a href="https://github.com/ggerganov/llama.cpp" rel="nofollow">Depozitul GitHub llama.cpp</a></li> <li><a href="https://github.com/abetlen/llama-cpp-python" rel="nofollow">Depozitul llama-cpp-python</a></li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/course/blob/main/chapters/rum/chapter2/8.mdx" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p> | |
| <script> | |
| { | |
| __sveltekit_1ftlxhy = { | |
| assets: "/docs/course/pr_1069/rum", | |
| base: "/docs/course/pr_1069/rum", | |
| env: {} | |
| }; | |
| const element = document.currentScript.parentElement; | |
| const data = [null,null]; | |
| Promise.all([ | |
| import("/docs/course/pr_1069/rum/_app/immutable/entry/start.1de7c3d2.js"), | |
| import("/docs/course/pr_1069/rum/_app/immutable/entry/app.1f82014c.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 42], | |
| data, | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 109 kB
- Xet hash:
- 4ad296b9beefad33bfef467c6cb312fe6e10e029f8be98a355aad60809757fe3
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.