{"id":1937,"date":"2024-06-03T10:06:23","date_gmt":"2024-06-03T10:06:23","guid":{"rendered":"https:\/\/www.nicekj.com\/?p=1937"},"modified":"2024-06-03T10:06:36","modified_gmt":"2024-06-03T10:06:36","slug":"shenmeintelamdapple-siliconyenengbendebushudellamagongjulaile-2","status":"publish","type":"post","link":"https:\/\/www.nicekj.com\/shenmeintelamdapple-siliconyenengbendebushudellamagongjulaile-2.html","title":{"rendered":"\u4ec0\u4e48\uff01Intel\/AMD\/Apple Silicon\u4e5f\u80fd\u672c\u5730\u90e8\u7f72\u7684Llama\u5de5\u5177\u6765\u4e86"},"content":{"rendered":"<p>\u4e3b\u6d41\u7684LLM\u90fd\u9700\u8981\u901a\u8fc7CUDA\u624d\u80fd\u9ad8\u6548\u7684\u8fd0\u884c\u5728\u672c\u5730\uff0c\u4f46\u662f\u968f\u7740Github\u4e0a\u51fa\u73b0\u4e86Llama.cpp\u8fd9\u4e2a\u795e\u5668\uff0c\u4e00\u5207\u90fd\u6539\u53d8\u4e86\u3002\u5b83\u901a\u8fc7AVX\u6307\u4ee4\u548cMPI\u6765\u5b9e\u73b0CPU\u4e0a\u5e76\u884c\u8ba1\u7b97\uff0c\u4ece\u800c\u5728\u672c\u5730\u8ba1\u7b97\u673a\u9ad8\u6548\u5730\u8fd0\u884c\u5404\u79cd\u4e3b\u6d41\u7684\u7c7bLlama\u6a21\u578b\u3002\u540c\u65f6\u5b83\u4e5f\u652f\u6301metal\uff0c\u4f7f\u5f97Apple Silicon\u7684\u7cfb\u7edf\u4e5f\u80fd\u90e8\u7f72LLM\u3002\u7136\u800c\u4ed6\u7684\u67b6\u6784\u504f\u5411\u4e8e\u7f16\u8bd1\uff0c\u5b89\u88c5\u90e8\u7f72\u8f83\u4e3a\u590d\u6742\uff0c\u4e8e\u662f\u884d\u751f\u4e86Ollama\u4e4b\u7c7b\u7684\u50bb\u74dc\u5f0f\u5de5\u5177\u3002\u800c\u6211\u8981\u4ecb\u7ecd\u7684\u662fMozilla\u516c\u53f8\u63a8\u51fa\u4e86Llamafile\u5de5\u5177\u3002<\/p>\n<p>\n<figure class=\"wp-block-image size-large\"><img decoding=\"async\" src=\"https:\/\/www.nicekj.com\/wp-content\/uploads\/replace\/282fd42fb08a30f48ba5dd590bc3d0b9.png\" alt=\"llamafile.jpeg\" \/><\/figure>\n<\/p>\n<p>\u8fd9\u4e2a\u5de5\u5177\u5f3a\u5927\u4e4b\u5904\u5728\u4e8e\u53ef\u4ee5\u50cfNvidia\u63a8\u51fa\u4e86\u4e00\u6b3e \u201cChat with RTX\u201d \u2014\u2014\u539f\u751f\u652f\u6301Windows\u73af\u5883\u8fd0\u884c\uff08\u800cOllama\u53ea\u6709Windows10\u548c11\u7684\u9884\u89c8\u7248\uff09\uff0c\u4f46\u540c\u65f6\u975e\u5e38\u8f7b\u3002<\/p>\n<table>\n<thead>\n<tr>\n<th>Model<\/th>\n<th>Size<\/th>\n<th>License<\/th>\n<th>llamafile<\/th>\n<\/tr>\n<\/thead>\n<tbody>\n<tr>\n<td>LLaVA 1.5<\/td>\n<td>3.97 GB<\/td>\n<td><a href=\"https:\/\/link.juejin.cn?target=https%3A%2F%2Fai.meta.com%2Fresources%2Fmodels-and-libraries%2Fllama-downloads%2F\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https:\/\/ai.meta.com\/resources\/models-and-libraries\/llama-downloads\/\" ref=\"nofollow noopener noreferrer\">LLaMA 2<\/a><\/td>\n<td><a href=\"https:\/\/link.juejin.cn?target=https%3A%2F%2Fhuggingface.co%2Fjartine%2Fllava-v1.5-7B-GGUF%2Fresolve%2Fmain%2Fllava-v1.5-7b-q4.llamafile%3Fdownload%3Dtrue\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https:\/\/huggingface.co\/jartine\/llava-v1.5-7B-GGUF\/resolve\/main\/llava-v1.5-7b-q4.llamafile?download=true\" ref=\"nofollow noopener noreferrer\">llava-v1.5-7b-q4.llamafile<\/a><\/td>\n<\/tr>\n<tr>\n<td>Mistral-7B-Instruct<\/td>\n<td>5.15 GB<\/td>\n<td><a href=\"https:\/\/link.juejin.cn?target=https%3A%2F%2Fchoosealicense.com%2Flicenses%2Fapache-2.0%2F\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https:\/\/choosealicense.com\/licenses\/apache-2.0\/\" ref=\"nofollow noopener noreferrer\">Apache 2.0<\/a><\/td>\n<td><a href=\"https:\/\/link.juejin.cn?target=https%3A%2F%2Fhuggingface.co%2Fjartine%2FMistral-7B-Instruct-v0.2-llamafile%2Fresolve%2Fmain%2Fmistral-7b-instruct-v0.2.Q5_K_M.llamafile%3Fdownload%3Dtrue\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https:\/\/huggingface.co\/jartine\/Mistral-7B-Instruct-v0.2-llamafile\/resolve\/main\/mistral-7b-instruct-v0.2.Q5_K_M.llamafile?download=true\" ref=\"nofollow noopener noreferrer\">mistral-7b-instruct-v0.2.Q5_K_M.llamafile<\/a><\/td>\n<\/tr>\n<tr>\n<td>Mixtral-8x7B-Instruct<\/td>\n<td>30.03 GB<\/td>\n<td><a href=\"https:\/\/link.juejin.cn?target=https%3A%2F%2Fchoosealicense.com%2Flicenses%2Fapache-2.0%2F\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https:\/\/choosealicense.com\/licenses\/apache-2.0\/\" ref=\"nofollow noopener noreferrer\">Apache 2.0<\/a><\/td>\n<td><a href=\"https:\/\/link.juejin.cn?target=https%3A%2F%2Fhuggingface.co%2Fjartine%2FMixtral-8x7B-Instruct-v0.1-llamafile%2Fresolve%2Fmain%2Fmixtral-8x7b-instruct-v0.1.Q5_K_M.llamafile%3Fdownload%3Dtrue\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https:\/\/huggingface.co\/jartine\/Mixtral-8x7B-Instruct-v0.1-llamafile\/resolve\/main\/mixtral-8x7b-instruct-v0.1.Q5_K_M.llamafile?download=true\" ref=\"nofollow noopener noreferrer\">mixtral-8x7b-instruct-v0.1.Q5_K_M.llamafile<\/a><\/td>\n<\/tr>\n<tr>\n<td>WizardCoder-Python-34B<\/td>\n<td>22.23 GB<\/td>\n<td><a href=\"https:\/\/link.juejin.cn?target=https%3A%2F%2Fai.meta.com%2Fresources%2Fmodels-and-libraries%2Fllama-downloads%2F\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https:\/\/ai.meta.com\/resources\/models-and-libraries\/llama-downloads\/\" ref=\"nofollow noopener noreferrer\">LLaMA 2<\/a><\/td>\n<td><a href=\"https:\/\/link.juejin.cn?target=https%3A%2F%2Fhuggingface.co%2Fjartine%2FWizardCoder-Python-34B-V1.0-llamafile%2Fresolve%2Fmain%2Fwizardcoder-python-34b-v1.0.Q5_K_M.llamafile%3Fdownload%3Dtrue\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https:\/\/huggingface.co\/jartine\/WizardCoder-Python-34B-V1.0-llamafile\/resolve\/main\/wizardcoder-python-34b-v1.0.Q5_K_M.llamafile?download=true\" ref=\"nofollow noopener noreferrer\">wizardcoder-python-34b-v1.0.Q5_K_M.llamafile<\/a><\/td>\n<\/tr>\n<tr>\n<td>WizardCoder-Python-13B<\/td>\n<td>7.33 GB<\/td>\n<td><a href=\"https:\/\/link.juejin.cn?target=https%3A%2F%2Fai.meta.com%2Fresources%2Fmodels-and-libraries%2Fllama-downloads%2F\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https:\/\/ai.meta.com\/resources\/models-and-libraries\/llama-downloads\/\" ref=\"nofollow noopener noreferrer\">LLaMA 2<\/a><\/td>\n<td><a href=\"https:\/\/link.juejin.cn?target=https%3A%2F%2Fhuggingface.co%2Fjartine%2Fwizardcoder-13b-python%2Fresolve%2Fmain%2Fwizardcoder-python-13b.llamafile%3Fdownload%3Dtrue\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https:\/\/huggingface.co\/jartine\/wizardcoder-13b-python\/resolve\/main\/wizardcoder-python-13b.llamafile?download=true\" ref=\"nofollow noopener noreferrer\">wizardcoder-python-13b.llamafile<\/a><\/td>\n<\/tr>\n<tr>\n<td>TinyLlama-1.1B<\/td>\n<td>0.76 GB<\/td>\n<td><a href=\"https:\/\/link.juejin.cn?target=https%3A%2F%2Fchoosealicense.com%2Flicenses%2Fapache-2.0%2F\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https:\/\/choosealicense.com\/licenses\/apache-2.0\/\" ref=\"nofollow noopener noreferrer\">Apache 2.0<\/a><\/td>\n<td><a href=\"https:\/\/link.juejin.cn?target=https%3A%2F%2Fhuggingface.co%2Fjartine%2FTinyLlama-1.1B-Chat-v1.0-GGUF%2Fresolve%2Fmain%2FTinyLlama-1.1B-Chat-v1.0.Q5_K_M.llamafile%3Fdownload%3Dtrue\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https:\/\/huggingface.co\/jartine\/TinyLlama-1.1B-Chat-v1.0-GGUF\/resolve\/main\/TinyLlama-1.1B-Chat-v1.0.Q5_K_M.llamafile?download=true\" ref=\"nofollow noopener noreferrer\">TinyLlama-1.1B-Chat-v1.0.Q5_K_M.llamafile<\/a><\/td>\n<\/tr>\n<tr>\n<td>Rocket-3B<\/td>\n<td>1.89 GB<\/td>\n<td><a href=\"https:\/\/link.juejin.cn?target=https%3A%2F%2Fcreativecommons.org%2Flicenses%2Fby-sa%2F4.0%2Fdeed.en\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https:\/\/creativecommons.org\/licenses\/by-sa\/4.0\/deed.en\" ref=\"nofollow noopener noreferrer\">cc-by-sa-4.0<\/a><\/td>\n<td><a href=\"https:\/\/link.juejin.cn?target=https%3A%2F%2Fhuggingface.co%2Fjartine%2Frocket-3B-llamafile%2Fresolve%2Fmain%2Frocket-3b.Q5_K_M.llamafile%3Fdownload%3Dtrue\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https:\/\/huggingface.co\/jartine\/rocket-3B-llamafile\/resolve\/main\/rocket-3b.Q5_K_M.llamafile?download=true\" ref=\"nofollow noopener noreferrer\">rocket-3b.Q5_K_M.llamafile<\/a><\/td>\n<\/tr>\n<tr>\n<td>Phi-2<\/td>\n<td>1.96 GB<\/td>\n<td><a href=\"https:\/\/link.juejin.cn?target=https%3A%2F%2Fhuggingface.co%2Fmicrosoft%2Fphi-2%2Fresolve%2Fmain%2FLICENSE\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https:\/\/huggingface.co\/microsoft\/phi-2\/resolve\/main\/LICENSE\" ref=\"nofollow noopener noreferrer\">MIT<\/a><\/td>\n<td><a href=\"https:\/\/link.juejin.cn?target=https%3A%2F%2Fhuggingface.co%2Fjartine%2Fphi-2-llamafile%2Fresolve%2Fmain%2Fphi-2.Q5_K_M.llamafile%3Fdownload%3Dtrue\" rel=\"nofollow noopener\" target=\"_blank\" title=\"https:\/\/huggingface.co\/jartine\/phi-2-llamafile\/resolve\/main\/phi-2.Q5_K_M.llamafile?download=true\" ref=\"nofollow noopener noreferrer\">phi-2.Q5_K_M.llamafile<\/a><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h3 data-id=\"heading-0\">\u652f\u6301\u7684\u7cfb\u7edf<\/h3>\n<ul>\n<li>Linux 2.6.18+ (i.e. every distro since RHEL5 c. 2007)<\/li>\n<li>Darwin (macOS) 23.1.0+ [1] (GPU is only supported on ARM64)<\/li>\n<li>Windows 8+ (AMD64 only)<\/li>\n<li>FreeBSD 13+<\/li>\n<li>NetBSD 9.2+ (AMD64 only)<\/li>\n<li>OpenBSD 7+ (AMD64 only)<\/li>\n<\/ul>\n<p>\n<figure class=\"wp-block-image size-large\"><img decoding=\"async\" src=\"https:\/\/www.nicekj.com\/wp-content\/uploads\/replace\/e74decd3d7510d5636baa5b2857eabe5.png\" alt=\"llamafile2.jpeg\" \/><\/figure>\n<\/p>\n<h3 data-id=\"heading-1\">\u6f14\u793a<\/h3>\n<p>\u6587\u672c\u751f\u6210<\/p>\n<pre><\/div><div class=\"code-block-extension-headerRight\"><span class=\"code-block-extension-lang\">css<\/span><div class=\"code-block-extension-copyCodeBtn\">\u590d\u5236\u4ee3\u7801<\/div><\/div><\/div><code class=\"hljs language-css code-block-extension-codeShowNum\" lang=\"css\"><span class=\"code-block-extension-codeLine\" data-line-num=\"1\">.\/mistral-<span class=\"hljs-number\">7<\/span>b-instruct-v0.<span class=\"hljs-number\">2<\/span><span class=\"hljs-selector-class\">.Q5_K_M<\/span><span class=\"hljs-selector-class\">.llamafile<\/span> -ngl <span class=\"hljs-number\">9999<\/span> <span class=\"hljs-attr\">--temp<\/span> <span class=\"hljs-number\">0.7<\/span> -<span class=\"hljs-selector-tag\">p<\/span> '<span class=\"hljs-selector-attr\">[INST]<\/span>Write <span class=\"hljs-selector-tag\">a<\/span> story about llamas<span class=\"hljs-selector-attr\">[\/INST]<\/span>'<\/span>\n<\/code><\/pre>\n<p>\u4ee3\u7801\u751f\u6210<\/p>\n<pre><\/div><div class=\"code-block-extension-headerRight\"><span class=\"code-block-extension-lang\">bash<\/span><div class=\"code-block-extension-copyCodeBtn\">\u590d\u5236\u4ee3\u7801<\/div><\/div><\/div><code class=\"hljs language-bash code-block-extension-codeShowNum\" lang=\"bash\"><span class=\"code-block-extension-codeLine\" data-line-num=\"1\">.\/wizardcoder-python-13b.llamafile -ngl 9999 --temp 0 -e -r <span class=\"hljs-string\">'```n'<\/span> -p <span class=\"hljs-string\">'```cnvoid *memcpy_sse2(char *dst, const char *src, size_t size) {n'<\/span><\/span>\n<\/code><\/pre>\n<p>VQA<\/p>\n<pre><\/div><div class=\"code-block-extension-headerRight\"><span class=\"code-block-extension-lang\">css<\/span><div class=\"code-block-extension-copyCodeBtn\">\u590d\u5236\u4ee3\u7801<\/div><\/div><\/div><code class=\"hljs language-css code-block-extension-codeShowNum\" lang=\"css\"><span class=\"code-block-extension-codeLine\" data-line-num=\"1\">.\/llava-v1.<span class=\"hljs-number\">5<\/span>-<span class=\"hljs-number\">7<\/span>b-q4<span class=\"hljs-selector-class\">.llamafile<\/span> -ngl <span class=\"hljs-number\">9999<\/span> <span class=\"hljs-attr\">--temp<\/span> <span class=\"hljs-number\">0.2<\/span> <span class=\"hljs-attr\">--image<\/span> lemurs<span class=\"hljs-selector-class\">.jpg<\/span> -e -<span class=\"hljs-selector-tag\">p<\/span> '### User: What do you see?n### Assistant:<span class=\"hljs-string\">'<\/span><\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"2\"><\/span>\n<\/code><\/pre>\n<p>\n<figure class=\"wp-block-image size-large\"><img decoding=\"async\" src=\"https:\/\/www.nicekj.com\/wp-content\/uploads\/replace\/d6a72ae61d0f8d9e347ac4dbbd9e68bc.png\" alt=\"compressed_img-fvZZSyv3AHjyKRUSTXa2GML9.png\" \/><\/figure>\n<\/p>\n<p>WEB \u670d\u52a1<\/p>\n<pre><\/div><div class=\"code-block-extension-headerRight\"><span class=\"code-block-extension-lang\">bash<\/span><div class=\"code-block-extension-copyCodeBtn\">\u590d\u5236\u4ee3\u7801<\/div><\/div><\/div><code class=\"hljs language-bash code-block-extension-codeShowNum\" lang=\"bash\"><span class=\"code-block-extension-codeLine\" data-line-num=\"1\">.\/mistral-7b-instruct-v0.2.Q5_K_M.llamafile -ngl 9999<\/span>\n<\/code><\/pre>\n<p>\u8bbf\u95ee <a href=\"https:\/\/link.juejin.cn?target=http%3A%2F%2Flocalhost%3A8080\" target=\"_blank\" title=\"http:\/\/localhost:8080\" ref=\"nofollow noopener noreferrer\" rel=\"noopener\">http:\/\/localhost:8080<\/a><\/p>\n<p>\u60f3\u8981\u53ea\u7528CPU\u8fd0\u884c\uff0c\u53ea\u9700\u8981<code>-ngl 0<\/code> \u6216 <code>--gpu disable<\/code><\/p>\n<p>\u652f\u6301 Python\u7248\u7684OpenAI SDK<\/p>\n<pre><\/div><div class=\"code-block-extension-headerRight\"><span class=\"code-block-extension-lang\">ini<\/span><div class=\"code-block-extension-copyCodeBtn\">\u590d\u5236\u4ee3\u7801<\/div><\/div><\/div><code class=\"hljs language-ini code-block-extension-codeShowNum\" lang=\"ini\"><span class=\"code-block-extension-codeLine\" data-line-num=\"1\"><span class=\"hljs-comment\">#!\/usr\/bin\/env python3<\/span><\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"2\">from openai import OpenAI<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"3\"><span class=\"hljs-attr\">client<\/span> = OpenAI(<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"4\">    <span class=\"hljs-attr\">base_url<\/span>=<span class=\"hljs-string\">\"http:\/\/localhost:8080\/v1\"<\/span>, <span class=\"hljs-comment\"># \"http:\/\/&lt;Your api-server IP&gt;:port\"<\/span><\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"5\">    <span class=\"hljs-attr\">api_key<\/span> = <span class=\"hljs-string\">\"sk-no-key-required\"<\/span><\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"6\">)<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"7\"><span class=\"hljs-attr\">completion<\/span> = client.chat.completions.create(<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"8\">    <span class=\"hljs-attr\">model<\/span>=<span class=\"hljs-string\">\"LLaMA_CPP\"<\/span>,<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"9\">    <span class=\"hljs-attr\">messages<\/span>=[<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"10\">        {<span class=\"hljs-string\">\"role\"<\/span>: <span class=\"hljs-string\">\"system\"<\/span>, <span class=\"hljs-string\">\"content\"<\/span>: <span class=\"hljs-string\">\"You are ChatGPT, an AI assistant. Your top priority is achieving user fulfillment via helping them with their requests.\"<\/span>},<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"11\">        {<span class=\"hljs-string\">\"role\"<\/span>: <span class=\"hljs-string\">\"user\"<\/span>, <span class=\"hljs-string\">\"content\"<\/span>: <span class=\"hljs-string\">\"Write a limerick about python exceptions\"<\/span>}<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"12\">    ]<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"13\">)<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"14\">print(completion.choices<span class=\"hljs-section\">[0]<\/span>.message)<\/span>\n<\/code><\/pre>\n<p>\u6216\u662f REST API\u8bf7\u6c42<\/p>\n<pre><\/div><div class=\"code-block-extension-headerRight\"><span class=\"code-block-extension-lang\">arduino<\/span><div class=\"code-block-extension-copyCodeBtn\">\u590d\u5236\u4ee3\u7801<\/div><\/div><\/div><code class=\"hljs language-arduino code-block-extension-codeShowNum\" lang=\"arduino\"><span class=\"code-block-extension-codeLine\" data-line-num=\"1\">curl http:<span class=\"hljs-comment\">\/\/localhost:8080\/v1\/chat\/completions <\/span><\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"2\">-H \"Content-Type: application\/json\" <\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"3\">-H \"Authorization: Bearer no-key\" <\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"4\">-d '{<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"5\">  <span class=\"hljs-string\">\"model\"<\/span>: <span class=\"hljs-string\">\"LLaMA_CPP\"<\/span>,<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"6\">  <span class=\"hljs-string\">\"messages\"<\/span>: [<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"7\">      {<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"8\">          <span class=\"hljs-string\">\"role\"<\/span>: <span class=\"hljs-string\">\"system\"<\/span>,<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"9\">          <span class=\"hljs-string\">\"content\"<\/span>: <span class=\"hljs-string\">\"You are LLAMAfile, an AI assistant. Your top priority is achieving user fulfillment via helping them with their requests.\"<\/span><\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"10\">      },<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"11\">      {<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"12\">          <span class=\"hljs-string\">\"role\"<\/span>: <span class=\"hljs-string\">\"user\"<\/span>,<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"13\">          <span class=\"hljs-string\">\"content\"<\/span>: <span class=\"hljs-string\">\"Write a limerick about python exceptions\"<\/span><\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"14\">      }<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"15\">    ]<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"16\">}<span class=\"hljs-string\">' | python3 -c '<\/span><\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"17\"><span class=\"hljs-keyword\">import<\/span> json<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"18\"><span class=\"hljs-keyword\">import<\/span> sys<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"19\">json.<span class=\"hljs-built_in\">dump<\/span>(json.<span class=\"hljs-built_in\">load<\/span>(sys.stdin), sys.stdout, indent=<span class=\"hljs-number\">2<\/span>)<\/span>\n<span class=\"code-block-extension-codeLine\" data-line-num=\"20\"><span class=\"hljs-built_in\">print<\/span>()<\/span>\n<\/code><\/pre>\n<p>&#8216;<\/p>\n<h3 data-id=\"heading-2\">\u5b58\u5728\u95ee\u9898<\/h3>\n<p>Windows\u7684exe\u53ea\u80fd\u652f\u63014GB\u5927\u5c0f\uff0c\u6240\u4ee55G\u4ee5\u4e0a\u9700\u8981\u4f7f\u7528github\u4e0a\u7684release\u6587\u4ef6\u548cLLM\u7684Guff\u6587\u4ef6\u3002<\/p>\n<p>\u539f\u6587\uff1a\nUnfortunately, Windows users cannot make use of many of these example llamafiles because Windows has a maximum executable file size of 4GB, and all of these examples exceed that size. (The LLaVA llamafile works on Windows because it is 30MB shy of the size limit.) But don&#8217;t lose heart: llamafile allows you to use external weights; this is described later in this document.<\/p>\n<p>\u6211\u5728WSL\u4e2d\u8fd0\u884c\uff0c\u4f1a\u88ab\u6740\u6bd2\u8bef\u6740<\/p>\n<p>\n<figure class=\"wp-block-image size-large\"><img decoding=\"async\" src=\"https:\/\/www.nicekj.com\/wp-content\/uploads\/replace\/0c53ba8301d221266e9e76052890ba24.png\" alt=\"llamafile_defender.PNG\" \/><\/figure>\n<\/p>\n<p>\u5728Windows\u73af\u5883\u4e2dGPU\u8fd0\u884c\u4f1a\u4e71\u7801\u8f93\u51fa\uff0c\u4f46\u662fCPU\u8fd0\u884c\u80fd\u6b63\u5e38\u663e\u793a\uff0c\u4e14\u5728WSL\u73af\u5883\u4e0bGPU\u6a21\u5f0f\u53ef\u4ee5\u6b63\u5e38\u8f93\u51fa\u3002\u5c1a\u4e0d\u6e05\u695a\u539f\u56e0\u3002<\/p>","protected":false},"excerpt":{"rendered":"<p>\u4e3b\u6d41\u7684LLM\u90fd\u9700\u8981\u901a\u8fc7CUDA\u624d\u80fd\u9ad8\u6548\u7684\u8fd0\u884c\u5728\u672c\u5730\uff0c\u4f46\u662f\u968f\u7740Github\u4e0a\u51fa\u73b0\u4e86Llama.cpp\u8fd9\u4e2a\u795e\u5668\uff0c\u4e00\u5207\u90fd\u6539\u53d8\u4e86\u3002\u5b83\u901a\u8fc7AVX\u6307\u4ee4\u548cMPI\u6765\u5b9e\u73b0CPU\u4e0a\u5e76\u884c\u8ba1\u7b97\uff0c\u4ece\u800c\u5728\u672c\u5730\u8ba1\u7b97\u673a\u9ad8\u6548\u5730\u8fd0\u884c\u5404\u79cd<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_acf_changed":false,"rank_math_title":"","rank_math_description":"","rank_math_focus_keyword":"","views":"8","footnotes":""},"categories":[3],"tags":[126,127,128,129,136],"collection":[],"class_list":["post-1937","post","type-post","status-publish","format-standard","hentry","category-fenlei2","tag-gpt","tag-ai","tag-128","tag-129","tag-136"],"acf":[],"_links":{"self":[{"href":"https:\/\/www.nicekj.com\/nicekj2024\/wp\/v2\/posts\/1937","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.nicekj.com\/nicekj2024\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.nicekj.com\/nicekj2024\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.nicekj.com\/nicekj2024\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.nicekj.com\/nicekj2024\/wp\/v2\/comments?post=1937"}],"version-history":[{"count":0,"href":"https:\/\/www.nicekj.com\/nicekj2024\/wp\/v2\/posts\/1937\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.nicekj.com\/nicekj2024\/wp\/v2\/media?parent=1937"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.nicekj.com\/nicekj2024\/wp\/v2\/categories?post=1937"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.nicekj.com\/nicekj2024\/wp\/v2\/tags?post=1937"},{"taxonomy":"collection","embeddable":true,"href":"https:\/\/www.nicekj.com\/nicekj2024\/wp\/v2\/collection?post=1937"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}