Latest update

simon-mo · simon-mo · commit f2041aaaef7b · 2025-01-30T11:55:51.000-08:00
Signed-off-by: simon-mo &lt;simon.mo@hey.com&gt;
diff --git a/2023/06/20/vllm.html b/2023/06/20/vllm.html
@@ -210,7 +210,7 @@ <h3 id="get-started-with-vllm">Get started with vLLM</h3>
           </a>
         </p> -->
         <ul class="contact-list">
-          <li class="p-name">© 2024. vLLM Team. All rights reserved.</li>
+          <li class="p-name">© 2025. vLLM Team. All rights reserved.</li>
           <li><a href="https://github.com/vllm-project/vllm">https://github.com/vllm-project/vllm</a></li>
         </ul>
       </div>
diff --git a/2023/11/14/notes-vllm-vs-deepspeed.html b/2023/11/14/notes-vllm-vs-deepspeed.html
@@ -191,7 +191,7 @@ <h3 id="appendix-feature-comparison">Appendix: Feature Comparison</h3>
           </a>
         </p> -->
         <ul class="contact-list">
-          <li class="p-name">© 2024. vLLM Team. All rights reserved.</li>
+          <li class="p-name">© 2025. vLLM Team. All rights reserved.</li>
           <li><a href="https://github.com/vllm-project/vllm">https://github.com/vllm-project/vllm</a></li>
         </ul>
       </div>
diff --git a/2024/07/23/llama31.html b/2024/07/23/llama31.html
@@ -140,7 +140,7 @@ <h3 id="acknowledgements">Acknowledgements</h3>
           </a>
         </p> -->
         <ul class="contact-list">
-          <li class="p-name">© 2024. vLLM Team. All rights reserved.</li>
+          <li class="p-name">© 2025. vLLM Team. All rights reserved.</li>
           <li><a href="https://github.com/vllm-project/vllm">https://github.com/vllm-project/vllm</a></li>
         </ul>
       </div>
diff --git a/2024/07/25/lfai-perf.html b/2024/07/25/lfai-perf.html
@@ -143,7 +143,7 @@ <h3 id="more-resources">More Resources</h3>
           </a>
         </p> -->
         <ul class="contact-list">
-          <li class="p-name">© 2024. vLLM Team. All rights reserved.</li>
+          <li class="p-name">© 2025. vLLM Team. All rights reserved.</li>
           <li><a href="https://github.com/vllm-project/vllm">https://github.com/vllm-project/vllm</a></li>
         </ul>
       </div>
diff --git a/2024/09/05/perf-update.html b/2024/09/05/perf-update.html
@@ -284,7 +284,7 @@ <h5 id="llama-3-70b-on-4xh100">Llama 3 70B on 4xH100</h5>
           </a>
         </p> -->
         <ul class="contact-list">
-          <li class="p-name">© 2024. vLLM Team. All rights reserved.</li>
+          <li class="p-name">© 2025. vLLM Team. All rights reserved.</li>
           <li><a href="https://github.com/vllm-project/vllm">https://github.com/vllm-project/vllm</a></li>
         </ul>
       </div>
diff --git a/2024/10/17/spec-decode.html b/2024/10/17/spec-decode.html
@@ -269,7 +269,7 @@ <h2 id="conclusion-the-future-of-speculative-decoding-in-vllm">Conclusion: The F
           </a>
         </p> -->
         <ul class="contact-list">
-          <li class="p-name">© 2024. vLLM Team. All rights reserved.</li>
+          <li class="p-name">© 2025. vLLM Team. All rights reserved.</li>
           <li><a href="https://github.com/vllm-project/vllm">https://github.com/vllm-project/vllm</a></li>
         </ul>
       </div>
diff --git a/2024/10/23/vllm-serving-amd.html b/2024/10/23/vllm-serving-amd.html
@@ -420,7 +420,7 @@ <h4 id="detail-benchmark-configuration">Detail Benchmark Configuration</h4>
           </a>
         </p> -->
         <ul class="contact-list">
-          <li class="p-name">© 2024. vLLM Team. All rights reserved.</li>
+          <li class="p-name">© 2025. vLLM Team. All rights reserved.</li>
           <li><a href="https://github.com/vllm-project/vllm">https://github.com/vllm-project/vllm</a></li>
         </ul>
       </div>
diff --git a/2025/01/10/dev-experience.html b/2025/01/10/dev-experience.html
@@ -176,7 +176,7 @@ <h2 id="acknowledgments">Acknowledgments</h2>
           </a>
         </p> -->
         <ul class="contact-list">
-          <li class="p-name">© 2024. vLLM Team. All rights reserved.</li>
+          <li class="p-name">© 2025. vLLM Team. All rights reserved.</li>
           <li><a href="https://github.com/vllm-project/vllm">https://github.com/vllm-project/vllm</a></li>
         </ul>
       </div>
diff --git a/2025/01/10/vllm-2024-wrapped-2025-vision.html b/2025/01/10/vllm-2024-wrapped-2025-vision.html
@@ -272,7 +272,7 @@ <h2 id="join-the-journey">Join the Journey</h2>
           </a>
         </p> -->
         <ul class="contact-list">
-          <li class="p-name">© 2024. vLLM Team. All rights reserved.</li>
+          <li class="p-name">© 2025. vLLM Team. All rights reserved.</li>
           <li><a href="https://github.com/vllm-project/vllm">https://github.com/vllm-project/vllm</a></li>
         </ul>
       </div>
diff --git a/2025/01/14/struct-decode-intro.html b/2025/01/14/struct-decode-intro.html
@@ -288,7 +288,7 @@ <h2 id="references">References</h2>
           </a>
         </p> -->
         <ul class="contact-list">
-          <li class="p-name">© 2024. vLLM Team. All rights reserved.</li>
+          <li class="p-name">© 2025. vLLM Team. All rights reserved.</li>
           <li><a href="https://github.com/vllm-project/vllm">https://github.com/vllm-project/vllm</a></li>
         </ul>
       </div>
diff --git a/2025/01/21/stack-release.html b/2025/01/21/stack-release.html
@@ -173,7 +173,7 @@ <h2 id="conclusion">Conclusion</h2>
           </a>
         </p> -->
         <ul class="contact-list">
-          <li class="p-name">© 2024. vLLM Team. All rights reserved.</li>
+          <li class="p-name">© 2025. vLLM Team. All rights reserved.</li>
           <li><a href="https://github.com/vllm-project/vllm">https://github.com/vllm-project/vllm</a></li>
         </ul>
       </div>
diff --git a/2025/01/27/intro-to-llama-stack-with-vllm.html b/2025/01/27/intro-to-llama-stack-with-vllm.html
@@ -170,7 +170,7 @@ <h3 id="start-llama-stack-server">Start Llama Stack Server</h3>
   provider_model_id: null
 </code></pre></div></div>
 
-<p>Then we can start the LlamaStack Server with the image we built via <code class="language-plaintext highlighter-rouge">llama stack run</code>:</p>
+<p>Then we can start the Llama Stack Server with the image we built via <code class="language-plaintext highlighter-rouge">llama stack run</code>:</p>
 <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>export INFERENCE_ADDR=host.containers.internal
 export INFERENCE_PORT=8000
 export INFERENCE_MODEL=meta-llama/Llama-3.2-1B-Instruct
@@ -338,12 +338,12 @@ <h2 id="deployment-on-kubernetes">Deployment on Kubernetes</h2>
   - provider_id: vllm
     provider_type: remote::vllm
     config:
-      url: ${env.VLLM_URL}
-      max_tokens: ${env.VLLM_MAX_TOKENS:4096}
-      api_token: ${env.VLLM_API_TOKEN:fake}
+      url: http://vllm-server.default.svc.cluster.local:8000/v1
+      max_tokens: 4096
+      api_token: fake
 </code></pre></div></div>
 
-<p>Once we have defined the run configuration for Llama Stack, we can build an image with that configuration the server source code:</p>
+<p>Once we have defined the run configuration for Llama Stack, we can build an image with that configuration and the server source code:</p>
 
 <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>cat &gt;/tmp/test-vllm-llama-stack/Containerfile.llama-stack-run-k8s &lt;&lt;EOF
 FROM distribution-myenv:dev
@@ -356,7 +356,7 @@ <h2 id="deployment-on-kubernetes">Deployment on Kubernetes</h2>
 podman build -f /tmp/test-vllm-llama-stack/Containerfile.llama-stack-run-k8s -t llama-stack-run-k8s /tmp/test-vllm-llama-stack
 </code></pre></div></div>
 
-<p>We can then start the LlamaStack server by deploying a Kubernetes Pod and Service:</p>
+<p>We can then start the Llama Stack server by deploying a Kubernetes Pod and Service:</p>
 <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>cat &lt;&lt;EOF |kubectl apply -f -
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -406,7 +406,7 @@ <h2 id="deployment-on-kubernetes">Deployment on Kubernetes</h2>
 EOF
 </code></pre></div></div>
 
-<p>We can check that the LlamaStack server has started:</p>
+<p>We can check that the Llama Stack server has started:</p>
 <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>$ kubectl logs vllm-server
 ...
 INFO:     Started server process [1]
@@ -446,7 +446,7 @@ <h2 id="acknowledgement">Acknowledgement</h2>
           </a>
         </p> -->
         <ul class="contact-list">
-          <li class="p-name">© 2024. vLLM Team. All rights reserved.</li>
+          <li class="p-name">© 2025. vLLM Team. All rights reserved.</li>
           <li><a href="https://github.com/vllm-project/vllm">https://github.com/vllm-project/vllm</a></li>
         </ul>
       </div>
diff --git a/2025/01/27/v1-alpha-release.html b/2025/01/27/v1-alpha-release.html
@@ -203,7 +203,7 @@ <h1 id="limitations--future-work">Limitations &amp; Future Work</h1>
 <p>While vLLM V1 shows promising results, it is still in its alpha stage and lacks several features from V0. Here’s a clarification:</p>
 
 <p><strong>Model Support:</strong><br />
-V1 supports decoder-only Transformers like Llama, mixture-of-experts (MoE) models like Mixtral, and several VLMs such as Qwen2-VL. All quantization methods are supported. However, V1 currently does not support encoder-decoder architectures like multimodal Llama 3.2, Mamba-based models like Jamba, or embedding models. Please check out <a href="">our documentation</a> for a more detailed list of the supported models.</p>
+V1 supports decoder-only Transformers like Llama, mixture-of-experts (MoE) models like Mixtral, and several VLMs such as Qwen2-VL. All quantization methods are supported. However, V1 currently does not support encoder-decoder architectures like multimodal Llama 3.2, Mamba-based models like Jamba, or embedding models. Please check out <a href="https://docs.vllm.ai/en/latest/models/supported_models.html">our documentation</a> for a more detailed list of the supported models.</p>
 
 <p><strong>Feature Limitations:</strong><br />
 V1 currently lacks support for log probs, prompt log probs sampling parameters, pipeline parallelism, structured decoding, speculative decoding, prometheus metrics, and LoRA. We are actively working to close this feature gap and add brand-new optimizations to the V1 engine.</p>
@@ -274,7 +274,7 @@ <h1 id="acknowledgment">Acknowledgment</h1>
           </a>
         </p> -->
         <ul class="contact-list">
-          <li class="p-name">© 2024. vLLM Team. All rights reserved.</li>
+          <li class="p-name">© 2025. vLLM Team. All rights reserved.</li>
           <li><a href="https://github.com/vllm-project/vllm">https://github.com/vllm-project/vllm</a></li>
         </ul>
       </div>
diff --git a/404.html b/404.html
@@ -6,7 +6,7 @@
 <title>vLLM Blog</title>
 <meta name="generator" content="Jekyll v4.3.3" />
 <meta property="og:title" content="vLLM Blog" />
-<meta name="author" content="© 2024. vLLM Team. All rights reserved." />
+<meta name="author" content="© 2025. vLLM Team. All rights reserved." />
 <meta property="og:locale" content="en_US" />
 <link rel="canonical" href="https://blog.vllm.ai/404.html" />
 <meta property="og:url" content="https://blog.vllm.ai/404.html" />
@@ -15,7 +15,7 @@
 <meta name="twitter:card" content="summary" />
 <meta property="twitter:title" content="vLLM Blog" />
 <script type="application/ld+json">
-{"@context":"https://schema.org","@type":"WebPage","author":{"@type":"Person","name":"© 2024. vLLM Team. All rights reserved."},"headline":"vLLM Blog","url":"https://blog.vllm.ai/404.html"}</script>
+{"@context":"https://schema.org","@type":"WebPage","author":{"@type":"Person","name":"© 2025. vLLM Team. All rights reserved."},"headline":"vLLM Blog","url":"https://blog.vllm.ai/404.html"}</script>
 <!-- End Jekyll SEO tag -->
 <link rel="stylesheet" href="/assets/css/style.css"><link type="application/atom+xml" rel="alternate" href="https://blog.vllm.ai/feed.xml" title="vLLM Blog" /><script async src="https://www.googletagmanager.com/gtag/js?id=G-9C5R3JR3QS"></script>
 <script>
@@ -71,7 +71,7 @@ <h1>404</h1>
           </a>
         </p> -->
         <ul class="contact-list">
-          <li class="p-name">© 2024. vLLM Team. All rights reserved.</li>
+          <li class="p-name">© 2025. vLLM Team. All rights reserved.</li>
           <li><a href="https://github.com/vllm-project/vllm">https://github.com/vllm-project/vllm</a></li>
         </ul>
       </div>
diff --git a/feed.xml b/feed.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" ><generator uri="https://jekyllrb.com/" version="4.3.3">Jekyll</generator><link href="https://blog.vllm.ai/feed.xml" rel="self" type="application/atom+xml" /><link href="https://blog.vllm.ai/" rel="alternate" type="text/html" /><updated>2025-01-27T11:53:46-08:00</updated><id>https://blog.vllm.ai/feed.xml</id><title type="html">vLLM Blog</title><author><name>© 2024. vLLM Team. All rights reserved.</name></author><entry><title type="html">Introducing vLLM Inference Provider in Llama Stack</title><link href="https://blog.vllm.ai/2025/01/27/intro-to-llama-stack-with-vllm.html" rel="alternate" type="text/html" title="Introducing vLLM Inference Provider in Llama Stack" /><published>2025-01-27T00:00:00-08:00</published><updated>2025-01-27T00:00:00-08:00</updated><id>https://blog.vllm.ai/2025/01/27/intro-to-llama-stack-with-vllm</id><content type="html" xml:base="https://blog.vllm.ai/2025/01/27/intro-to-llama-stack-with-vllm.html"><![CDATA[<p>We are excited to announce that vLLM inference provider is now available in <a href="https://github.com/meta-llama/llama-stack">Llama Stack</a> through the collaboration between the Red Hat AI Engineering team and the Llama Stack team from Meta. This article provides an introduction to this integration and a tutorial to help you get started using it locally or deploying it in a Kubernetes cluster.</p>
+<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" ><generator uri="https://jekyllrb.com/" version="4.3.3">Jekyll</generator><link href="https://blog.vllm.ai/feed.xml" rel="self" type="application/atom+xml" /><link href="https://blog.vllm.ai/" rel="alternate" type="text/html" /><updated>2025-01-30T11:55:29-08:00</updated><id>https://blog.vllm.ai/feed.xml</id><title type="html">vLLM Blog</title><author><name>© 2025. vLLM Team. All rights reserved.</name></author><entry><title type="html">Introducing vLLM Inference Provider in Llama Stack</title><link href="https://blog.vllm.ai/2025/01/27/intro-to-llama-stack-with-vllm.html" rel="alternate" type="text/html" title="Introducing vLLM Inference Provider in Llama Stack" /><published>2025-01-27T00:00:00-08:00</published><updated>2025-01-27T00:00:00-08:00</updated><id>https://blog.vllm.ai/2025/01/27/intro-to-llama-stack-with-vllm</id><content type="html" xml:base="https://blog.vllm.ai/2025/01/27/intro-to-llama-stack-with-vllm.html"><![CDATA[<p>We are excited to announce that vLLM inference provider is now available in <a href="https://github.com/meta-llama/llama-stack">Llama Stack</a> through the collaboration between the Red Hat AI Engineering team and the Llama Stack team from Meta. This article provides an introduction to this integration and a tutorial to help you get started using it locally or deploying it in a Kubernetes cluster.</p>
 
 <h1 id="what-is-llama-stack">What is Llama Stack?</h1>
 
@@ -119,7 +119,7 @@ LLAMA_STACK_DIR=. PYTHONPATH=. python -m llama_stack.cli.llama stack build --con
   provider_model_id: null
 </code></pre></div></div>
 
-<p>Then we can start the LlamaStack Server with the image we built via <code class="language-plaintext highlighter-rouge">llama stack run</code>:</p>
+<p>Then we can start the Llama Stack Server with the image we built via <code class="language-plaintext highlighter-rouge">llama stack run</code>:</p>
 <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>export INFERENCE_ADDR=host.containers.internal
 export INFERENCE_PORT=8000
 export INFERENCE_MODEL=meta-llama/Llama-3.2-1B-Instruct
@@ -287,12 +287,12 @@ INFO:     Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)
   - provider_id: vllm
     provider_type: remote::vllm
     config:
-      url: ${env.VLLM_URL}
-      max_tokens: ${env.VLLM_MAX_TOKENS:4096}
-      api_token: ${env.VLLM_API_TOKEN:fake}
+      url: http://vllm-server.default.svc.cluster.local:8000/v1
+      max_tokens: 4096
+      api_token: fake
 </code></pre></div></div>
 
-<p>Once we have defined the run configuration for Llama Stack, we can build an image with that configuration the server source code:</p>
+<p>Once we have defined the run configuration for Llama Stack, we can build an image with that configuration and the server source code:</p>
 
 <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>cat &gt;/tmp/test-vllm-llama-stack/Containerfile.llama-stack-run-k8s &lt;&lt;EOF
 FROM distribution-myenv:dev
@@ -305,7 +305,7 @@ EOF
 podman build -f /tmp/test-vllm-llama-stack/Containerfile.llama-stack-run-k8s -t llama-stack-run-k8s /tmp/test-vllm-llama-stack
 </code></pre></div></div>
 
-<p>We can then start the LlamaStack server by deploying a Kubernetes Pod and Service:</p>
+<p>We can then start the Llama Stack server by deploying a Kubernetes Pod and Service:</p>
 <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>cat &lt;&lt;EOF |kubectl apply -f -
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -355,7 +355,7 @@ spec:
 EOF
 </code></pre></div></div>
 
-<p>We can check that the LlamaStack server has started:</p>
+<p>We can check that the Llama Stack server has started:</p>
 <div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>$ kubectl logs vllm-server
 ...
 INFO:     Started server process [1]
@@ -530,7 +530,7 @@ Stay tuned for more updates!</p>
 <p>While vLLM V1 shows promising results, it is still in its alpha stage and lacks several features from V0. Here’s a clarification:</p>
 
 <p><strong>Model Support:</strong><br />
-V1 supports decoder-only Transformers like Llama, mixture-of-experts (MoE) models like Mixtral, and several VLMs such as Qwen2-VL. All quantization methods are supported. However, V1 currently does not support encoder-decoder architectures like multimodal Llama 3.2, Mamba-based models like Jamba, or embedding models. Please check out <a href="">our documentation</a> for a more detailed list of the supported models.</p>
+V1 supports decoder-only Transformers like Llama, mixture-of-experts (MoE) models like Mixtral, and several VLMs such as Qwen2-VL. All quantization methods are supported. However, V1 currently does not support encoder-decoder architectures like multimodal Llama 3.2, Mamba-based models like Jamba, or embedding models. Please check out <a href="https://docs.vllm.ai/en/latest/models/supported_models.html">our documentation</a> for a more detailed list of the supported models.</p>
 
 <p><strong>Feature Limitations:</strong><br />
 V1 currently lacks support for log probs, prompt log probs sampling parameters, pipeline parallelism, structured decoding, speculative decoding, prometheus metrics, and LoRA. We are actively working to close this feature gap and add brand-new optimizations to the V1 engine.</p>
diff --git a/index.html b/index.html
@@ -6,7 +6,7 @@
 <title>vLLM Blog</title>
 <meta name="generator" content="Jekyll v4.3.3" />
 <meta property="og:title" content="vLLM Blog" />
-<meta name="author" content="© 2024. vLLM Team. All rights reserved." />
+<meta name="author" content="© 2025. vLLM Team. All rights reserved." />
 <meta property="og:locale" content="en_US" />
 <link rel="canonical" href="https://blog.vllm.ai/" />
 <meta property="og:url" content="https://blog.vllm.ai/" />
@@ -15,7 +15,7 @@
 <meta name="twitter:card" content="summary" />
 <meta property="twitter:title" content="vLLM Blog" />
 <script type="application/ld+json">
-{"@context":"https://schema.org","@type":"WebSite","author":{"@type":"Person","name":"© 2024. vLLM Team. All rights reserved."},"headline":"vLLM Blog","name":"vLLM Blog","url":"https://blog.vllm.ai/"}</script>
+{"@context":"https://schema.org","@type":"WebSite","author":{"@type":"Person","name":"© 2025. vLLM Team. All rights reserved."},"headline":"vLLM Blog","name":"vLLM Blog","url":"https://blog.vllm.ai/"}</script>
 <!-- End Jekyll SEO tag -->
 <link rel="stylesheet" href="/assets/css/style.css"><link type="application/atom+xml" rel="alternate" href="https://blog.vllm.ai/feed.xml" title="vLLM Blog" /><script async src="https://www.googletagmanager.com/gtag/js?id=G-9C5R3JR3QS"></script>
 <script>
@@ -136,7 +136,7 @@ <h3>
           </a>
         </p> -->
         <ul class="contact-list">
-          <li class="p-name">© 2024. vLLM Team. All rights reserved.</li>
+          <li class="p-name">© 2025. vLLM Team. All rights reserved.</li>
           <li><a href="https://github.com/vllm-project/vllm">https://github.com/vllm-project/vllm</a></li>
         </ul>
       </div>
