From 1d4a951383c18693e47c2942be14c52d96f0738b Mon Sep 17 00:00:00 2001 From: Syeda Anjum Date: Fri, 14 Aug 2026 22:31:17 +0000 Subject: [PATCH 1/8] Fix typo in chat-template path for gemma-4 models, switch benchmark to synthetic, and add qwix TPU configuration --- .../templates/configmap-benchmark.tpl.yaml | 4 +- .../patch-vllm-args.yaml | 2 +- .../h100-gemma-4-31b-it/patch-vllm-args.yaml | 2 +- .../patch-vllm-args.yaml | 2 +- .../l4-gemma-4-31b-it/patch-vllm-args.yaml | 2 +- .../l4-gemma-4-e2b-it/patch-vllm-args.yaml | 2 +- .../l4-gemma-4-e4b-it/patch-vllm-args.yaml | 2 +- .../patch-vllm-args.yaml | 2 +- .../patch-vllm-args.yaml | 2 +- .../patch-vllm-args.yaml | 2 +- .../kustomization.yaml | 145 ++++++++++++++++++ .../v6e-gemma-4-31b-it-qwix/patch-env.yaml | 24 +++ .../patch-nodeselector.yaml | 24 +++ .../patch-resources.yaml | 29 ++++ .../patch-vllm-args.yaml | 38 +++++ .../vllm/v6e-gemma-4-31b-it-qwix/runtime.env | 6 + .../v6e-gemma-4-31b-it/patch-vllm-args.yaml | 2 +- 17 files changed, 278 insertions(+), 12 deletions(-) create mode 100644 platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/kustomization.yaml create mode 100644 platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-env.yaml create mode 100644 platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-nodeselector.yaml create mode 100644 platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-resources.yaml create mode 100644 platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-vllm-args.yaml create mode 100644 platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/runtime.env diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/inference-perf-bench/vllm/templates/configmap-benchmark.tpl.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/inference-perf-bench/vllm/templates/configmap-benchmark.tpl.yaml index f2990c1f7..80a25c0a8 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/inference-perf-bench/vllm/templates/configmap-benchmark.tpl.yaml +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/inference-perf-bench/vllm/templates/configmap-benchmark.tpl.yaml @@ -31,7 +31,7 @@ data: worker_max_concurrency: 15 worker_max_tcp_connections: 2500 api: - type: completion + type: chat streaming: true server: type: vllm @@ -41,7 +41,7 @@ data: tokenizer: pretrained_model_name_or_path: ${TOKENIZER_ID} data: - type: shareGPT + type: synthetic metrics: type: prometheus prometheus: diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/h100-gemma-4-26b-a4b-it/patch-vllm-args.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/h100-gemma-4-26b-a4b-it/patch-vllm-args.yaml index 86fda830a..c418722aa 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/h100-gemma-4-26b-a4b-it/patch-vllm-args.yaml +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/h100-gemma-4-26b-a4b-it/patch-vllm-args.yaml @@ -14,4 +14,4 @@ - op: add path: /spec/template/spec/containers/1/args/- - value: "--chat-template=/gcs/google/gemma-4-26b-a4b/chat_template.jinja" + value: "--chat-template=/gcs/google/gemma-4-26b-a4b-it/chat_template.jinja" diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/h100-gemma-4-31b-it/patch-vllm-args.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/h100-gemma-4-31b-it/patch-vllm-args.yaml index ea89e9f93..24b056e07 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/h100-gemma-4-31b-it/patch-vllm-args.yaml +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/h100-gemma-4-31b-it/patch-vllm-args.yaml @@ -14,4 +14,4 @@ - op: add path: /spec/template/spec/containers/1/args/- - value: "--chat-template=/gcs/google/gemma-4-31b/chat_template.jinja" + value: "--chat-template=/gcs/google/gemma-4-31b-it/chat_template.jinja" diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/l4-gemma-4-26b-a4b-it/patch-vllm-args.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/l4-gemma-4-26b-a4b-it/patch-vllm-args.yaml index 86fda830a..c418722aa 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/l4-gemma-4-26b-a4b-it/patch-vllm-args.yaml +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/l4-gemma-4-26b-a4b-it/patch-vllm-args.yaml @@ -14,4 +14,4 @@ - op: add path: /spec/template/spec/containers/1/args/- - value: "--chat-template=/gcs/google/gemma-4-26b-a4b/chat_template.jinja" + value: "--chat-template=/gcs/google/gemma-4-26b-a4b-it/chat_template.jinja" diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/l4-gemma-4-31b-it/patch-vllm-args.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/l4-gemma-4-31b-it/patch-vllm-args.yaml index ea89e9f93..24b056e07 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/l4-gemma-4-31b-it/patch-vllm-args.yaml +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/l4-gemma-4-31b-it/patch-vllm-args.yaml @@ -14,4 +14,4 @@ - op: add path: /spec/template/spec/containers/1/args/- - value: "--chat-template=/gcs/google/gemma-4-31b/chat_template.jinja" + value: "--chat-template=/gcs/google/gemma-4-31b-it/chat_template.jinja" diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/l4-gemma-4-e2b-it/patch-vllm-args.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/l4-gemma-4-e2b-it/patch-vllm-args.yaml index 188c7401c..e79d26601 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/l4-gemma-4-e2b-it/patch-vllm-args.yaml +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/l4-gemma-4-e2b-it/patch-vllm-args.yaml @@ -14,4 +14,4 @@ - op: add path: /spec/template/spec/containers/1/args/- - value: "--chat-template=/gcs/google/gemma-4-e2b/chat_template.jinja" + value: "--chat-template=/gcs/google/gemma-4-e2b-it/chat_template.jinja" diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/l4-gemma-4-e4b-it/patch-vllm-args.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/l4-gemma-4-e4b-it/patch-vllm-args.yaml index 2aee9f849..96d8c9796 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/l4-gemma-4-e4b-it/patch-vllm-args.yaml +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/l4-gemma-4-e4b-it/patch-vllm-args.yaml @@ -14,4 +14,4 @@ - op: add path: /spec/template/spec/containers/1/args/- - value: "--chat-template=/gcs/google/gemma-4-e4b/chat_template.jinja" + value: "--chat-template=/gcs/google/gemma-4-e4b-it/chat_template.jinja" diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/rtx-pro-6000-gemma-4-26b-a4b-it/patch-vllm-args.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/rtx-pro-6000-gemma-4-26b-a4b-it/patch-vllm-args.yaml index 86fda830a..c418722aa 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/rtx-pro-6000-gemma-4-26b-a4b-it/patch-vllm-args.yaml +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/rtx-pro-6000-gemma-4-26b-a4b-it/patch-vllm-args.yaml @@ -14,4 +14,4 @@ - op: add path: /spec/template/spec/containers/1/args/- - value: "--chat-template=/gcs/google/gemma-4-26b-a4b/chat_template.jinja" + value: "--chat-template=/gcs/google/gemma-4-26b-a4b-it/chat_template.jinja" diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/rtx-pro-6000-gemma-4-31b-it/patch-vllm-args.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/rtx-pro-6000-gemma-4-31b-it/patch-vllm-args.yaml index ea89e9f93..24b056e07 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/rtx-pro-6000-gemma-4-31b-it/patch-vllm-args.yaml +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-gpu/vllm/rtx-pro-6000-gemma-4-31b-it/patch-vllm-args.yaml @@ -14,4 +14,4 @@ - op: add path: /spec/template/spec/containers/1/args/- - value: "--chat-template=/gcs/google/gemma-4-31b/chat_template.jinja" + value: "--chat-template=/gcs/google/gemma-4-31b-it/chat_template.jinja" diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-26b-a4b-it/patch-vllm-args.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-26b-a4b-it/patch-vllm-args.yaml index 835df7aec..c67894753 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-26b-a4b-it/patch-vllm-args.yaml +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-26b-a4b-it/patch-vllm-args.yaml @@ -14,4 +14,4 @@ --- - op: add path: /spec/template/spec/containers/1/args/- - value: "--chat-template=/gcs/google/gemma-4-26b-a4b/chat_template.jinja" + value: "--chat-template=/gcs/google/gemma-4-26b-a4b-it/chat_template.jinja" diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/kustomization.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/kustomization.yaml new file mode 100644 index 000000000..d4730122b --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/kustomization.yaml @@ -0,0 +1,145 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +configMapGenerator: + - envs: + - runtime.env + name: runtime + namespace: replaced-by-kustomize + +images: + - name: replaced-by-kustomize + newName: vllm/vllm-tpu + newTag: v0.26.0 + +nameSuffix: -v6e-gemma-4-31b-qwix + +patches: + - path: patch-vllm-args.yaml + target: + kind: Deployment + name: vllm + - path: patch-env.yaml + target: + kind: Deployment + name: vllm + - path: patch-nodeselector.yaml + - path: patch-resources.yaml + + +replacements: + - source: + fieldPath: data.APP_LABEL + kind: ConfigMap + name: runtime + targets: + - fieldPaths: + - spec.selector.matchLabels.app + - spec.template.metadata.labels.app + select: + kind: Deployment + - fieldPaths: + - spec.selector.app + select: + kind: Service + # - source: + # fieldPath: data.CONTAINER_IMAGE_URL + # kind: ConfigMap + # name: vllm + # targets: + # - fieldPaths: + # - spec.template.spec.containers.[name=inference-server].image + # select: + # kind: Deployment + - source: + fieldPath: data.INFERENCE_KUBERNETES_NAMESPACE + kind: ConfigMap + name: deployment + targets: + - fieldPaths: + - metadata.namespace + select: + kind: ConfigMap + - fieldPaths: + - metadata.namespace + select: + kind: Deployment + - fieldPaths: + - metadata.namespace + select: + kind: Service + - fieldPaths: + - metadata.namespace + select: + kind: ServiceAccount + - source: + fieldPath: data.INFERENCE_KUBERNETES_SERVICE_ACCOUNT + kind: ConfigMap + name: deployment + targets: + - fieldPaths: + - spec.template.spec.serviceAccountName + select: + kind: Deployment + - fieldPaths: + - metadata.name + select: + kind: ServiceAccount + - source: + fieldPath: data.MODEL_BUCKET_NAME + kind: ConfigMap + name: deployment + targets: + - fieldPaths: + - spec.template.spec.volumes.[name=huggingface-hub-model-bucket].csi.volumeAttributes.bucketName + options: + delimiter: . + index: 0 + select: + kind: Deployment + - source: + fieldPath: data.MODEL_ID + kind: ConfigMap + name: runtime + targets: + - fieldPaths: + - spec.template.spec.volumes.[name=huggingface-hub-model-bucket].csi.volumeAttributes.mountOptions + options: + delimiter: "only-dir:" + index: 1 + select: + kind: Deployment + - fieldPaths: + - spec.template.spec.containers.[name=fetch-safetensors].volumeMounts.[name=huggingface-hub-model-bucket].mountPath + - spec.template.spec.containers.[name=inference-server].volumeMounts.[name=huggingface-hub-model-bucket].mountPath + options: + delimiter: / + index: 2 + select: + kind: Deployment + - source: + fieldPath: data.MODEL_NAME + kind: ConfigMap + name: runtime + targets: + - fieldPaths: + - spec.template.metadata.labels.[ai.gke.io/model] + select: + kind: Deployment + +resources: + - ../base diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-env.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-env.yaml new file mode 100644 index 000000000..86fd8da1e --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-env.yaml @@ -0,0 +1,24 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- op: add + path: /spec/template/spec/containers/1/env/- + value: + name: MODEL_IMPL_TYPE + value: "flax_nnx" +- op: add + path: /spec/template/spec/containers/1/env/- + value: + name: USE_BATCHED_RPA_KERNEL + value: "1" diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-nodeselector.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-nodeselector.yaml new file mode 100644 index 000000000..f984468b3 --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-nodeselector.yaml @@ -0,0 +1,24 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vllm + namespace: replaced-by-kustomize +spec: + template: + spec: + nodeSelector: + cloud.google.com/compute-class: tpu-v6e-2x4 diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-resources.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-resources.yaml new file mode 100644 index 000000000..a2f2513e0 --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-resources.yaml @@ -0,0 +1,29 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vllm + namespace: replaced-by-kustomize +spec: + template: + spec: + containers: + - name: inference-server + resources: + limits: + google.com/tpu: "8" + requests: + google.com/tpu: "8" diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-vllm-args.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-vllm-args.yaml new file mode 100644 index 000000000..af71ffa57 --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-vllm-args.yaml @@ -0,0 +1,38 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- op: add + path: /spec/template/spec/containers/1/args/- + value: "--chat-template=/gcs/google/gemma-4-31b-it/chat_template.jinja" +- op: add + path: /spec/template/spec/containers/1/args/- + value: "--data-parallel-size=2" +- op: add + path: /spec/template/spec/containers/1/args/- + value: "--block-size=256" +- op: add + path: /spec/template/spec/containers/1/args/- + value: "--kv-cache-dtype=fp8" +- op: add + path: /spec/template/spec/containers/1/args/- + value: "--limit-mm-per-prompt={\"image\": 0, \"video\": 0, \"audio\": 0}" +- op: add + path: /spec/template/spec/containers/1/args/- + value: "--mm-processor-cache-gb=0" +- op: add + path: /spec/template/spec/containers/1/args/- + value: '--additional_config={"quantization": {"qwix": {"rules": [{"module_path": ".*", "weight_qtype": "float8_e4m3fn", "act_qtype": "float8_e4m3fn"}]}}}' +- op: add + path: /spec/template/spec/containers/1/args/- + value: "--load-format=runai_streamer" diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/runtime.env b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/runtime.env new file mode 100644 index 000000000..3af6bbfca --- /dev/null +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/runtime.env @@ -0,0 +1,6 @@ +APP_LABEL=vllm-v6e-gemma-4-31b-it-qwix +GPU_MEMORY_UTILIZATION=0.90 +MAX_MODEL_LEN=16384 +MODEL_ID=google/gemma-4-31b-it +MODEL_NAME=gemma-4-31b-it-qwix +TENSOR_PARALLEL_SIZE=4 diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it/patch-vllm-args.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it/patch-vllm-args.yaml index ea89e9f93..24b056e07 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it/patch-vllm-args.yaml +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it/patch-vllm-args.yaml @@ -14,4 +14,4 @@ - op: add path: /spec/template/spec/containers/1/args/- - value: "--chat-template=/gcs/google/gemma-4-31b/chat_template.jinja" + value: "--chat-template=/gcs/google/gemma-4-31b-it/chat_template.jinja" From 6a23bab7e7475f9674fdac48b9d0815a7755c88d Mon Sep 17 00:00:00 2001 From: Syeda Anjum Date: Fri, 14 Aug 2026 22:33:07 +0000 Subject: [PATCH 2/8] Update cspell dictionary with qtype and qwix --- .github/workflows/dictionary/accelerated-platforms.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/dictionary/accelerated-platforms.txt b/.github/workflows/dictionary/accelerated-platforms.txt index d18afda30..3cdfc980e 100644 --- a/.github/workflows/dictionary/accelerated-platforms.txt +++ b/.github/workflows/dictionary/accelerated-platforms.txt @@ -58,7 +58,9 @@ peft prereqs pretrained psutil +qtype qwiklabs +qwix rayutil rtxpro rueth From 74f96a1b109172222cb9725e1f624a93601af61c Mon Sep 17 00:00:00 2001 From: Syeda Anjum Date: Mon, 17 Aug 2026 16:58:09 +0000 Subject: [PATCH 3/8] Enable vLLM batched RPA kernel for Gemma-4 Trillium deployment and fix inference-perf prometheus bug --- .../vllm/templates/configmap-benchmark.tpl.yaml | 15 +++------------ .../vllm/v6e-gemma-4-31b-it-qwix/runtime.env | 2 ++ 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/inference-perf-bench/vllm/templates/configmap-benchmark.tpl.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/inference-perf-bench/vllm/templates/configmap-benchmark.tpl.yaml index 80a25c0a8..4b28e2fee 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/inference-perf-bench/vllm/templates/configmap-benchmark.tpl.yaml +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/inference-perf-bench/vllm/templates/configmap-benchmark.tpl.yaml @@ -25,8 +25,8 @@ data: sweep: type: linear timeout: 250 - num_stages: 7 - stage_duration: 30 + num_stages: 4 + stage_duration: 15 num_workers: 20 worker_max_concurrency: 15 worker_max_tcp_connections: 2500 @@ -41,21 +41,12 @@ data: tokenizer: pretrained_model_name_or_path: ${TOKENIZER_ID} data: - type: synthetic - metrics: - type: prometheus - prometheus: - scrape_interval: 15 - google_managed: true # Whether using Google Managed Prometheus - filters: [] + type: mock report: request_lifecycle: summary: true per_stage: true per_request: true - prometheus: - summary: true - per_stage: true storage: google_cloud_storage: bucket_name: ${hub_models_bucket_bench_results_name} # Required GCS bucket diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/runtime.env b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/runtime.env index 3af6bbfca..93943f824 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/runtime.env +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/runtime.env @@ -4,3 +4,5 @@ MAX_MODEL_LEN=16384 MODEL_ID=google/gemma-4-31b-it MODEL_NAME=gemma-4-31b-it-qwix TENSOR_PARALLEL_SIZE=4 +MODEL_IMPL_TYPE=flax_nnx +USE_BATCHED_RPA_KERNEL=1 From ec6cd9e5baf6129e1020c042e287ed36c7124b23 Mon Sep 17 00:00:00 2001 From: Syeda Anjum Date: Tue, 18 Aug 2026 04:06:48 +0000 Subject: [PATCH 4/8] Remove commented-out CONTAINER_IMAGE_URL replacement block in v6e-gemma-4-31b-it-qwix kustomization --- .../vllm/v6e-gemma-4-31b-it-qwix/kustomization.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/kustomization.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/kustomization.yaml index d4730122b..fe49249cf 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/kustomization.yaml +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/kustomization.yaml @@ -56,15 +56,6 @@ replacements: - spec.selector.app select: kind: Service - # - source: - # fieldPath: data.CONTAINER_IMAGE_URL - # kind: ConfigMap - # name: vllm - # targets: - # - fieldPaths: - # - spec.template.spec.containers.[name=inference-server].image - # select: - # kind: Deployment - source: fieldPath: data.INFERENCE_KUBERNETES_NAMESPACE kind: ConfigMap From f6691b74685511be24de8e3a07b3eef20d9fc30f Mon Sep 17 00:00:00 2001 From: Syeda Anjum Date: Wed, 19 Aug 2026 20:31:01 +0000 Subject: [PATCH 5/8] revert to previous benchmarking template --- .../templates/configmap-benchmark.tpl.yaml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/inference-perf-bench/vllm/templates/configmap-benchmark.tpl.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/inference-perf-bench/vllm/templates/configmap-benchmark.tpl.yaml index 4b28e2fee..c7274cc87 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/inference-perf-bench/vllm/templates/configmap-benchmark.tpl.yaml +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/inference-perf-bench/vllm/templates/configmap-benchmark.tpl.yaml @@ -25,13 +25,13 @@ data: sweep: type: linear timeout: 250 - num_stages: 4 - stage_duration: 15 + num_stages: 7 + stage_duration: 30 num_workers: 20 worker_max_concurrency: 15 worker_max_tcp_connections: 2500 api: - type: chat + type: completion streaming: true server: type: vllm @@ -41,14 +41,23 @@ data: tokenizer: pretrained_model_name_or_path: ${TOKENIZER_ID} data: - type: mock + type: shareGPT + metrics: + type: prometheus + prometheus: + scrape_interval: 15 + google_managed: true # Whether using Google Managed Prometheus + filters: [] report: request_lifecycle: summary: true per_stage: true per_request: true + prometheus: + summary: true + per_stage: true storage: google_cloud_storage: bucket_name: ${hub_models_bucket_bench_results_name} # Required GCS bucket report_file_prefix: null # Optional filename prefix - + \ No newline at end of file From 518f0577dbc63c87f64898f818b27a81cae90d7b Mon Sep 17 00:00:00 2001 From: Syeda Anjum Date: Thu, 20 Aug 2026 17:21:49 +0000 Subject: [PATCH 6/8] fix: resolve final new line and trailing whitespace lint errors in configmap-benchmark.tpl.yaml --- .../vllm/templates/configmap-benchmark.tpl.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/inference-perf-bench/vllm/templates/configmap-benchmark.tpl.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/inference-perf-bench/vllm/templates/configmap-benchmark.tpl.yaml index c7274cc87..a3120deb4 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/inference-perf-bench/vllm/templates/configmap-benchmark.tpl.yaml +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/inference-perf-bench/vllm/templates/configmap-benchmark.tpl.yaml @@ -30,7 +30,7 @@ data: num_workers: 20 worker_max_concurrency: 15 worker_max_tcp_connections: 2500 - api: + api: type: completion streaming: true server: @@ -47,7 +47,7 @@ data: prometheus: scrape_interval: 15 google_managed: true # Whether using Google Managed Prometheus - filters: [] + filters: [] report: request_lifecycle: summary: true @@ -57,7 +57,6 @@ data: summary: true per_stage: true storage: - google_cloud_storage: + google_cloud_storage: bucket_name: ${hub_models_bucket_bench_results_name} # Required GCS bucket report_file_prefix: null # Optional filename prefix - \ No newline at end of file From 947d015ba5cfde3352960a7eff4563626a86b4a9 Mon Sep 17 00:00:00 2001 From: Syeda Anjum Date: Thu, 20 Aug 2026 17:36:25 +0000 Subject: [PATCH 7/8] fix: parameterize data-parallel-size in qwix tpu template --- .../vllm/v6e-gemma-4-31b-it-qwix/patch-vllm-args.yaml | 2 +- .../vllm/v6e-gemma-4-31b-it-qwix/runtime.env | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-vllm-args.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-vllm-args.yaml index af71ffa57..3d815f6b7 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-vllm-args.yaml +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-vllm-args.yaml @@ -17,7 +17,7 @@ value: "--chat-template=/gcs/google/gemma-4-31b-it/chat_template.jinja" - op: add path: /spec/template/spec/containers/1/args/- - value: "--data-parallel-size=2" + value: "--data-parallel-size=$(DATA_PARALLEL_SIZE)" - op: add path: /spec/template/spec/containers/1/args/- value: "--block-size=256" diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/runtime.env b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/runtime.env index 93943f824..779fb23bc 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/runtime.env +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/runtime.env @@ -6,3 +6,4 @@ MODEL_NAME=gemma-4-31b-it-qwix TENSOR_PARALLEL_SIZE=4 MODEL_IMPL_TYPE=flax_nnx USE_BATCHED_RPA_KERNEL=1 +DATA_PARALLEL_SIZE=2 From 118bd185afb4924821eb9188009b582e625f219c Mon Sep 17 00:00:00 2001 From: Syeda Anjum Date: Thu, 20 Aug 2026 17:40:52 +0000 Subject: [PATCH 8/8] fix: map DATA_PARALLEL_SIZE to container env in qwix template --- .../vllm/v6e-gemma-4-31b-it-qwix/patch-env.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-env.yaml b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-env.yaml index 86fd8da1e..32eb2b580 100644 --- a/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-env.yaml +++ b/platforms/gke/base/use-cases/inference-ref-arch/kubernetes-manifests/online-inference-tpu/vllm/v6e-gemma-4-31b-it-qwix/patch-env.yaml @@ -22,3 +22,11 @@ value: name: USE_BATCHED_RPA_KERNEL value: "1" +- op: add + path: /spec/template/spec/containers/1/env/- + value: + name: DATA_PARALLEL_SIZE + valueFrom: + configMapKeyRef: + name: runtime + key: DATA_PARALLEL_SIZE