make eval script also handle performance measurement

Summary:

1. refactors the eval script to also handle performance measurement in
   vllm
2. adds a simple `vllm bench latency` script to bench in vllm

The script is broken on every single recipe, we'll have to fix and
enable things in future PRs, will update the performance tables
afterwards.

Also, add convenience flags to skip model creation, lm_eval, vllm as
needed to enable running just a single model + single step.

Test Plan:

```
SKIP_MODEL_CREATE=1 SKIP_LM_EVAL=1 SKIP_VLLM=0 with-proxy ./benchmarks/quantization/measure_accuracy_and_performance.sh h100
```

Reviewers:

Subscribers:

Tasks:

Tags:
ghstack-source-id: 15f7481
ghstack-comment-id: 3634216524
Pull-Request: #3473

Author: vkuzo

benchmarks/quantization/eval_accuracy_for_readme.sh

@@ -0,0 +1,159 @@
+#!/bin/bash
+
+# measure_accuracy_and_performance.sh - Evaluate quantization recipe accuracy
+#
+# Usage: ./measure_accuracy_and_performance.sh [TAG_OR_RECIPE] [MODEL_ID] [LOG_FILE]
+#
+# Arguments:
+#   TAG_OR_RECIPE  (optional) Tag group, single recipe name, or "all" (default: all)
+#                  Valid tags: all, h100
+#                  Valid recipes: None, float8_rowwise,
+#                                 int4_groupwise_weight_float8_rowwise_activation,
+#                                 int4_groupwise_hqq_weight_only,
+#                                 int8_rowwise_weight_only, int8_rowwise
+#   MODEL_ID       (optional) HuggingFace model ID (default: meta-llama/Llama-3.1-8B)
+#   LOG_FILE       (optional) Output log file path (default: benchmarks/data/measure_accuracy_and_performance_log.txt)
+#
+# Environment Variables:
+#   SKIP_MODEL_CREATE  If set to 1, skip creating quantized models (assumes models already exist)
+#   SKIP_LM_EVAL       If set to 1, skip running lm_eval (only creates quantized models)
+#   SKIP_VLLM          If set to 1, skip running vllm performance benchmarking
+#
+# Examples:
+#   ./measure_accuracy_and_performance.sh                    # Run all recipes with default model
+#   ./measure_accuracy_and_performance.sh h100               # Run H100-compatible recipes only
+#   ./measure_accuracy_and_performance.sh float8_rowwise     # Run single recipe
+#   ./measure_accuracy_and_performance.sh h100 meta-llama/Llama-3.2-8B  # Custom model with H100 recipes
+#   ./measure_accuracy_and_performance.sh int8_rowwise meta-llama/Llama-3.2-8B my_log.txt  # All custom args
+#   SKIP_MODEL_CREATE=1 ./measure_accuracy_and_performance.sh h100  # Skip model creation, only run eval
+#   SKIP_LM_EVAL=1 ./measure_accuracy_and_performance.sh h100  # Skip lm_eval, only create models
+#   SKIP_VLLM=1 ./measure_accuracy_and_performance.sh h100  # Skip vllm benchmarking
+
+set -e
+
+# Define all available quantization recipes
+QUANT_RECIPES_ALL=(
+  # no quantization (baseline)
+  "None"
+  "float8_rowwise"
+  "int4_groupwise_weight_float8_rowwise_activation"
+  # note: below only works on A100
+  "int4_groupwise_hqq_weight_only"
+  "int8_rowwise_weight_only"
+  "int8_rowwise"
+)
+
+# Define H100-compatible recipes (excludes A100-only recipes)
+# TODO(future PR): add `int4_groupwise_weight_float8_rowwise_activation` here,
+#   need to fix https://gist.github.com/vkuzo/6b128681b628744d445c553cdeac8a85
+QUANT_RECIPES_H100=(
+  "None"
+  "float8_rowwise"
+  "int8_rowwise_weight_only"
+  "int8_rowwise"
+)
+
+# Define recipes that are known to be broken in vllm
+VLLM_BROKEN_RECIPES=(
+  # TODO(future PR): fix this
+  # current stack trace: https://gist.github.com/vkuzo/eed4894c5f3434e15d70b163e6077f60
+  "float8_rowwise"
+  # as of this PR, this recipe is still using AQT and CUDA graph capture time
+  # in vLLM is really slow (>5 mins)
+  # TODO(future PR): reenable this once we migrate this recipe off of AQT
+  "int8_rowwise_weight_only"
+  # TODO(future PR): fix this
+  # error: https://gist.github.com/vkuzo/5bf389079442bb9851ef315cdcb797b4
+  "int8_rowwise"
+)
+
+# TODO(future PR): add A100 and B200 tag groups
+
+# Get tag/recipe as first positional argument (optional, default: all)
+TAG_OR_RECIPE="${1:-all}"
+
+# Get model_id as second positional argument (optional)
+MODEL_ID="${2:-meta-llama/Llama-3.1-8B}"
+
+# Get log file as third positional argument (optional)
+LOG_FILE="${3:-benchmarks/data/measure_accuracy_and_performance_log.txt}"
+
+# Select recipes based on tag or specific recipe
+if [ "$TAG_OR_RECIPE" = "all" ]; then
+  QUANT_RECIPES=("${QUANT_RECIPES_ALL[@]}")
+elif [ "$TAG_OR_RECIPE" = "h100" ]; then
+  QUANT_RECIPES=("${QUANT_RECIPES_H100[@]}")
+else
+  # Check if it's a valid recipe name
+  VALID_RECIPE=false
+  for recipe in "${QUANT_RECIPES_ALL[@]}"; do
+    if [ "$recipe" = "$TAG_OR_RECIPE" ]; then
+      VALID_RECIPE=true
+      QUANT_RECIPES=("$TAG_OR_RECIPE")
+      break
+    fi
+  done
+
+  if [ "$VALID_RECIPE" = false ]; then
+    echo "Error: Invalid tag or recipe name: '$TAG_OR_RECIPE'"
+    echo ""
+    echo "Valid tags:"
+    echo "  - all"
+    echo "  - h100"
+    echo ""
+    echo "Valid recipe names:"
+    for recipe in "${QUANT_RECIPES_ALL[@]}"; do
+      echo "  - $recipe"
+    done
+    exit 1
+  fi
+fi
+
+rm -rf $LOG_FILE
+touch $LOG_FILE
+
+for quant_recipe in "${QUANT_RECIPES[@]}"; do
+
+  echo "processing $quant_recipe"
+
+  OUTPUT_DIR="benchmarks/data/quantized_model/$MODEL_ID-$quant_recipe/"
+
+  # create quantized model (unless skipped via environment variable)
+  if [ "${SKIP_MODEL_CREATE:-0}" != "1" ]; then
+    # Note: the -u flag is to prevent python from buffering stdout and stderr
+    # and make the output log file be in chronological order
+    rm -rf $OUTPUT_DIR && python -u benchmarks/quantization/create_quantized_model.py --model_id $MODEL_ID --output_dir $OUTPUT_DIR --quant_recipe_name $quant_recipe 2>&1 | tee -a "$LOG_FILE"
+  else
+    echo "Skipping model creation (SKIP_MODEL_CREATE=1), using existing model at $OUTPUT_DIR"
+  fi
+
+  # run eval (unless skipped via environment variable)
+  if [ "${SKIP_LM_EVAL:-0}" != "1" ]; then
+    lm_eval --model hf --model_args "pretrained=$OUTPUT_DIR" --tasks "wikitext,winogrande" --device "cuda:0" --batch_size auto --output_path "$OUTPUT_DIR/lm_eval_outputs/" 2>&1 | tee -a "$LOG_FILE"
+  else
+    echo "Skipping lm_eval (SKIP_LM_EVAL=1)"
+  fi
+
+  # simple performance test (unless skipped via environment variable)
+  if [ "${SKIP_VLLM:-0}" != "1" ]; then
+    # Check if this recipe is known to be broken in vllm
+    RECIPE_BROKEN_IN_VLLM=false
+    for broken_recipe in "${VLLM_BROKEN_RECIPES[@]}"; do
+      if [ "$quant_recipe" = "$broken_recipe" ]; then
+        RECIPE_BROKEN_IN_VLLM=true
+        break
+      fi
+    done
+
+    if [ "$RECIPE_BROKEN_IN_VLLM" = true ]; then
+      echo "Skipping vllm benchmarking for $quant_recipe (known to be broken in vllm)"
+    else
+      vllm bench latency --input_len 256 --output_len 256 --model $OUTPUT_DIR --batch_size 1 2>&1 | tee -a "$LOG_FILE"
+    fi
+  else
+    echo "Skipping vllm benchmarking (SKIP_VLLM=1)"
+  fi
+
+done
+
+# TODO(future PR): script to parse the log file instead of manual copy-paste