Merge 3 existing PRs for OnnxDiscrepancyCheck + llama.cpp integration (with dedicated GGUF conversion pass)

docs/source/how-to/cli/cli-fast-test.md

@@ -99,11 +99,10 @@ olive run \
     --output_path out/qwen-test-run
 ```
 
-`--test_llama_path` points to the virtual environment that contains `llama-cpp-python` and `convert_hf_to_gguf.py` (from the llama.cpp repository). When provided, Olive:
+`--test_llama_path` points to the virtual environment that contains `llama-cpp-python` and `convert_hf_to_gguf.py` (from the llama.cpp repository). When provided, Olive injects a `ConvertHfToGGUF` pass before model conversion and then:
 
-1. Saves the reference HuggingFace model to `<output_path>/hf_model` using `save_pretrained`.
-2. Calls `convert_hf_to_gguf.py` inside the virtual environment to produce a GGUF F32 file at `<output_path>/model.gguf`.
-3. Runs inference with `llama_cpp.Llama` inside the virtual environment and reports first-token latency and speedup metrics alongside the regular MAE and ONNX speedup results.
+1. Converts the reduced test HuggingFace model to GGUF (`<test_model_path>/model.gguf`) via `convert_hf_to_gguf.py`.
+2. Reuses that GGUF file in `OnnxDiscrepancyCheck` for llama.cpp inference and reports first-token latency and speedup metrics alongside the regular MAE and ONNX speedup results.
 
 All `llama-cpp-python` imports are strictly isolated to the subprocess — the main Olive process never imports them.
 

olive/cli/base.py

@@ -112,7 +112,7 @@ def warn_unused_test_metrics(test, metrics: Optional[list], llama_path: Optional
 def add_discrepancy_check_pass(
     run_config: dict, metrics: Optional[list] = None, llama_env_path: Optional[str] = None
 ) -> dict:
-    """Inject or update a SaveTestModelConfig and an OnnxDiscrepancyCheck pass when --test is active.
+    """Inject or update test-related passes when --test is active.
 
     ``metrics`` selects which test metrics to evaluate. Supported values are defined in
     ``TEST_METRICS`` (``"mae"`` for the max-absolute-error accuracy check and ``"speedup"`` for the
@@ -123,13 +123,16 @@ def add_discrepancy_check_pass(
     When provided, the ``llama_cpp`` flag is enabled on the pass and the path is forwarded as
     ``llama_cpp_env_path``.
 
-    Two passes are managed:
+    Managed passes:
 
     * ``SaveTestModelConfig`` — inserted at the *beginning* of the passes dict so that the
       test-model directory (containing only ``config.json`` and the marker file) is created
       before any other pass runs.  This ensures subsequent passes can find the directory even
       on the first ``olive run`` after ``olive optimize --dry_run --test``.
 
+    * ``ConvertHfToGGUF`` — inserted after ``SaveTestModelConfig`` when ``llama_env_path`` is
+      provided, and converts the test HuggingFace directory to GGUF in advance.
+
     * ``OnnxDiscrepancyCheck`` — appended at the end to compare the ONNX model against the
       reference HuggingFace model.  If an instance is already present in the config (e.g.
       from a previous ``--dry_run --test`` invocation), its dynamic runtime fields
@@ -162,6 +165,41 @@ def add_discrepancy_check_pass(
         passes = new_passes
         run_config["passes"] = passes
 
+    # --- ConvertHfToGGUF pass (optional, only with --test_llama_path) ---
+    if llama_env_path:
+        has_gguf_pass = any(
+            isinstance(cfg, dict) and cfg.get("type", "").lower() == "converthftogguf" for cfg in passes.values()
+        )
+        if not has_gguf_pass:
+            new_passes = {}
+            inserted = False
+            for name, cfg in passes.items():
+                new_passes[name] = cfg
+                if not inserted and isinstance(cfg, dict) and cfg.get("type", "").lower() == "savetestmodelconfig":
+                    new_passes["convert_hf_to_gguf"] = {
+                        "type": "ConvertHfToGGUF",
+                        "llama_cpp_env_path": llama_env_path,
+                        "reference_model_path": reference_model_path,
+                    }
+                    inserted = True
+            if not inserted:
+                new_passes = {
+                    "convert_hf_to_gguf": {
+                        "type": "ConvertHfToGGUF",
+                        "llama_cpp_env_path": llama_env_path,
+                        "reference_model_path": reference_model_path,
+                    },
+                    **new_passes,
+                }
+            passes = new_passes
+            run_config["passes"] = passes
+        else:
+            for pass_cfg in passes.values():
+                if isinstance(pass_cfg, dict) and pass_cfg.get("type", "").lower() == "converthftogguf":
+                    pass_cfg["llama_cpp_env_path"] = llama_env_path
+                    pass_cfg["reference_model_path"] = reference_model_path
+                    break
+
     # Determine output directory for discrepancy results
     report_dir = run_config.get("output_dir") or run_config.get("engine", {}).get("output_dir")
     if report_dir and Path(report_dir).suffix and not Path(report_dir).is_dir():
@@ -305,7 +343,7 @@ def _parse_extra_options(kv_items):
 
         from onnxruntime_genai.models.builder import parse_extra_options
 
-        return parse_extra_options(kv_items)
+        return parse_extra_options(kv_items)  # pylint: disable=no-value-for-parameter
 
     @staticmethod
     def _save_config_file(config: dict):

olive/olive_config.json

@@ -650,6 +650,14 @@
             "supported_algorithms": [  ],
             "supported_quantization_encodings": [  ]
         },
+        "ConvertHfToGGUF": {
+            "module_path": "olive.passes.pytorch.convert_hf_to_gguf.ConvertHfToGGUF",
+            "supported_providers": [ "*" ],
+            "supported_accelerators": [ "*" ],
+            "supported_precisions": [ "*" ],
+            "supported_algorithms": [  ],
+            "supported_quantization_encodings": [  ]
+        },
         "SelectiveMixedPrecision": {
             "module_path": "olive.passes.pytorch.selective_mixed_precision.SelectiveMixedPrecision",
             "supported_providers": [ "*" ],

olive/passes/onnx/discrepancy_check.py

@@ -610,15 +610,24 @@ def _run_for_config(
 
         # llama.cpp comparison: convert reference model to GGUF and compare latencies
         if config.llama_cpp:
-            llama_results = self.compare_llama_cpp(
-                config,
-                ref_model,
-                output_dir=report_dir,
-                pytorch_latency_s=results.get("pytorch_latency_s"),
-                onnx_latency_s=results.get("onnx_latency_s"),
-                ref_model_path=ref_path,
-            )
-            results.update(llama_results)
+            preconverted_gguf_path = None
+            if model.model_attributes:
+                preconverted_gguf_path = model.model_attributes.get("reference_gguf_model_path")
+            try:
+                llama_results = self.compare_llama_cpp(
+                    config,
+                    ref_model,
+                    output_dir=report_dir,
+                    pytorch_latency_s=results.get("pytorch_latency_s"),
+                    onnx_latency_s=results.get("onnx_latency_s"),
+                    ref_model_path=ref_path,
+                    preconverted_gguf_path=preconverted_gguf_path,
+                )
+                results.update(llama_results)
+            except Exception as exc:
+                logger.exception("OnnxDiscrepancyCheck llama.cpp comparison failed.")
+                results["status"] = "failed"
+                results.setdefault("failures", []).append(f"llama.cpp comparison failed: {exc}")
 
         # Save results to disk
         report_path = Path(report_dir) / "discrepancy_check_results.json"
@@ -880,6 +889,7 @@ def compare_llama_cpp(
         onnx_latency_s: Optional[float] = None,
         *,
         ref_model_path: str,
+        preconverted_gguf_path: Optional[str] = None,
     ) -> dict:
         """Convert the reference model to GGUF and compare inference with llama.cpp.
 
@@ -904,7 +914,6 @@ def compare_llama_cpp(
         # Resolve the llama_env Python interpreter and conversion script
         env_path = config.llama_cpp_env_path or "llama_env"
         python_path = self._get_llama_env_python(env_path)
-        convert_script = self._get_convert_script(env_path)
 
         # Tokenize the generation prompt using the main-env tokenizer
         tokenizer = AutoTokenizer.from_pretrained(ref_model_path)
@@ -926,19 +935,24 @@ def compare_llama_cpp(
         gguf_path = str(output_dir_path / "model.gguf")
         script_path = str(output_dir_path / "llama_cpp_helper.py")
 
-        # Save model and tokenizer in standard HuggingFace format.
-        ref_model.save_pretrained(model_dir, safe_serialization=True)
-        tokenizer.save_pretrained(model_dir)
-        logger.info("Saved reference HuggingFace model and tokenizer to %s", model_dir)
-
-        # Step 1: Convert to GGUF using the official convert_hf_to_gguf.py CLI.
-        subprocess.run(
-            [python_path, convert_script, model_dir, "--outfile", gguf_path, "--outtype", "f32"],
-            capture_output=True,
-            text=True,
-            check=True,
-        )
-        logger.info("Converted HuggingFace model to GGUF at %s", gguf_path)
+        if preconverted_gguf_path and Path(preconverted_gguf_path).exists():
+            gguf_path = preconverted_gguf_path
+            logger.info("Using pre-converted GGUF from %s", gguf_path)
+        else:
+            convert_script = self._get_convert_script(env_path)
+            # Save model and tokenizer in standard HuggingFace format.
+            ref_model.save_pretrained(model_dir, safe_serialization=True)
+            tokenizer.save_pretrained(model_dir)
+            logger.info("Saved reference HuggingFace model and tokenizer to %s", model_dir)
+
+            # Step 1: Convert to GGUF using the official convert_hf_to_gguf.py CLI.
+            subprocess.run(
+                [python_path, convert_script, model_dir, "--outfile", gguf_path, "--outtype", "f32"],
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+            logger.info("Converted HuggingFace model to GGUF at %s", gguf_path)
 
         # Step 2: Run inference inside llama_env using the pre-converted GGUF file.
         (output_dir_path / "llama_cpp_helper.py").write_text(_LLAMA_CPP_HELPER_SCRIPT)

olive/passes/pytorch/convert_hf_to_gguf.py

@@ -0,0 +1,86 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+import logging
+import subprocess
+import sys
+from pathlib import Path
+
+from olive.hardware.accelerator import AcceleratorSpec
+from olive.model import HfModelHandler
+from olive.passes import Pass
+from olive.passes.pass_config import BasePassConfig, PassConfigParam
+
+logger = logging.getLogger(__name__)
+
+
+class ConvertHfToGGUF(Pass):
+    """Convert the test HuggingFace model directory to a GGUF file."""
+
+    @classmethod
+    def _default_config(cls, accelerator_spec: AcceleratorSpec) -> dict[str, PassConfigParam]:
+        return {
+            "llama_cpp_env_path": PassConfigParam(
+                type_=str,
+                default_value="llama_env",
+                description="Path to the llama.cpp virtual environment containing convert_hf_to_gguf.py.",
+            ),
+            "reference_model_path": PassConfigParam(
+                type_=str,
+                default_value=None,
+                description="Fallback model path to convert when test_model_path is not set.",
+            ),
+            "gguf_file_name": PassConfigParam(
+                type_=str,
+                default_value="model.gguf",
+                description="GGUF output filename.",
+            ),
+        }
+
+    @staticmethod
+    def _get_python_executable(env_path: Path) -> str:
+        if sys.platform.startswith("win"):
+            return str(env_path / "Scripts" / "python.exe")
+        return str(env_path / "bin" / "python")
+
+    def _run_for_config(
+        self, model: HfModelHandler, config: type[BasePassConfig], output_model_path: str
+    ) -> HfModelHandler:
+        source_path = Path(model.test_model_path or config.reference_model_path or "")
+        if not source_path.is_dir():
+            logger.info("ConvertHfToGGUF skipped: source model directory does not exist: %s", source_path)
+            return model
+
+        gguf_path = source_path / config.gguf_file_name
+        if gguf_path.exists():
+            logger.info("ConvertHfToGGUF skipped: GGUF already exists at %s", gguf_path)
+            model_attributes = dict(model.model_attributes) if model.model_attributes else {}
+            model_attributes["reference_gguf_model_path"] = str(gguf_path)
+            model.model_attributes = model_attributes
+            return model
+
+        env_path = Path(config.llama_cpp_env_path).resolve()
+        convert_script = env_path / "convert_hf_to_gguf.py"
+        conversion_pkg = env_path / "conversion"
+        python_path = self._get_python_executable(env_path)
+
+        if not Path(python_path).exists():
+            raise RuntimeError(f"Could not find llama_env python executable: {python_path}")
+        if not convert_script.exists():
+            raise RuntimeError(f"Could not find convert_hf_to_gguf.py at: {convert_script}")
+        if not conversion_pkg.exists():
+            raise RuntimeError(f"Could not find conversion package at: {conversion_pkg}")
+
+        subprocess.run(
+            [python_path, str(convert_script), str(source_path), "--outfile", str(gguf_path), "--outtype", "f32"],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        logger.info("Converted test model to GGUF at %s", gguf_path)
+
+        model_attributes = dict(model.model_attributes) if model.model_attributes else {}
+        model_attributes["reference_gguf_model_path"] = str(gguf_path)
+        model.model_attributes = model_attributes
+        return model

test/cli/test_base.py

@@ -427,6 +427,8 @@ def test_add_discrepancy_check_pass_llama_env_path_sets_config():
     passes = run_config["passes"]
     first_key = next(iter(passes))
     assert passes[first_key]["type"] == "SaveTestModelConfig"
+    assert passes["convert_hf_to_gguf"]["type"] == "ConvertHfToGGUF"
+    assert passes["convert_hf_to_gguf"]["llama_cpp_env_path"] == "/path/to/llama_env"
 
     pass_config = passes["discrepancy_check"]
     assert pass_config["llama_cpp"] is True
@@ -443,6 +445,7 @@ def test_add_discrepancy_check_pass_no_llama_env_path_omits_llama_config():
     assert passes[first_key]["type"] == "SaveTestModelConfig"
 
     pass_config = passes["discrepancy_check"]
+    assert "convert_hf_to_gguf" not in passes
     assert "llama_cpp" not in pass_config
     assert "llama_cpp_env_path" not in pass_config
 
@@ -465,12 +468,15 @@ def test_add_discrepancy_check_pass_updates_existing_pass():
     config["input_model"]["test_model_path"] = "new_ref_model"
     config["output_dir"] = "new_out_dir"
 
-    result = add_discrepancy_check_pass(config, metrics=["mae", "speedup"])
+    result = add_discrepancy_check_pass(config, metrics=["mae", "speedup"], llama_env_path="/path/to/llama_env")
 
     passes = result["passes"]
     # SaveTestModelConfig must be injected at the beginning
     first_key = next(iter(passes))
     assert passes[first_key]["type"] == "SaveTestModelConfig"
+    assert passes["convert_hf_to_gguf"]["type"] == "ConvertHfToGGUF"
+    assert passes["convert_hf_to_gguf"]["llama_cpp_env_path"] == "/path/to/llama_env"
+    assert passes["convert_hf_to_gguf"]["reference_model_path"] == str(Path("new_ref_model").resolve())
 
     pass_config = passes["discrepancy_check"]
     # Reference model path and output dir must be updated to the current values.

test/passes/onnx/test_discrepancy_check.py

@@ -596,3 +596,54 @@ def test_compare_llama_cpp_no_latency_baselines(self, tmp_path):
         assert result["llama_cpp_speedup_vs_onnx"] is None
         assert result["llama_cpp_first_token_id"] == 7
         assert result["llama_cpp_first_token_matches_pytorch"] is True
+
+    def test_compare_llama_cpp_uses_preconverted_gguf(self, tmp_path):
+        import json
+
+        import torch
+
+        from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck
+
+        config = self._make_config()
+        gguf_path = tmp_path / "prebuilt.gguf"
+        gguf_path.write_text("ok")
+
+        mock_ref_model = MagicMock()
+        mock_ref_model.device = torch.device("cpu")
+        mock_ref_model.generate.return_value = torch.tensor([[1, 2, 3, 7]])
+
+        llama_output = {
+            "first_token_id": 7,
+            "generated_tokens": [7, 8],
+            "ttft": 0.10,
+            "ttfn": None,
+            "total_time": 0.20,
+        }
+
+        mock_proc = MagicMock()
+        mock_proc.stdout = json.dumps(llama_output)
+
+        encoded = MagicMock()
+        encoded.__getitem__ = MagicMock(side_effect=lambda k: torch.tensor([[1, 2, 3]]) if k == "input_ids" else None)
+        mock_tokenizer = MagicMock()
+        mock_tokenizer.return_value = encoded
+        mock_tokenizer.get_vocab = MagicMock(return_value={})
+
+        with (
+            patch.object(OnnxDiscrepancyCheck, "_get_llama_env_python", return_value="/mock/llama_env/bin/python"),
+            patch.object(OnnxDiscrepancyCheck, "_get_convert_script") as mock_convert_script,
+            patch("subprocess.run", return_value=mock_proc) as mock_subprocess_run,
+            patch("transformers.AutoTokenizer.from_pretrained", return_value=mock_tokenizer),
+        ):
+            pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck)
+            result = pass_instance.compare_llama_cpp(
+                config,
+                mock_ref_model,
+                output_dir=str(tmp_path),
+                ref_model_path=config.reference_model_path,
+                preconverted_gguf_path=str(gguf_path),
+            )
+
+        assert result["llama_cpp_first_token_id"] == 7
+        mock_convert_script.assert_not_called()
+        assert mock_subprocess_run.call_count == 1