From 4403ed305376ffa656212eb92e837b72a0c6ca91 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:13:21 +0000 Subject: [PATCH 1/2] Initial plan From 7b0afa69ad52bdcae642a626860d19e5f0a11139 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:35:17 +0000 Subject: [PATCH 2/2] Add ConvertHfToGGUF pass and integrate llama test flow --- docs/source/how-to/cli/cli-fast-test.md | 7 +- olive/cli/base.py | 44 +++++++++- olive/olive_config.json | 8 ++ olive/passes/onnx/discrepancy_check.py | 60 ++++++++----- olive/passes/pytorch/convert_hf_to_gguf.py | 86 +++++++++++++++++++ test/cli/test_base.py | 8 +- test/passes/onnx/test_discrepancy_check.py | 51 +++++++++++ .../passes/pytorch/test_convert_hf_to_gguf.py | 66 ++++++++++++++ 8 files changed, 299 insertions(+), 31 deletions(-) create mode 100644 olive/passes/pytorch/convert_hf_to_gguf.py create mode 100644 test/passes/pytorch/test_convert_hf_to_gguf.py diff --git a/docs/source/how-to/cli/cli-fast-test.md b/docs/source/how-to/cli/cli-fast-test.md index 588ada470..0779e065f 100644 --- a/docs/source/how-to/cli/cli-fast-test.md +++ b/docs/source/how-to/cli/cli-fast-test.md @@ -99,11 +99,10 @@ olive run \ --output_path out/qwen-test-run ``` -`--test_llama_path` points to the virtual environment that contains `llama-cpp-python` and `convert_hf_to_gguf.py` (from the llama.cpp repository). When provided, Olive: +`--test_llama_path` points to the virtual environment that contains `llama-cpp-python` and `convert_hf_to_gguf.py` (from the llama.cpp repository). When provided, Olive injects a `ConvertHfToGGUF` pass before model conversion and then: -1. Saves the reference HuggingFace model to `/hf_model` using `save_pretrained`. -2. Calls `convert_hf_to_gguf.py` inside the virtual environment to produce a GGUF F32 file at `/model.gguf`. -3. Runs inference with `llama_cpp.Llama` inside the virtual environment and reports first-token latency and speedup metrics alongside the regular MAE and ONNX speedup results. +1. Converts the reduced test HuggingFace model to GGUF (`/model.gguf`) via `convert_hf_to_gguf.py`. +2. Reuses that GGUF file in `OnnxDiscrepancyCheck` for llama.cpp inference and reports first-token latency and speedup metrics alongside the regular MAE and ONNX speedup results. All `llama-cpp-python` imports are strictly isolated to the subprocess — the main Olive process never imports them. diff --git a/olive/cli/base.py b/olive/cli/base.py index 0642e186d..7a3f2db77 100644 --- a/olive/cli/base.py +++ b/olive/cli/base.py @@ -112,7 +112,7 @@ def warn_unused_test_metrics(test, metrics: Optional[list], llama_path: Optional def add_discrepancy_check_pass( run_config: dict, metrics: Optional[list] = None, llama_env_path: Optional[str] = None ) -> dict: - """Inject or update a SaveTestModelConfig and an OnnxDiscrepancyCheck pass when --test is active. + """Inject or update test-related passes when --test is active. ``metrics`` selects which test metrics to evaluate. Supported values are defined in ``TEST_METRICS`` (``"mae"`` for the max-absolute-error accuracy check and ``"speedup"`` for the @@ -123,13 +123,16 @@ def add_discrepancy_check_pass( When provided, the ``llama_cpp`` flag is enabled on the pass and the path is forwarded as ``llama_cpp_env_path``. - Two passes are managed: + Managed passes: * ``SaveTestModelConfig`` — inserted at the *beginning* of the passes dict so that the test-model directory (containing only ``config.json`` and the marker file) is created before any other pass runs. This ensures subsequent passes can find the directory even on the first ``olive run`` after ``olive optimize --dry_run --test``. + * ``ConvertHfToGGUF`` — inserted after ``SaveTestModelConfig`` when ``llama_env_path`` is + provided, and converts the test HuggingFace directory to GGUF in advance. + * ``OnnxDiscrepancyCheck`` — appended at the end to compare the ONNX model against the reference HuggingFace model. If an instance is already present in the config (e.g. from a previous ``--dry_run --test`` invocation), its dynamic runtime fields @@ -162,6 +165,41 @@ def add_discrepancy_check_pass( passes = new_passes run_config["passes"] = passes + # --- ConvertHfToGGUF pass (optional, only with --test_llama_path) --- + if llama_env_path: + has_gguf_pass = any( + isinstance(cfg, dict) and cfg.get("type", "").lower() == "converthftogguf" for cfg in passes.values() + ) + if not has_gguf_pass: + new_passes = {} + inserted = False + for name, cfg in passes.items(): + new_passes[name] = cfg + if not inserted and isinstance(cfg, dict) and cfg.get("type", "").lower() == "savetestmodelconfig": + new_passes["convert_hf_to_gguf"] = { + "type": "ConvertHfToGGUF", + "llama_cpp_env_path": llama_env_path, + "reference_model_path": reference_model_path, + } + inserted = True + if not inserted: + new_passes = { + "convert_hf_to_gguf": { + "type": "ConvertHfToGGUF", + "llama_cpp_env_path": llama_env_path, + "reference_model_path": reference_model_path, + }, + **new_passes, + } + passes = new_passes + run_config["passes"] = passes + else: + for pass_cfg in passes.values(): + if isinstance(pass_cfg, dict) and pass_cfg.get("type", "").lower() == "converthftogguf": + pass_cfg["llama_cpp_env_path"] = llama_env_path + pass_cfg["reference_model_path"] = reference_model_path + break + # Determine output directory for discrepancy results report_dir = run_config.get("output_dir") or run_config.get("engine", {}).get("output_dir") if report_dir and Path(report_dir).suffix and not Path(report_dir).is_dir(): @@ -305,7 +343,7 @@ def _parse_extra_options(kv_items): from onnxruntime_genai.models.builder import parse_extra_options - return parse_extra_options(kv_items) + return parse_extra_options(kv_items) # pylint: disable=no-value-for-parameter @staticmethod def _save_config_file(config: dict): diff --git a/olive/olive_config.json b/olive/olive_config.json index d3b21ae89..ecd8c59c8 100644 --- a/olive/olive_config.json +++ b/olive/olive_config.json @@ -650,6 +650,14 @@ "supported_algorithms": [ ], "supported_quantization_encodings": [ ] }, + "ConvertHfToGGUF": { + "module_path": "olive.passes.pytorch.convert_hf_to_gguf.ConvertHfToGGUF", + "supported_providers": [ "*" ], + "supported_accelerators": [ "*" ], + "supported_precisions": [ "*" ], + "supported_algorithms": [ ], + "supported_quantization_encodings": [ ] + }, "SelectiveMixedPrecision": { "module_path": "olive.passes.pytorch.selective_mixed_precision.SelectiveMixedPrecision", "supported_providers": [ "*" ], diff --git a/olive/passes/onnx/discrepancy_check.py b/olive/passes/onnx/discrepancy_check.py index 4441f7161..29e69a2c4 100644 --- a/olive/passes/onnx/discrepancy_check.py +++ b/olive/passes/onnx/discrepancy_check.py @@ -610,15 +610,24 @@ def _run_for_config( # llama.cpp comparison: convert reference model to GGUF and compare latencies if config.llama_cpp: - llama_results = self.compare_llama_cpp( - config, - ref_model, - output_dir=report_dir, - pytorch_latency_s=results.get("pytorch_latency_s"), - onnx_latency_s=results.get("onnx_latency_s"), - ref_model_path=ref_path, - ) - results.update(llama_results) + preconverted_gguf_path = None + if model.model_attributes: + preconverted_gguf_path = model.model_attributes.get("reference_gguf_model_path") + try: + llama_results = self.compare_llama_cpp( + config, + ref_model, + output_dir=report_dir, + pytorch_latency_s=results.get("pytorch_latency_s"), + onnx_latency_s=results.get("onnx_latency_s"), + ref_model_path=ref_path, + preconverted_gguf_path=preconverted_gguf_path, + ) + results.update(llama_results) + except Exception as exc: + logger.exception("OnnxDiscrepancyCheck llama.cpp comparison failed.") + results["status"] = "failed" + results.setdefault("failures", []).append(f"llama.cpp comparison failed: {exc}") # Save results to disk report_path = Path(report_dir) / "discrepancy_check_results.json" @@ -880,6 +889,7 @@ def compare_llama_cpp( onnx_latency_s: Optional[float] = None, *, ref_model_path: str, + preconverted_gguf_path: Optional[str] = None, ) -> dict: """Convert the reference model to GGUF and compare inference with llama.cpp. @@ -904,7 +914,6 @@ def compare_llama_cpp( # Resolve the llama_env Python interpreter and conversion script env_path = config.llama_cpp_env_path or "llama_env" python_path = self._get_llama_env_python(env_path) - convert_script = self._get_convert_script(env_path) # Tokenize the generation prompt using the main-env tokenizer tokenizer = AutoTokenizer.from_pretrained(ref_model_path) @@ -926,19 +935,24 @@ def compare_llama_cpp( gguf_path = str(output_dir_path / "model.gguf") script_path = str(output_dir_path / "llama_cpp_helper.py") - # Save model and tokenizer in standard HuggingFace format. - ref_model.save_pretrained(model_dir, safe_serialization=True) - tokenizer.save_pretrained(model_dir) - logger.info("Saved reference HuggingFace model and tokenizer to %s", model_dir) - - # Step 1: Convert to GGUF using the official convert_hf_to_gguf.py CLI. - subprocess.run( - [python_path, convert_script, model_dir, "--outfile", gguf_path, "--outtype", "f32"], - capture_output=True, - text=True, - check=True, - ) - logger.info("Converted HuggingFace model to GGUF at %s", gguf_path) + if preconverted_gguf_path and Path(preconverted_gguf_path).exists(): + gguf_path = preconverted_gguf_path + logger.info("Using pre-converted GGUF from %s", gguf_path) + else: + convert_script = self._get_convert_script(env_path) + # Save model and tokenizer in standard HuggingFace format. + ref_model.save_pretrained(model_dir, safe_serialization=True) + tokenizer.save_pretrained(model_dir) + logger.info("Saved reference HuggingFace model and tokenizer to %s", model_dir) + + # Step 1: Convert to GGUF using the official convert_hf_to_gguf.py CLI. + subprocess.run( + [python_path, convert_script, model_dir, "--outfile", gguf_path, "--outtype", "f32"], + capture_output=True, + text=True, + check=True, + ) + logger.info("Converted HuggingFace model to GGUF at %s", gguf_path) # Step 2: Run inference inside llama_env using the pre-converted GGUF file. (output_dir_path / "llama_cpp_helper.py").write_text(_LLAMA_CPP_HELPER_SCRIPT) diff --git a/olive/passes/pytorch/convert_hf_to_gguf.py b/olive/passes/pytorch/convert_hf_to_gguf.py new file mode 100644 index 000000000..cc8765382 --- /dev/null +++ b/olive/passes/pytorch/convert_hf_to_gguf.py @@ -0,0 +1,86 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- +import logging +import subprocess +import sys +from pathlib import Path + +from olive.hardware.accelerator import AcceleratorSpec +from olive.model import HfModelHandler +from olive.passes import Pass +from olive.passes.pass_config import BasePassConfig, PassConfigParam + +logger = logging.getLogger(__name__) + + +class ConvertHfToGGUF(Pass): + """Convert the test HuggingFace model directory to a GGUF file.""" + + @classmethod + def _default_config(cls, accelerator_spec: AcceleratorSpec) -> dict[str, PassConfigParam]: + return { + "llama_cpp_env_path": PassConfigParam( + type_=str, + default_value="llama_env", + description="Path to the llama.cpp virtual environment containing convert_hf_to_gguf.py.", + ), + "reference_model_path": PassConfigParam( + type_=str, + default_value=None, + description="Fallback model path to convert when test_model_path is not set.", + ), + "gguf_file_name": PassConfigParam( + type_=str, + default_value="model.gguf", + description="GGUF output filename.", + ), + } + + @staticmethod + def _get_python_executable(env_path: Path) -> str: + if sys.platform.startswith("win"): + return str(env_path / "Scripts" / "python.exe") + return str(env_path / "bin" / "python") + + def _run_for_config( + self, model: HfModelHandler, config: type[BasePassConfig], output_model_path: str + ) -> HfModelHandler: + source_path = Path(model.test_model_path or config.reference_model_path or "") + if not source_path.is_dir(): + logger.info("ConvertHfToGGUF skipped: source model directory does not exist: %s", source_path) + return model + + gguf_path = source_path / config.gguf_file_name + if gguf_path.exists(): + logger.info("ConvertHfToGGUF skipped: GGUF already exists at %s", gguf_path) + model_attributes = dict(model.model_attributes) if model.model_attributes else {} + model_attributes["reference_gguf_model_path"] = str(gguf_path) + model.model_attributes = model_attributes + return model + + env_path = Path(config.llama_cpp_env_path).resolve() + convert_script = env_path / "convert_hf_to_gguf.py" + conversion_pkg = env_path / "conversion" + python_path = self._get_python_executable(env_path) + + if not Path(python_path).exists(): + raise RuntimeError(f"Could not find llama_env python executable: {python_path}") + if not convert_script.exists(): + raise RuntimeError(f"Could not find convert_hf_to_gguf.py at: {convert_script}") + if not conversion_pkg.exists(): + raise RuntimeError(f"Could not find conversion package at: {conversion_pkg}") + + subprocess.run( + [python_path, str(convert_script), str(source_path), "--outfile", str(gguf_path), "--outtype", "f32"], + capture_output=True, + text=True, + check=True, + ) + logger.info("Converted test model to GGUF at %s", gguf_path) + + model_attributes = dict(model.model_attributes) if model.model_attributes else {} + model_attributes["reference_gguf_model_path"] = str(gguf_path) + model.model_attributes = model_attributes + return model diff --git a/test/cli/test_base.py b/test/cli/test_base.py index 8d6c25391..47a7619b3 100644 --- a/test/cli/test_base.py +++ b/test/cli/test_base.py @@ -427,6 +427,8 @@ def test_add_discrepancy_check_pass_llama_env_path_sets_config(): passes = run_config["passes"] first_key = next(iter(passes)) assert passes[first_key]["type"] == "SaveTestModelConfig" + assert passes["convert_hf_to_gguf"]["type"] == "ConvertHfToGGUF" + assert passes["convert_hf_to_gguf"]["llama_cpp_env_path"] == "/path/to/llama_env" pass_config = passes["discrepancy_check"] assert pass_config["llama_cpp"] is True @@ -443,6 +445,7 @@ def test_add_discrepancy_check_pass_no_llama_env_path_omits_llama_config(): assert passes[first_key]["type"] == "SaveTestModelConfig" pass_config = passes["discrepancy_check"] + assert "convert_hf_to_gguf" not in passes assert "llama_cpp" not in pass_config assert "llama_cpp_env_path" not in pass_config @@ -465,12 +468,15 @@ def test_add_discrepancy_check_pass_updates_existing_pass(): config["input_model"]["test_model_path"] = "new_ref_model" config["output_dir"] = "new_out_dir" - result = add_discrepancy_check_pass(config, metrics=["mae", "speedup"]) + result = add_discrepancy_check_pass(config, metrics=["mae", "speedup"], llama_env_path="/path/to/llama_env") passes = result["passes"] # SaveTestModelConfig must be injected at the beginning first_key = next(iter(passes)) assert passes[first_key]["type"] == "SaveTestModelConfig" + assert passes["convert_hf_to_gguf"]["type"] == "ConvertHfToGGUF" + assert passes["convert_hf_to_gguf"]["llama_cpp_env_path"] == "/path/to/llama_env" + assert passes["convert_hf_to_gguf"]["reference_model_path"] == str(Path("new_ref_model").resolve()) pass_config = passes["discrepancy_check"] # Reference model path and output dir must be updated to the current values. diff --git a/test/passes/onnx/test_discrepancy_check.py b/test/passes/onnx/test_discrepancy_check.py index e128682ea..7286e3ab0 100644 --- a/test/passes/onnx/test_discrepancy_check.py +++ b/test/passes/onnx/test_discrepancy_check.py @@ -596,3 +596,54 @@ def test_compare_llama_cpp_no_latency_baselines(self, tmp_path): assert result["llama_cpp_speedup_vs_onnx"] is None assert result["llama_cpp_first_token_id"] == 7 assert result["llama_cpp_first_token_matches_pytorch"] is True + + def test_compare_llama_cpp_uses_preconverted_gguf(self, tmp_path): + import json + + import torch + + from olive.passes.onnx.discrepancy_check import OnnxDiscrepancyCheck + + config = self._make_config() + gguf_path = tmp_path / "prebuilt.gguf" + gguf_path.write_text("ok") + + mock_ref_model = MagicMock() + mock_ref_model.device = torch.device("cpu") + mock_ref_model.generate.return_value = torch.tensor([[1, 2, 3, 7]]) + + llama_output = { + "first_token_id": 7, + "generated_tokens": [7, 8], + "ttft": 0.10, + "ttfn": None, + "total_time": 0.20, + } + + mock_proc = MagicMock() + mock_proc.stdout = json.dumps(llama_output) + + encoded = MagicMock() + encoded.__getitem__ = MagicMock(side_effect=lambda k: torch.tensor([[1, 2, 3]]) if k == "input_ids" else None) + mock_tokenizer = MagicMock() + mock_tokenizer.return_value = encoded + mock_tokenizer.get_vocab = MagicMock(return_value={}) + + with ( + patch.object(OnnxDiscrepancyCheck, "_get_llama_env_python", return_value="/mock/llama_env/bin/python"), + patch.object(OnnxDiscrepancyCheck, "_get_convert_script") as mock_convert_script, + patch("subprocess.run", return_value=mock_proc) as mock_subprocess_run, + patch("transformers.AutoTokenizer.from_pretrained", return_value=mock_tokenizer), + ): + pass_instance = OnnxDiscrepancyCheck.__new__(OnnxDiscrepancyCheck) + result = pass_instance.compare_llama_cpp( + config, + mock_ref_model, + output_dir=str(tmp_path), + ref_model_path=config.reference_model_path, + preconverted_gguf_path=str(gguf_path), + ) + + assert result["llama_cpp_first_token_id"] == 7 + mock_convert_script.assert_not_called() + assert mock_subprocess_run.call_count == 1 diff --git a/test/passes/pytorch/test_convert_hf_to_gguf.py b/test/passes/pytorch/test_convert_hf_to_gguf.py new file mode 100644 index 000000000..e8ed5d8eb --- /dev/null +++ b/test/passes/pytorch/test_convert_hf_to_gguf.py @@ -0,0 +1,66 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- +# pylint: disable=protected-access + +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from olive.passes.pytorch.convert_hf_to_gguf import ConvertHfToGGUF + + +def test_convert_hf_to_gguf_skips_when_missing_source(tmp_path): + pass_instance = ConvertHfToGGUF.__new__(ConvertHfToGGUF) + model = SimpleNamespace(test_model_path=str(tmp_path / "missing"), model_attributes=None) + config = SimpleNamespace( + llama_cpp_env_path=str(tmp_path / "llama_env"), + reference_model_path=str(tmp_path / "missing"), + gguf_file_name="model.gguf", + ) + + result = pass_instance._run_for_config(model, config, str(tmp_path / "out")) + assert result is model + + +def test_convert_hf_to_gguf_uses_existing_gguf(tmp_path): + source = tmp_path / "test_model" + source.mkdir(parents=True, exist_ok=True) + gguf_path = source / "model.gguf" + gguf_path.write_text("ok") + + pass_instance = ConvertHfToGGUF.__new__(ConvertHfToGGUF) + model = SimpleNamespace(test_model_path=str(source), model_attributes={}) + config = SimpleNamespace( + llama_cpp_env_path=str(tmp_path / "llama_env"), + reference_model_path=str(source), + gguf_file_name="model.gguf", + ) + + result = pass_instance._run_for_config(model, config, str(tmp_path / "out")) + assert result.model_attributes["reference_gguf_model_path"] == str(gguf_path) + + +def test_convert_hf_to_gguf_runs_conversion(tmp_path): + source = tmp_path / "test_model" + source.mkdir(parents=True, exist_ok=True) + env = tmp_path / "llama_env" + (env / "bin").mkdir(parents=True, exist_ok=True) + (env / "bin" / "python").write_text("") + (env / "convert_hf_to_gguf.py").write_text("") + (env / "conversion").mkdir(parents=True, exist_ok=True) + + pass_instance = ConvertHfToGGUF.__new__(ConvertHfToGGUF) + model = SimpleNamespace(test_model_path=str(source), model_attributes={}) + config = SimpleNamespace( + llama_cpp_env_path=str(env), + reference_model_path=str(source), + gguf_file_name="model.gguf", + ) + + with patch("olive.passes.pytorch.convert_hf_to_gguf.subprocess.run") as mock_run: + result = pass_instance._run_for_config(model, config, str(tmp_path / "out")) + + assert mock_run.call_count == 1 + assert Path(result.model_attributes["reference_gguf_model_path"]).name == "model.gguf"