feat(model-manager): load ComfyUI single-file CLIP-L/CLIP-G encoders

docs/src/content/docs/Users Guide/03.Models/03.model-format-support.mdx

@@ -1,7 +1,7 @@
 ---
 title: Model Format Support
 description: Which file formats and quantizations InvokeAI loads for every model family, text encoder, VAE and adapter.
-lastUpdated: 2026-10-10
+lastUpdated: 2026-10-11
 sidebar:
   order: 3
 ---
@@ -55,7 +55,7 @@ Ministral 3B encoder. Others are recognized by their layer names and fail, or mi
 | Encoder | Used by | Folder | Single file (bf16) | fp8 scaled | int8 | nvfp4 | GGUF | SDNQ |
 |---|---|---|---|---|---|---|---|---|
 | T5-XXL | FLUX.1, SD 3 | ✓ | ✗ | ✗ | ✗ | ✗ | ✓ | ✓ (and bitsandbytes int8) |
-| CLIP-L / CLIP-G | SD, SDXL, FLUX.1, SD 3 | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | CLIP-L inside a FLUX.1 SDNQ pipeline |
+| CLIP-L / CLIP-G | SD, SDXL, FLUX.1, SD 3 | ✓ | ✓ | R | R | – | ✗ | CLIP-L inside a FLUX.1 SDNQ pipeline |
 | Qwen3 0.6B / 4B / 8B | Z-Image, FLUX.2 Klein, Anima | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
 | Qwen3-VL 4B / 8B | Krea-2, Ideogram 4 | ✓ | ✓ | ✓ | ✓ | ✗ | ✓ | ✗ |
 | Qwen3-VL 32B (truncated) | MiniMax H3 | in the Diffusers folder | ✓ | L | ✓ | L | ✗ | ✗ |
@@ -67,6 +67,11 @@ Ministral 3B encoder. Others are recognized by their layer names and fail, or mi
 | Gemma-4 12B | LTX-2 | ✓ (bf16 or int8) | ✗ | L | in the folder | L | ✗ | ✗ |
 | UMT5 | Wan 2.2 | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ |
 
+The single-file CLIP builds are ComfyUI's `clip_l` and `clip_g`, as used by FLUX.1 and SD 3. SD and SDXL take their
+CLIP encoders from the main model only, so a separate CLIP file does not change them. A CLIP-G file needs its
+`text_projection` for SD 3 and is refused without it, as are extended-context CLIP-L variants such as Long-CLIP.
+FLUX.1 takes CLIP-L only.
+
 The Qwen3-VL GGUF encoder is the language model only; a separate `mmproj` vision file is refused, and split
 multi-part GGUFs have to be joined before installing. The Gemma-4 encoder installs only as a folder: its
 `config.json` and tokenizer files next to exactly one weight file, bf16 or int8. A standalone `.safetensors` file is

docs/src/content/docs/Users Guide/03.Models/Local Models/sd-3.mdx

@@ -58,6 +58,9 @@ you can replace any of them with a model you have installed separately:
 - **CLIP G**
 - **VAE**: only SD3 VAEs are offered
 
+The CLIP encoders can also be the single files ComfyUI uses, `clip_l` and `clip_g`; SD 3 then still tokenizes
+with the main model's tokenizers.
+
 ## Generation settings
 
 When you select an SD3 model, InvokeAI applies these defaults, which come from the SD3.5 Medium

invokeai/app/invocations/flux/flux_model_loader.py

@@ -24,7 +24,7 @@
 from invokeai.backend.flux.util import get_flux_max_seq_length
 from invokeai.backend.model_manager.configs.base import Checkpoint_Config_Base
 from invokeai.backend.model_manager.configs.main import Main_SDNQ_Diffusers_FLUX_Config
-from invokeai.backend.model_manager.taxonomy import BaseModelType, ModelType, SubModelType
+from invokeai.backend.model_manager.taxonomy import BaseModelType, ClipVariantType, ModelType, SubModelType
 
 
 @invocation_output("flux_model_loader_output")
@@ -73,6 +73,7 @@ class FluxModelLoaderInvocation(BaseInvocation):
         description=FieldDescriptions.clip_embed_model,
         title="CLIP Embed",
         ui_model_type=ModelType.CLIPEmbed,
+        ui_model_variant=ClipVariantType.L,
     )
 
     vae_model: ModelIdentifierField | None = InputField(

invokeai/app/invocations/sd3/sd3_model_loader.py

@@ -93,8 +93,10 @@ def invoke(self, context: InvocationContext) -> Sd3ModelLoaderOutput:
             else self.model.model_copy(update={"submodel_type": SubModelType.TextEncoder})
         )
         tokenizer_g = self.model.model_copy(update={"submodel_type": SubModelType.Tokenizer2})
+        # A separately selected CLIP-G is a model of its own, whose encoder is its `TextEncoder`; `TextEncoder2`
+        # names CLIP-G only inside the SD 3 pipeline.
         clip_encoder_g = (
-            self.clip_g_model.model_copy(update={"submodel_type": SubModelType.TextEncoder2})
+            self.clip_g_model.model_copy(update={"submodel_type": SubModelType.TextEncoder})
             if self.clip_g_model
             else self.model.model_copy(update={"submodel_type": SubModelType.TextEncoder2})
         )

invokeai/app/invocations/text_encoder/flux_text_encoder.py

@@ -149,12 +149,17 @@ def _clip_encode(self, context: InvocationContext) -> torch.Tensor:
             context.models.load(self.clip.tokenizer) as clip_tokenizer,
             ExitStack() as exit_stack,
         ):
-            assert isinstance(clip_text_encoder, CLIPTextModel)
+            if not isinstance(clip_text_encoder, CLIPTextModel):
+                # A CLIP-G loads with its projection; FLUX.1 conditions on CLIP-L alone.
+                raise ValueError(
+                    f"FLUX.1 needs a CLIP-L text encoder; '{clip_text_encoder_config.name}' is not one. "
+                    "Select a CLIP-L encoder such as clip_l or clip-vit-large-patch14."
+                )
             assert isinstance(clip_tokenizer, CLIPTokenizer)
 
             # Apply LoRA models to the CLIP encoder.
             # Note: We apply the LoRA after the transformer has been moved to its target device for faster patching.
-            if clip_text_encoder_config.format in [ModelFormat.Diffusers]:
+            if clip_text_encoder_config.format in [ModelFormat.Diffusers, ModelFormat.Checkpoint]:
                 # The model is non-quantized, so we can apply the LoRA weights directly into the model.
                 exit_stack.enter_context(
                     LayerPatcher.apply_smart_model_patches(

invokeai/app/invocations/text_encoder/sd3_text_encoder.py

@@ -164,7 +164,7 @@ def _clip_encode(
 
             # Apply LoRA models to the CLIP encoder.
             # Note: We apply the LoRA after the transformer has been moved to its target device for faster patching.
-            if clip_text_encoder_config.format in [ModelFormat.Diffusers]:
+            if clip_text_encoder_config.format in [ModelFormat.Diffusers, ModelFormat.Checkpoint]:
                 # The model is non-quantized, so we can apply the LoRA weights directly into the model.
                 exit_stack.enter_context(
                     LayerPatcher.apply_smart_model_patches(
@@ -202,7 +202,12 @@ def _clip_encode(
                     f" {tokenizer_max_length} tokens: {removed_text}"
                 )
             prompt_embeds = clip_text_encoder(input_ids=text_input_ids.to(clip_device), output_hidden_states=True)
-            pooled_prompt_embeds = prompt_embeds[0]
+            # SD 3 conditions on the projected pooled output. A separately installed CLIP-L is a plain CLIPTextModel,
+            # whose unprojected pooled output is the same value: SD 3's CLIP-L projection is the identity.
+            if isinstance(clip_text_encoder, CLIPTextModelWithProjection):
+                pooled_prompt_embeds = prompt_embeds.text_embeds
+            else:
+                pooled_prompt_embeds = prompt_embeds.pooler_output
             prompt_embeds = prompt_embeds.hidden_states[-2]
 
             return prompt_embeds, pooled_prompt_embeds

invokeai/backend/clip/clip_g_text_config.json

@@ -0,0 +1,19 @@
+{
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_size": 1280,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 5120,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 20,
+  "num_hidden_layers": 32,
+  "pad_token_id": 1,
+  "projection_dim": 1280,
+  "vocab_size": 49408
+}

invokeai/backend/clip/clip_l_text_config.json

@@ -0,0 +1,19 @@
+{
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "quick_gelu",
+  "hidden_size": 768,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "projection_dim": 768,
+  "vocab_size": 49408
+}

invokeai/backend/clip/clip_text_encoder.py

@@ -0,0 +1,40 @@
+"""Bundled tokenizer and architectures for CLIP-L and CLIP-G text encoders installed as single files.
+
+ComfyUI distributes the text towers SD 3 and FLUX.1 use as bare state dicts (``clip_l``, ``clip_g``) with no
+config and no tokenizer. Both are fixed architectures, so their configs are vendored here rather than inferred:
+CLIP-L as ``openai/clip-vit-large-patch14`` ships it, CLIP-G as SD 3's ``text_encoder_2``. The tokenizer is
+``openai/clip-vit-large-patch14``'s (MIT); CLIP-G shares its vocabulary and merges byte for byte. Vocabulary and
+merges are stored gzip-compressed (see ``invokeai.backend.util.bundled_tokenizer``).
+"""
+
+import json
+from functools import lru_cache
+from pathlib import Path
+
+from transformers import CLIPTextConfig, CLIPTokenizer
+
+from invokeai.backend.model_manager.taxonomy import ClipVariantType
+from invokeai.backend.util.bundled_tokenizer import load_gzipped_tokenizer_dir
+
+_PACKAGE_DIR = Path(__file__).parent
+
+_TEXT_CONFIG_FILES = {
+    ClipVariantType.L: "clip_l_text_config.json",
+    ClipVariantType.G: "clip_g_text_config.json",
+}
+
+
+@lru_cache(maxsize=1)
+def load_bundled_clip_tokenizer() -> CLIPTokenizer:
+    """Load the vendored CLIP tokenizer. Result is cached for the process."""
+    tokenizer = load_gzipped_tokenizer_dir(_PACKAGE_DIR / "tokenizer")
+    assert isinstance(tokenizer, CLIPTokenizer)
+    return tokenizer
+
+
+@lru_cache(maxsize=None)
+def clip_text_config(variant: ClipVariantType) -> CLIPTextConfig:
+    """The text tower architecture of a CLIP variant."""
+    return CLIPTextConfig.from_dict(
+        json.loads((_PACKAGE_DIR / _TEXT_CONFIG_FILES[variant]).read_text(encoding="utf-8"))
+    )

invokeai/backend/clip/tokenizer/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

invokeai/backend/clip/tokenizer/tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "49406": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49407": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "do_lower_case": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 77,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "CLIPTokenizer",
+  "unk_token": "<|endoftext|>"
+}

invokeai/backend/model_manager/configs/base.py

@@ -191,7 +191,7 @@ def get_model_discriminator_value(v: Any) -> str:
             # Special case: CLIP Embed models also need the variant to distinguish them.
             if (
                 type_ == ModelType.CLIPEmbed.value
-                and format_ == ModelFormat.Diffusers.value
+                and format_ in (ModelFormat.Diffusers.value, ModelFormat.Checkpoint.value)
                 and base_ == BaseModelType.Any.value
             ):
                 if variant_ := v.get("variant"):

invokeai/backend/model_manager/configs/clip_embed.py

@@ -1,18 +1,24 @@
+import re
 from typing import (
     Literal,
     Self,
 )
 
+import torch
 from pydantic import Field
 from typing_extensions import Any
 
-from invokeai.backend.model_manager.configs.base import Config_Base, Diffusers_Config_Base
+from invokeai.backend.clip.clip_text_encoder import clip_text_config
+from invokeai.backend.model_manager.configs.base import Checkpoint_Config_Base, Config_Base, Diffusers_Config_Base
 from invokeai.backend.model_manager.configs.identification_utils import (
+    InvalidMatchError,
     NotAMatchError,
     get_config_dict_or_raise,
     raise_for_class_name,
     raise_for_override_fields,
     raise_if_not_dir,
+    raise_if_not_file,
+    state_dict_has_any_keys_starting_with,
 )
 from invokeai.backend.model_manager.model_on_disk import ModelOnDisk
 from invokeai.backend.model_manager.taxonomy import (
@@ -90,3 +96,103 @@ class CLIPEmbed_Diffusers_G_Config(CLIPEmbed_Diffusers_Config_Base, Config_Base)
 
 class CLIPEmbed_Diffusers_L_Config(CLIPEmbed_Diffusers_Config_Base, Config_Base):
     variant: Literal[ClipVariantType.L] = Field(default=ClipVariantType.L)
+
+
+_CLIP_LAYER_INDEX = re.compile(r"^text_model\.encoder\.layers\.(\d+)\.")
+_FLOAT_WEIGHT_DTYPES = (torch.float16, torch.bfloat16, torch.float32)
+
+
+class CLIPEmbed_Checkpoint_Config_Base(Checkpoint_Config_Base):
+    """A CLIP text encoder in a single safetensors file with transformers key names.
+
+    This is how ComfyUI distributes the CLIP-L and CLIP-G text towers (``clip_l``, ``clip_g``): the text model's
+    state dict under ``text_model.``, with CLIP-G's ``text_projection`` beside it, and no config or tokenizer.
+    """
+
+    base: Literal[BaseModelType.Any] = Field(default=BaseModelType.Any)
+    type: Literal[ModelType.CLIPEmbed] = Field(default=ModelType.CLIPEmbed)
+    format: Literal[ModelFormat.Checkpoint] = Field(default=ModelFormat.Checkpoint)
+    cpu_only: bool | None = Field(default=None, description="Whether this model should run on CPU only")
+
+    @classmethod
+    def from_model_on_disk(cls, mod: ModelOnDisk, override_fields: dict[str, Any]) -> Self:
+        raise_if_not_file(mod)
+
+        raise_for_override_fields(cls, override_fields)
+
+        if mod.path.suffix != ".safetensors":
+            raise NotAMatchError("not a safetensors file")
+
+        state_dict = mod.load_state_dict()
+        if (
+            "text_model.encoder.layers.0.self_attn.q_proj.weight" not in state_dict
+            or "text_model.embeddings.token_embedding.weight" not in state_dict
+        ):
+            raise NotAMatchError("state dict does not look like a transformers CLIP text encoder")
+        if state_dict_has_any_keys_starting_with(state_dict, "vision_model."):
+            raise NotAMatchError("state dict carries a vision tower; only text encoders are supported")
+
+        expected_variant = cls.model_fields["variant"].default
+        recognized_variant = cls._variant_from_shapes(state_dict)
+        if recognized_variant is not expected_variant:
+            raise NotAMatchError(f"variant is {recognized_variant}, not {expected_variant}")
+
+        # Long-CLIP has CLIP-L's width and depth but a longer position table, which the vendored architecture
+        # cannot take: recognised and refused, rather than registered to fail at the first generation.
+        config = clip_text_config(expected_variant)
+        positions = state_dict.get("text_model.embeddings.position_embedding.weight")
+        found = (
+            int(positions.shape[0]) if positions is not None else 0,
+            int(state_dict["text_model.embeddings.token_embedding.weight"].shape[0]),
+        )
+        if found != (config.max_position_embeddings, config.vocab_size):
+            raise InvalidMatchError(
+                f"this CLIP text encoder has {found[0]} positions and a {found[1]}-token vocabulary, where "
+                f"CLIP-{expected_variant.name} has {config.max_position_embeddings} and {config.vocab_size}. "
+                "Extended-context variants such as Long-CLIP are not supported."
+            )
+
+        # SD 3 conditions on CLIP-G's projected pooled output, which a file without the projection cannot give.
+        if expected_variant is ClipVariantType.G and "text_projection.weight" not in state_dict:
+            raise InvalidMatchError(
+                "this CLIP-G text encoder has no text_projection, which SD 3 needs. Install a CLIP-G that includes it, "
+                "such as the clip_g.safetensors Stability AI and Comfy-Org distribute."
+            )
+
+        quantized = sorted(
+            key
+            for key, value in state_dict.items()
+            if isinstance(key, str)
+            and key.endswith(".weight")
+            and getattr(value, "dtype", None) not in _FLOAT_WEIGHT_DTYPES
+        )
+        if quantized:
+            raise InvalidMatchError(
+                f"this CLIP text encoder carries {len(quantized)} quantized weight(s) (e.g. '{quantized[0]}'), "
+                "which are not supported. Install the fp16 build."
+            )
+
+        return cls(**override_fields)
+
+    @staticmethod
+    def _variant_from_shapes(state_dict: dict[str | int, Any]) -> ClipVariantType | None:
+        """The variant whose width and depth the file has, or None for any other CLIP text tower."""
+        width = int(state_dict["text_model.embeddings.token_embedding.weight"].shape[1])
+        depth = 1 + max(
+            int(match.group(1))
+            for key in state_dict
+            if isinstance(key, str) and (match := _CLIP_LAYER_INDEX.match(key))
+        )
+        for variant in (ClipVariantType.L, ClipVariantType.G):
+            config = clip_text_config(variant)
+            if (width, depth) == (config.hidden_size, config.num_hidden_layers):
+                return variant
+        return None
+
+
+class CLIPEmbed_Checkpoint_G_Config(CLIPEmbed_Checkpoint_Config_Base, Config_Base):
+    variant: Literal[ClipVariantType.G] = Field(default=ClipVariantType.G)
+
+
+class CLIPEmbed_Checkpoint_L_Config(CLIPEmbed_Checkpoint_Config_Base, Config_Base):
+    variant: Literal[ClipVariantType.L] = Field(default=ClipVariantType.L)