migrate to transformers v5

.github/workflows/pr_tests.yml

@@ -92,7 +92,6 @@ jobs:
  runner: aws-general-8-plus
  image: diffusers/diffusers-pytorch-cpu
  report: torch_example_cpu
-
  name: ${{ matrix.config.name }}
 
  runs-on:
@@ -115,8 +114,7 @@ jobs:
  - name: Install dependencies
  run: |
  uv pip install -e ".[quality]"
- #uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
- uv pip uninstall transformers huggingface_hub && uv pip install transformers==4.57.1
+ uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
  uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
 
  - name: Environment
@@ -218,8 +216,6 @@ jobs:
 
  run_lora_tests:
  needs: [check_code_quality, check_repository_consistency]
- strategy:
- fail-fast: false
 
  name: LoRA tests with PEFT main
 
@@ -247,9 +243,8 @@ jobs:
  uv pip install -U peft@git+https://github.com/huggingface/peft.git --no-deps
  uv pip install -U tokenizers
  uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
- #uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
- uv pip uninstall transformers huggingface_hub && uv pip install transformers==4.57.1
-
+ uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
+
  - name: Environment
  run: |
  python utils/print_env.py
@@ -275,6 +270,6 @@ jobs:
  if: ${{ always() }}
  uses: actions/upload-artifact@v6
  with:
- name: pr_main_test_reports
+ name: pr_lora_test_reports
  path: reports
 

.github/workflows/pr_tests_gpu.yml

@@ -131,8 +131,7 @@ jobs:
  run: |
  uv pip install -e ".[quality]"
  uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
- #uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
- uv pip uninstall transformers huggingface_hub && uv pip install transformers==4.57.1
+ uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
 
  - name: Environment
  run: |
@@ -202,8 +201,7 @@ jobs:
  uv pip install -e ".[quality]"
  uv pip install peft@git+https://github.com/huggingface/peft.git
  uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
- #uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
- uv pip uninstall transformers huggingface_hub && uv pip install transformers==4.57.1
+ uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
 
  - name: Environment
  run: |
@@ -264,8 +262,7 @@ jobs:
  nvidia-smi
  - name: Install dependencies
  run: |
- #uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
- uv pip uninstall transformers huggingface_hub && uv pip install transformers==4.57.1
+ uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
  uv pip install -e ".[quality,training]"
 
  - name: Environment

.github/workflows/push_tests.yml

@@ -76,8 +76,7 @@ jobs:
  run: |
  uv pip install -e ".[quality]"
  uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
- #uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
- uv pip uninstall transformers huggingface_hub && uv pip install transformers==4.57.1
+ uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
  - name: Environment
  run: |
  python utils/print_env.py
@@ -129,8 +128,7 @@ jobs:
  uv pip install -e ".[quality]"
  uv pip install peft@git+https://github.com/huggingface/peft.git
  uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
- #uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
- uv pip uninstall transformers huggingface_hub && uv pip install transformers==4.57.1
+ uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
 
  - name: Environment
  run: |
@@ -182,8 +180,7 @@ jobs:
  - name: Install dependencies
  run: |
  uv pip install -e ".[quality,training]"
- #uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
- uv pip uninstall transformers huggingface_hub && uv pip install transformers==4.57.1
+ uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
  - name: Environment
  run: |
  python utils/print_env.py

examples/custom_diffusion/test_custom_diffusion.py

@@ -17,6 +17,9 @@
 import os
 import sys
 import tempfile
+import unittest
+
+from diffusers.utils import is_transformers_version
 
 
 sys.path.append("..")
@@ -30,6 +33,7 @@
 logger.addHandler(stream_handler)
 
 
+@unittest.skipIf(is_transformers_version(">=", "4.57.5"), "Size mismatch")
 class CustomDiffusion(ExamplesTestsAccelerate):
  def test_custom_diffusion(self):
  with tempfile.TemporaryDirectory() as tmpdir:

src/diffusers/hooks/_common.py

@@ -48,6 +48,7 @@
  torch.nn.ConvTranspose2d,
  torch.nn.ConvTranspose3d,
  torch.nn.Linear,
+ torch.nn.Embedding,
  # TODO(aryan): look into torch.nn.LayerNorm, torch.nn.GroupNorm later, seems to be causing some issues with CogVideoX
  # because of double invocation of the same norm layer in CogVideoXLayerNorm
 )

src/diffusers/loaders/textual_inversion.py

@@ -22,7 +22,12 @@
 from torch import nn
 
 from ..models.modeling_utils import load_state_dict
-from ..utils import _get_model_file, is_accelerate_available, is_transformers_available, logging
+from ..utils import (
+ _get_model_file,
+ is_accelerate_available,
+ is_transformers_available,
+ logging,
+)
 
 
 if is_transformers_available():

src/diffusers/pipelines/audioldm2/pipeline_audioldm2.py

@@ -502,6 +502,10 @@ def encode_prompt(
  text_input_ids,
  attention_mask=attention_mask,
  )
+ # Extract the pooler output if it's a BaseModelOutputWithPooling (Transformers v5+)
+ # otherwise use it directly (Transformers v4)
+ if hasattr(prompt_embeds, "pooler_output"):
+ prompt_embeds = prompt_embeds.pooler_output
  # append the seq-len dim: (bs, hidden_size) -> (bs, seq_len, hidden_size)
  prompt_embeds = prompt_embeds[:, None, :]
  # make sure that we attend to this single hidden-state
@@ -610,6 +614,10 @@ def encode_prompt(
  uncond_input_ids,
  attention_mask=negative_attention_mask,
  )
+ # Extract the pooler output if it's a BaseModelOutputWithPooling (Transformers v5+)
+ # otherwise use it directly (Transformers v4)
+ if hasattr(negative_prompt_embeds, "pooler_output"):
+ negative_prompt_embeds = negative_prompt_embeds.pooler_output
  # append the seq-len dim: (bs, hidden_size) -> (bs, seq_len, hidden_size)
  negative_prompt_embeds = negative_prompt_embeds[:, None, :]
  # make sure that we attend to this single hidden-state

src/diffusers/pipelines/cosmos/pipeline_cosmos2_5_predict.py

@@ -287,6 +287,9 @@ def _get_prompt_embeds(
  truncation=True,
  padding="max_length",
  )
+ input_ids = (
+ input_ids["input_ids"] if not isinstance(input_ids, list) and "input_ids" in input_ids else input_ids
+ )
  input_ids = torch.LongTensor(input_ids)
  input_ids_batch.append(input_ids)
 

src/diffusers/pipelines/cosmos/pipeline_cosmos2_5_transfer.py

@@ -262,6 +262,9 @@ def _get_prompt_embeds(
  truncation=True,
  padding="max_length",
  )
+ input_ids = (
+ input_ids["input_ids"] if not isinstance(input_ids, list) and "input_ids" in input_ids else input_ids
+ )
  input_ids = torch.LongTensor(input_ids)
  input_ids_batch.append(input_ids)
 

src/diffusers/pipelines/kandinsky/text_encoder.py

@@ -20,6 +20,8 @@ def __init__(self, config, *args, **kwargs):
  self.LinearTransformation = torch.nn.Linear(
  in_features=config.transformerDimensions, out_features=config.numDims
  )
+ if hasattr(self, "post_init"):
+ self.post_init()
 
  def forward(self, input_ids, attention_mask):
  embs = self.transformer(input_ids=input_ids, attention_mask=attention_mask)[0]

src/diffusers/pipelines/kolors/text_encoder.py

@@ -781,6 +781,9 @@ def __init__(self, config: ChatGLMConfig, device=None, empty_init=True):
  self.prefix_encoder = PrefixEncoder(config)
  self.dropout = torch.nn.Dropout(0.1)
 
+ if hasattr(self, "post_init"):
+ self.post_init()
+
  def get_input_embeddings(self):
  return self.embedding.word_embeddings
 
@@ -810,7 +813,7 @@ def forward(
  output_hidden_states = (
  output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
  )
- use_cache = use_cache if use_cache is not None else self.config.use_cache
+ use_cache = use_cache if use_cache is not None else getattr(self.config, "use_cache", None)
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
  batch_size, seq_length = input_ids.shape

src/diffusers/pipelines/pipeline_utils.py

@@ -341,6 +341,7 @@ def is_saveable_module(name, value):
  save_method_accept_safe = "safe_serialization" in save_method_signature.parameters
  save_method_accept_variant = "variant" in save_method_signature.parameters
  save_method_accept_max_shard_size = "max_shard_size" in save_method_signature.parameters
+ save_method_accept_peft_format = "save_peft_format" in save_method_signature.parameters
 
  save_kwargs = {}
  if save_method_accept_safe:
@@ -350,6 +351,11 @@ def is_saveable_module(name, value):
  if save_method_accept_max_shard_size and max_shard_size is not None:
  # max_shard_size is expected to not be None in ModelMixin
  save_kwargs["max_shard_size"] = max_shard_size
+ if save_method_accept_peft_format:
+ # Set save_peft_format=False for transformers>=5.0.0 compatibility
+ # In transformers 5.0.0+, the default save_peft_format=True adds "base_model.model" prefix
+ # to adapter keys, but from_pretrained expects keys without this prefix
+ save_kwargs["save_peft_format"] = False
 
  save_method(os.path.join(save_directory, pipeline_component_name), **save_kwargs)
 

src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2.py

@@ -17,7 +17,7 @@
 
 import regex as re
 import torch
-from transformers import AutoTokenizer, UMT5EncoderModel
+from transformers import AutoTokenizer, T5EncoderModel, UMT5EncoderModel
 
 from ...callbacks import MultiPipelineCallbacks, PipelineCallback
 from ...loaders import SkyReelsV2LoraLoaderMixin
@@ -132,7 +132,7 @@ class SkyReelsV2Pipeline(DiffusionPipeline, SkyReelsV2LoraLoaderMixin):
  def __init__(
  self,
  tokenizer: AutoTokenizer,
- text_encoder: UMT5EncoderModel,
+ text_encoder: T5EncoderModel | UMT5EncoderModel,
  transformer: SkyReelsV2Transformer3DModel,
  vae: AutoencoderKLWan,
  scheduler: UniPCMultistepScheduler,

src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing.py

@@ -19,7 +19,7 @@
 from typing import Any, Callable
 
 import torch
-from transformers import AutoTokenizer, UMT5EncoderModel
+from transformers import AutoTokenizer, T5EncoderModel, UMT5EncoderModel
 
 from ...callbacks import MultiPipelineCallbacks, PipelineCallback
 from ...loaders import SkyReelsV2LoraLoaderMixin
@@ -153,7 +153,7 @@ class SkyReelsV2DiffusionForcingPipeline(DiffusionPipeline, SkyReelsV2LoraLoader
  def __init__(
  self,
  tokenizer: AutoTokenizer,
- text_encoder: UMT5EncoderModel,
+ text_encoder: T5EncoderModel | UMT5EncoderModel,
  transformer: SkyReelsV2Transformer3DModel,
  vae: AutoencoderKLWan,
  scheduler: UniPCMultistepScheduler,

src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_i2v.py

@@ -20,7 +20,7 @@
 
 import PIL
 import torch
-from transformers import AutoTokenizer, UMT5EncoderModel
+from transformers import AutoTokenizer, T5EncoderModel, UMT5EncoderModel
 
 from diffusers.image_processor import PipelineImageInput
 from diffusers.utils.torch_utils import randn_tensor
@@ -158,7 +158,7 @@ class SkyReelsV2DiffusionForcingImageToVideoPipeline(DiffusionPipeline, SkyReels
  def __init__(
  self,
  tokenizer: AutoTokenizer,
- text_encoder: UMT5EncoderModel,
+ text_encoder: T5EncoderModel | UMT5EncoderModel,
  transformer: SkyReelsV2Transformer3DModel,
  vae: AutoencoderKLWan,
  scheduler: UniPCMultistepScheduler,

src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_v2v.py

@@ -21,7 +21,7 @@
 
 import torch
 from PIL import Image
-from transformers import AutoTokenizer, UMT5EncoderModel
+from transformers import AutoTokenizer, T5EncoderModel, UMT5EncoderModel
 
 from ...callbacks import MultiPipelineCallbacks, PipelineCallback
 from ...loaders import SkyReelsV2LoraLoaderMixin
@@ -214,7 +214,7 @@ class SkyReelsV2DiffusionForcingVideoToVideoPipeline(DiffusionPipeline, SkyReels
  def __init__(
  self,
  tokenizer: AutoTokenizer,
- text_encoder: UMT5EncoderModel,
+ text_encoder: T5EncoderModel | UMT5EncoderModel,
  transformer: SkyReelsV2Transformer3DModel,
  vae: AutoencoderKLWan,
  scheduler: UniPCMultistepScheduler,

src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_i2v.py

@@ -18,7 +18,7 @@
 import PIL
 import regex as re
 import torch
-from transformers import AutoTokenizer, CLIPProcessor, CLIPVisionModelWithProjection, UMT5EncoderModel
+from transformers import AutoTokenizer, CLIPProcessor, CLIPVisionModelWithProjection, T5EncoderModel, UMT5EncoderModel
 
 from ...callbacks import MultiPipelineCallbacks, PipelineCallback
 from ...image_processor import PipelineImageInput
@@ -157,7 +157,7 @@ class SkyReelsV2ImageToVideoPipeline(DiffusionPipeline, SkyReelsV2LoraLoaderMixi
  def __init__(
  self,
  tokenizer: AutoTokenizer,
- text_encoder: UMT5EncoderModel,
+ text_encoder: T5EncoderModel | UMT5EncoderModel,
  image_encoder: CLIPVisionModelWithProjection,
  image_processor: CLIPProcessor,
  transformer: SkyReelsV2Transformer3DModel,

src/diffusers/pipelines/transformers_loading_utils.py

@@ -112,6 +112,8 @@ def _load_transformers_model_from_dduf(
  tensors = safetensors.torch.load(mmap)
  # Update the state dictionary with tensors
  state_dict.update(tensors)
+ # `from_pretrained` sets the model to eval mode by default, which is the
+ # correct behavior for inference. Do not call `model.train()` here.
  return cls.from_pretrained(
  pretrained_model_name_or_path=None,
  config=config,

tests/models/test_models_auto.py

@@ -20,15 +20,19 @@ def test_load_from_config_diffusers_with_subfolder(self, mock_load_config):
  side_effect=[EnvironmentError("File not found"), {"model_type": "clip_text_model"}],
  )
  def test_load_from_config_transformers_with_subfolder(self, mock_load_config):
- model = AutoModel.from_pretrained("hf-internal-testing/tiny-stable-diffusion-torch", subfolder="text_encoder")
+ model = AutoModel.from_pretrained(
+ "hf-internal-testing/tiny-stable-diffusion-torch", subfolder="text_encoder", use_safetensors=False
+ )
  assert isinstance(model, CLIPTextModel)
 
  def test_load_from_config_without_subfolder(self):
  model = AutoModel.from_pretrained("hf-internal-testing/tiny-random-longformer")
  assert isinstance(model, LongformerModel)
 
  def test_load_from_model_index(self):
- model = AutoModel.from_pretrained("hf-internal-testing/tiny-stable-diffusion-torch", subfolder="text_encoder")
+ model = AutoModel.from_pretrained(
+ "hf-internal-testing/tiny-stable-diffusion-torch", subfolder="text_encoder", use_safetensors=False
+ )
  assert isinstance(model, CLIPTextModel)
 
 

tests/pipelines/audioldm2/test_audioldm2.py

@@ -282,6 +282,8 @@ def test_audioldm2_prompt_embeds(self):
  text_inputs = text_inputs["input_ids"].to(torch_device)
 
  clap_prompt_embeds = audioldm_pipe.text_encoder.get_text_features(text_inputs)
+ if hasattr(clap_prompt_embeds, "pooler_output"):
+ clap_prompt_embeds = clap_prompt_embeds.pooler_output
  clap_prompt_embeds = clap_prompt_embeds[:, None, :]
 
  text_inputs = audioldm_pipe.tokenizer_2(
@@ -341,6 +343,8 @@ def test_audioldm2_negative_prompt_embeds(self):
  text_inputs = text_inputs["input_ids"].to(torch_device)
 
  clap_prompt_embeds = audioldm_pipe.text_encoder.get_text_features(text_inputs)
+ if hasattr(clap_prompt_embeds, "pooler_output"):
+ clap_prompt_embeds = clap_prompt_embeds.pooler_output
  clap_prompt_embeds = clap_prompt_embeds[:, None, :]
 
  text_inputs = audioldm_pipe.tokenizer_2(

tests/pipelines/bria/test_pipeline_bria.py

@@ -19,7 +19,7 @@
 import numpy as np
 import torch
 from huggingface_hub import hf_hub_download
-from transformers import T5EncoderModel, T5TokenizerFast
+from transformers import AutoConfig, T5EncoderModel, T5TokenizerFast
 
 from diffusers import (
  AutoencoderKL,
@@ -89,7 +89,8 @@ def get_dummy_components(self):
  scheduler = FlowMatchEulerDiscreteScheduler()
 
  torch.manual_seed(0)
- text_encoder = T5EncoderModel.from_pretrained("hf-internal-testing/tiny-random-t5")
+ config = AutoConfig.from_pretrained("hf-internal-testing/tiny-random-t5")
+ text_encoder = T5EncoderModel(config)
  tokenizer = T5TokenizerFast.from_pretrained("hf-internal-testing/tiny-random-t5")
 
  components = {