diff --git a/src/diffusers/modular_pipelines/flux2/encoders.py b/src/diffusers/modular_pipelines/flux2/encoders.py index 09615c4becb6..99d7f92547fa 100644 --- a/src/diffusers/modular_pipelines/flux2/encoders.py +++ b/src/diffusers/modular_pipelines/flux2/encoders.py @@ -330,8 +330,9 @@ def _get_qwen3_prompt_embeds( all_input_ids.append(inputs["input_ids"]) all_attention_masks.append(inputs["attention_mask"]) - input_ids = torch.cat(all_input_ids, dim=0).to(device) - attention_mask = torch.cat(all_attention_masks, dim=0).to(device) + model_device = text_encoder.device + input_ids = torch.cat(all_input_ids, dim=0).to(model_device) + attention_mask = torch.cat(all_attention_masks, dim=0).to(model_device) # Forward pass through the model output = text_encoder( @@ -471,8 +472,9 @@ def _get_qwen3_prompt_embeds( all_input_ids.append(inputs["input_ids"]) all_attention_masks.append(inputs["attention_mask"]) - input_ids = torch.cat(all_input_ids, dim=0).to(device) - attention_mask = torch.cat(all_attention_masks, dim=0).to(device) + model_device = text_encoder.device + input_ids = torch.cat(all_input_ids, dim=0).to(model_device) + attention_mask = torch.cat(all_attention_masks, dim=0).to(model_device) # Forward pass through the model output = text_encoder( diff --git a/src/diffusers/modular_pipelines/stable_diffusion_xl/encoders.py b/src/diffusers/modular_pipelines/stable_diffusion_xl/encoders.py index 26e5524309f1..b19b568480fe 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_xl/encoders.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_xl/encoders.py @@ -117,7 +117,7 @@ def encode_image(components, image, device, num_images_per_prompt, output_hidden if not isinstance(image, torch.Tensor): image = components.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=components.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = components.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/animatediff/pipeline_animatediff.py b/src/diffusers/pipelines/animatediff/pipeline_animatediff.py index b9e5b40b65ff..947762bcefb6 100644 --- a/src/diffusers/pipelines/animatediff/pipeline_animatediff.py +++ b/src/diffusers/pipelines/animatediff/pipeline_animatediff.py @@ -236,17 +236,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -306,13 +307,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -340,7 +342,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/animatediff/pipeline_animatediff_controlnet.py b/src/diffusers/pipelines/animatediff/pipeline_animatediff_controlnet.py index a9630cc3c00f..ddb5166472bb 100644 --- a/src/diffusers/pipelines/animatediff/pipeline_animatediff_controlnet.py +++ b/src/diffusers/pipelines/animatediff/pipeline_animatediff_controlnet.py @@ -281,17 +281,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -351,13 +352,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -385,7 +387,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/animatediff/pipeline_animatediff_sdxl.py b/src/diffusers/pipelines/animatediff/pipeline_animatediff_sdxl.py index a221f3e1efeb..885f9cd30641 100644 --- a/src/diffusers/pipelines/animatediff/pipeline_animatediff_sdxl.py +++ b/src/diffusers/pipelines/animatediff/pipeline_animatediff_sdxl.py @@ -446,7 +446,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -507,7 +507,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) @@ -569,7 +569,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/animatediff/pipeline_animatediff_sparsectrl.py b/src/diffusers/pipelines/animatediff/pipeline_animatediff_sparsectrl.py index fcf260b47f3a..938078099b1c 100644 --- a/src/diffusers/pipelines/animatediff/pipeline_animatediff_sparsectrl.py +++ b/src/diffusers/pipelines/animatediff/pipeline_animatediff_sparsectrl.py @@ -290,17 +290,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -360,13 +361,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -394,7 +396,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py b/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py index b8aa82ab9d2f..a572514397cb 100644 --- a/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py +++ b/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py @@ -442,7 +442,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video_controlnet.py b/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video_controlnet.py index 7c649b501f32..cb6a6b69785e 100644 --- a/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video_controlnet.py +++ b/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video_controlnet.py @@ -473,7 +473,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/anyflow/pipeline_anyflow.py b/src/diffusers/pipelines/anyflow/pipeline_anyflow.py index c3e1dbf3a459..39d47e0e9174 100644 --- a/src/diffusers/pipelines/anyflow/pipeline_anyflow.py +++ b/src/diffusers/pipelines/anyflow/pipeline_anyflow.py @@ -161,7 +161,8 @@ def _get_t5_prompt_embeds( text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask seq_lens = mask.gt(0).sum(dim=1).long() - prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), mask.to(model_device)).last_hidden_state prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] prompt_embeds = torch.stack( diff --git a/src/diffusers/pipelines/anyflow/pipeline_anyflow_far.py b/src/diffusers/pipelines/anyflow/pipeline_anyflow_far.py index 96edc07a0043..e6179bf69076 100644 --- a/src/diffusers/pipelines/anyflow/pipeline_anyflow_far.py +++ b/src/diffusers/pipelines/anyflow/pipeline_anyflow_far.py @@ -178,7 +178,8 @@ def _get_t5_prompt_embeds( text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask seq_lens = mask.gt(0).sum(dim=1).long() - prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), mask.to(model_device)).last_hidden_state prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] prompt_embeds = torch.stack( diff --git a/src/diffusers/pipelines/chroma/pipeline_chroma.py b/src/diffusers/pipelines/chroma/pipeline_chroma.py index c375703ca89f..64294d1a2609 100644 --- a/src/diffusers/pipelines/chroma/pipeline_chroma.py +++ b/src/diffusers/pipelines/chroma/pipeline_chroma.py @@ -372,7 +372,7 @@ def encode_image(self, image, device, num_images_per_prompt): if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) image_embeds = self.image_encoder(image).image_embeds image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) return image_embeds diff --git a/src/diffusers/pipelines/chroma/pipeline_chroma_img2img.py b/src/diffusers/pipelines/chroma/pipeline_chroma_img2img.py index 80a4b6a8cf16..2756be9ab590 100644 --- a/src/diffusers/pipelines/chroma/pipeline_chroma_img2img.py +++ b/src/diffusers/pipelines/chroma/pipeline_chroma_img2img.py @@ -399,7 +399,7 @@ def encode_image(self, image, device, num_images_per_prompt): if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) image_embeds = self.image_encoder(image).image_embeds image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) return image_embeds diff --git a/src/diffusers/pipelines/chroma/pipeline_chroma_inpainting.py b/src/diffusers/pipelines/chroma/pipeline_chroma_inpainting.py index f1e9530e6029..9ca8e07a419f 100644 --- a/src/diffusers/pipelines/chroma/pipeline_chroma_inpainting.py +++ b/src/diffusers/pipelines/chroma/pipeline_chroma_inpainting.py @@ -401,7 +401,7 @@ def encode_image(self, image, device, num_images_per_prompt): if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) image_embeds = self.image_encoder(image).image_embeds image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) return image_embeds diff --git a/src/diffusers/pipelines/chronoedit/pipeline_chronoedit.py b/src/diffusers/pipelines/chronoedit/pipeline_chronoedit.py index 1e0cc0ea5c2a..87e2f9c1be9f 100644 --- a/src/diffusers/pipelines/chronoedit/pipeline_chronoedit.py +++ b/src/diffusers/pipelines/chronoedit/pipeline_chronoedit.py @@ -210,7 +210,8 @@ def _get_t5_prompt_embeds( text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask seq_lens = mask.gt(0).sum(dim=1).long() - prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), mask.to(model_device)).last_hidden_state prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] prompt_embeds = torch.stack( @@ -231,9 +232,9 @@ def encode_image( device: torch.device | None = None, ): device = device or self._execution_device - image = self.image_processor(images=image, return_tensors="pt").to(device) + image = self.image_processor(images=image, return_tensors="pt").to(self.image_encoder.device) image_embeds = self.image_encoder(**image, output_hidden_states=True) - return image_embeds.hidden_states[-2] + return image_embeds.hidden_states[-2].to(device) # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline.encode_prompt def encode_prompt( diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet.py index 7ef287ae8154..470633a9389b 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet.py @@ -380,17 +380,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -450,13 +451,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -484,7 +486,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -557,7 +559,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py index b40dda940959..ef6ad214bf40 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py @@ -358,17 +358,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -428,13 +429,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -462,7 +464,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -535,7 +537,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py index c981dd14abd6..81325249234a 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py @@ -364,17 +364,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -434,13 +435,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -468,7 +470,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -541,7 +543,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py index f27fcd8aa26f..b610a85dcb7f 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py @@ -410,7 +410,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -471,7 +471,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) @@ -533,7 +533,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py index 9a4badb8ce1e..48ca7cad0642 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py @@ -418,7 +418,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -479,7 +479,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) @@ -541,7 +541,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py index 1de2f5d66da0..402c0d2ba4ce 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py @@ -411,7 +411,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -472,7 +472,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) @@ -534,7 +534,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_inpaint_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_inpaint_sd_xl.py index 6d7e1f62beb3..f40dd24e28f3 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_inpaint_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_inpaint_sd_xl.py @@ -399,7 +399,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -460,7 +460,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) @@ -522,7 +522,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl.py index b8ddeb894aab..f583ba16a28f 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl.py @@ -410,7 +410,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -471,7 +471,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) @@ -533,7 +533,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl_img2img.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl_img2img.py index dee22942eb8f..eaf7568b0fad 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl_img2img.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl_img2img.py @@ -429,7 +429,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -490,7 +490,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) @@ -552,7 +552,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/controlnet_hunyuandit/pipeline_hunyuandit_controlnet.py b/src/diffusers/pipelines/controlnet_hunyuandit/pipeline_hunyuandit_controlnet.py index ba241bf4feb6..c8f7f6d75562 100644 --- a/src/diffusers/pipelines/controlnet_hunyuandit/pipeline_hunyuandit_controlnet.py +++ b/src/diffusers/pipelines/controlnet_hunyuandit/pipeline_hunyuandit_controlnet.py @@ -447,7 +447,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet.py b/src/diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet.py index 4530a424adb4..58ebaaafa7de 100644 --- a/src/diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet.py +++ b/src/diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet.py @@ -292,7 +292,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_3(text_input_ids.to(device))[0] + model_device = self.text_encoder_3.device + prompt_embeds = self.text_encoder_3(text_input_ids.to(model_device))[0] dtype = self.text_encoder_3.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -341,7 +342,7 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) pooled_prompt_embeds = prompt_embeds[0] if clip_skip is None: @@ -754,7 +755,7 @@ def encode_image(self, image: PipelineImageInput, device: torch.device) -> torch if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=self.dtype) + image = image.to(device=self.image_encoder.device, dtype=self.dtype) return self.image_encoder(image, output_hidden_states=True).hidden_states[-2] diff --git a/src/diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet_inpainting.py b/src/diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet_inpainting.py index d2890d55811c..0ae8b9d41203 100644 --- a/src/diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet_inpainting.py +++ b/src/diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet_inpainting.py @@ -314,7 +314,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_3(text_input_ids.to(device))[0] + model_device = self.text_encoder_3.device + prompt_embeds = self.text_encoder_3(text_input_ids.to(model_device))[0] dtype = self.text_encoder_3.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -363,7 +364,7 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) pooled_prompt_embeds = prompt_embeds[0] if clip_skip is None: @@ -921,7 +922,7 @@ def encode_image(self, image: PipelineImageInput, device: torch.device) -> torch if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=self.dtype) + image = image.to(device=self.image_encoder.device, dtype=self.dtype) return self.image_encoder(image, output_hidden_states=True).hidden_states[-2] diff --git a/src/diffusers/pipelines/deprecated/controlnet_xs/pipeline_controlnet_xs.py b/src/diffusers/pipelines/deprecated/controlnet_xs/pipeline_controlnet_xs.py index 049f4994a4cf..29edb80f433b 100644 --- a/src/diffusers/pipelines/deprecated/controlnet_xs/pipeline_controlnet_xs.py +++ b/src/diffusers/pipelines/deprecated/controlnet_xs/pipeline_controlnet_xs.py @@ -314,17 +314,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -384,13 +385,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -420,7 +422,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/deprecated/controlnet_xs/pipeline_controlnet_xs_sd_xl.py b/src/diffusers/pipelines/deprecated/controlnet_xs/pipeline_controlnet_xs_sd_xl.py index 18d546b0a822..fd7da53f054b 100644 --- a/src/diffusers/pipelines/deprecated/controlnet_xs/pipeline_controlnet_xs_sd_xl.py +++ b/src/diffusers/pipelines/deprecated/controlnet_xs/pipeline_controlnet_xs_sd_xl.py @@ -342,7 +342,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -403,7 +403,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) diff --git a/src/diffusers/pipelines/deprecated/paint_by_example/pipeline_paint_by_example.py b/src/diffusers/pipelines/deprecated/paint_by_example/pipeline_paint_by_example.py index 665eae146b72..ea0c358a6e02 100644 --- a/src/diffusers/pipelines/deprecated/paint_by_example/pipeline_paint_by_example.py +++ b/src/diffusers/pipelines/deprecated/paint_by_example/pipeline_paint_by_example.py @@ -226,7 +226,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/deprecated/pia/pipeline_pia.py b/src/diffusers/pipelines/deprecated/pia/pipeline_pia.py index d98de77aa871..ef079a519557 100644 --- a/src/diffusers/pipelines/deprecated/pia/pipeline_pia.py +++ b/src/diffusers/pipelines/deprecated/pia/pipeline_pia.py @@ -298,17 +298,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -368,13 +369,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -402,7 +404,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/deprecated/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py b/src/diffusers/pipelines/deprecated/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py index bb3009d238a4..944efa5970b1 100644 --- a/src/diffusers/pipelines/deprecated/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py +++ b/src/diffusers/pipelines/deprecated/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py @@ -108,7 +108,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py b/src/diffusers/pipelines/deprecated/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py index 6a9e031f8d14..14a4db4c9ea6 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py @@ -378,17 +378,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -448,13 +449,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -484,7 +486,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py b/src/diffusers/pipelines/deprecated/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py index ee8675678f2d..a3166d753610 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py @@ -504,17 +504,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -574,13 +575,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -610,7 +612,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py b/src/diffusers/pipelines/deprecated/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py index 38f5af842e1b..7c5f464f7b1e 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py @@ -302,17 +302,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -372,13 +373,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -408,7 +410,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py b/src/diffusers/pipelines/deprecated/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py index d72d12a64945..e1387044d1ce 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py @@ -333,17 +333,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -403,13 +404,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -439,7 +441,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py b/src/diffusers/pipelines/deprecated/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py index 70a16f5d522f..eca45e89ea3a 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py @@ -394,17 +394,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -464,13 +465,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -498,7 +500,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py b/src/diffusers/pipelines/deprecated/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py index 481c9c93ddde..e6e12d1d8ce4 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py @@ -365,17 +365,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -435,13 +436,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -469,7 +471,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -542,7 +544,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_safe/pipeline_stable_diffusion_safe.py b/src/diffusers/pipelines/deprecated/stable_diffusion_safe/pipeline_stable_diffusion_safe.py index 35c7f9b970b9..f7dcc794e55f 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_safe/pipeline_stable_diffusion_safe.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_safe/pipeline_stable_diffusion_safe.py @@ -498,7 +498,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_sag/pipeline_stable_diffusion_sag.py b/src/diffusers/pipelines/deprecated/stable_diffusion_sag/pipeline_stable_diffusion_sag.py index 678ef74f387c..4a2e53159113 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_sag/pipeline_stable_diffusion_sag.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_sag/pipeline_stable_diffusion_sag.py @@ -293,17 +293,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -363,13 +364,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -397,7 +399,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -458,7 +460,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py index c5a1127c2908..007a0789bf03 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py @@ -370,17 +370,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -440,13 +441,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -545,7 +547,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py index 00d2a77269fd..582565e21ea4 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py @@ -341,17 +341,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -411,13 +412,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -447,7 +449,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py index 851820c00aed..13e1c6663993 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py @@ -274,17 +274,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -344,13 +345,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -380,7 +382,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py index ea81be87a0f4..d6503f46ad82 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py @@ -271,17 +271,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -341,13 +342,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -377,7 +379,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py index f88c6d8fbc30..79ab04ec2246 100644 --- a/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py +++ b/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py @@ -489,17 +489,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -559,13 +560,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -595,7 +597,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/deprecated/text_to_video_synthesis/pipeline_text_to_video_synth.py b/src/diffusers/pipelines/deprecated/text_to_video_synthesis/pipeline_text_to_video_synth.py index b3d52f1de279..102554cb3823 100644 --- a/src/diffusers/pipelines/deprecated/text_to_video_synthesis/pipeline_text_to_video_synth.py +++ b/src/diffusers/pipelines/deprecated/text_to_video_synthesis/pipeline_text_to_video_synth.py @@ -241,17 +241,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -311,13 +312,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] diff --git a/src/diffusers/pipelines/deprecated/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py b/src/diffusers/pipelines/deprecated/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py index c0d71c6ffd88..00edee1f28dc 100644 --- a/src/diffusers/pipelines/deprecated/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py +++ b/src/diffusers/pipelines/deprecated/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py @@ -276,17 +276,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -346,13 +347,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] diff --git a/src/diffusers/pipelines/deprecated/text_to_video_synthesis/pipeline_text_to_video_zero.py b/src/diffusers/pipelines/deprecated/text_to_video_synthesis/pipeline_text_to_video_zero.py index 6ea24ae2c817..9cb945bd6e71 100644 --- a/src/diffusers/pipelines/deprecated/text_to_video_synthesis/pipeline_text_to_video_zero.py +++ b/src/diffusers/pipelines/deprecated/text_to_video_synthesis/pipeline_text_to_video_zero.py @@ -788,7 +788,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) @@ -898,17 +900,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -968,13 +971,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] diff --git a/src/diffusers/pipelines/deprecated/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py b/src/diffusers/pipelines/deprecated/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py index 9af63e5044bd..862de96605c7 100644 --- a/src/diffusers/pipelines/deprecated/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py +++ b/src/diffusers/pipelines/deprecated/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py @@ -704,7 +704,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -765,7 +765,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) diff --git a/src/diffusers/pipelines/deprecated/unidiffuser/pipeline_unidiffuser.py b/src/diffusers/pipelines/deprecated/unidiffuser/pipeline_unidiffuser.py index 3b3db863744b..9f2a493df88b 100644 --- a/src/diffusers/pipelines/deprecated/unidiffuser/pipeline_unidiffuser.py +++ b/src/diffusers/pipelines/deprecated/unidiffuser/pipeline_unidiffuser.py @@ -446,17 +446,18 @@ def encode_prompt( f" {self.clip_tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -516,13 +517,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] diff --git a/src/diffusers/pipelines/flux/pipeline_flux.py b/src/diffusers/pipelines/flux/pipeline_flux.py index d3e0682c5419..d33e6e4bb09d 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux.py +++ b/src/diffusers/pipelines/flux/pipeline_flux.py @@ -250,7 +250,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0] + model_device = self.text_encoder_2.device + prompt_embeds = self.text_encoder_2(text_input_ids.to(model_device), output_hidden_states=False)[0] dtype = self.text_encoder_2.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -295,7 +296,8 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False) + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), output_hidden_states=False) # Use pooled output of CLIPTextModel prompt_embeds = prompt_embeds.pooler_output @@ -392,7 +394,7 @@ def encode_image(self, image, device, num_images_per_prompt): if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) image_embeds = self.image_encoder(image).image_embeds image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) return image_embeds @@ -958,6 +960,7 @@ def __call__( else: latents = self._unpack_latents(latents, height, width, self.vae_scale_factor) latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor + latents = latents.to(self.vae.device) image = self.vae.decode(latents, return_dict=False)[0] image = self.image_processor.postprocess(image, output_type=output_type) diff --git a/src/diffusers/pipelines/flux/pipeline_flux_control.py b/src/diffusers/pipelines/flux/pipeline_flux_control.py index 46671c44cca8..f8e270cba83b 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_control.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_control.py @@ -262,7 +262,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0] + model_device = self.text_encoder_2.device + prompt_embeds = self.text_encoder_2(text_input_ids.to(model_device), output_hidden_states=False)[0] dtype = self.text_encoder_2.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -308,7 +309,8 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False) + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), output_hidden_states=False) # Use pooled output of CLIPTextModel prompt_embeds = prompt_embeds.pooler_output diff --git a/src/diffusers/pipelines/flux/pipeline_flux_control_img2img.py b/src/diffusers/pipelines/flux/pipeline_flux_control_img2img.py index b455c611e0ae..663390114d3e 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_control_img2img.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_control_img2img.py @@ -273,7 +273,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0] + model_device = self.text_encoder_2.device + prompt_embeds = self.text_encoder_2(text_input_ids.to(model_device), output_hidden_states=False)[0] dtype = self.text_encoder_2.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -319,7 +320,8 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False) + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), output_hidden_states=False) # Use pooled output of CLIPTextModel prompt_embeds = prompt_embeds.pooler_output diff --git a/src/diffusers/pipelines/flux/pipeline_flux_control_inpaint.py b/src/diffusers/pipelines/flux/pipeline_flux_control_inpaint.py index 15e27653c3e2..44a0b85336e4 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_control_inpaint.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_control_inpaint.py @@ -312,7 +312,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0] + model_device = self.text_encoder_2.device + prompt_embeds = self.text_encoder_2(text_input_ids.to(model_device), output_hidden_states=False)[0] dtype = self.text_encoder_2.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -358,7 +359,8 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False) + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), output_hidden_states=False) # Use pooled output of CLIPTextModel prompt_embeds = prompt_embeds.pooler_output diff --git a/src/diffusers/pipelines/flux/pipeline_flux_controlnet.py b/src/diffusers/pipelines/flux/pipeline_flux_controlnet.py index e7792d667f16..7d01307dfd1d 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_controlnet.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_controlnet.py @@ -282,7 +282,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0] + model_device = self.text_encoder_2.device + prompt_embeds = self.text_encoder_2(text_input_ids.to(model_device), output_hidden_states=False)[0] dtype = self.text_encoder_2.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -327,7 +328,8 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False) + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), output_hidden_states=False) # Use pooled output of CLIPTextModel prompt_embeds = prompt_embeds.pooler_output @@ -428,7 +430,7 @@ def encode_image(self, image, device, num_images_per_prompt): if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) image_embeds = self.image_encoder(image).image_embeds image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) return image_embeds diff --git a/src/diffusers/pipelines/flux/pipeline_flux_controlnet_image_to_image.py b/src/diffusers/pipelines/flux/pipeline_flux_controlnet_image_to_image.py index 61c9da0c9496..be3253c6e634 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_controlnet_image_to_image.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_controlnet_image_to_image.py @@ -274,7 +274,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0] + model_device = self.text_encoder_2.device + prompt_embeds = self.text_encoder_2(text_input_ids.to(model_device), output_hidden_states=False)[0] dtype = self.text_encoder_2.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -320,7 +321,8 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False) + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), output_hidden_states=False) # Use pooled output of CLIPTextModel prompt_embeds = prompt_embeds.pooler_output diff --git a/src/diffusers/pipelines/flux/pipeline_flux_controlnet_inpainting.py b/src/diffusers/pipelines/flux/pipeline_flux_controlnet_inpainting.py index eed671152bc9..b93c3f969f91 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_controlnet_inpainting.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_controlnet_inpainting.py @@ -285,7 +285,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0] + model_device = self.text_encoder_2.device + prompt_embeds = self.text_encoder_2(text_input_ids.to(model_device), output_hidden_states=False)[0] dtype = self.text_encoder_2.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -331,7 +332,8 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False) + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), output_hidden_states=False) # Use pooled output of CLIPTextModel prompt_embeds = prompt_embeds.pooler_output diff --git a/src/diffusers/pipelines/flux/pipeline_flux_fill.py b/src/diffusers/pipelines/flux/pipeline_flux_fill.py index ab4431b5b768..a3f6ffc9349b 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_fill.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_fill.py @@ -277,7 +277,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0] + model_device = self.text_encoder_2.device + prompt_embeds = self.text_encoder_2(text_input_ids.to(model_device), output_hidden_states=False)[0] dtype = self.text_encoder_2.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -323,7 +324,8 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False) + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), output_hidden_states=False) # Use pooled output of CLIPTextModel prompt_embeds = prompt_embeds.pooler_output diff --git a/src/diffusers/pipelines/flux/pipeline_flux_img2img.py b/src/diffusers/pipelines/flux/pipeline_flux_img2img.py index 94582a84cf84..59908cfd91f3 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_img2img.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_img2img.py @@ -271,7 +271,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0] + model_device = self.text_encoder_2.device + prompt_embeds = self.text_encoder_2(text_input_ids.to(model_device), output_hidden_states=False)[0] dtype = self.text_encoder_2.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -317,7 +318,8 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False) + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), output_hidden_states=False) # Use pooled output of CLIPTextModel prompt_embeds = prompt_embeds.pooler_output @@ -416,7 +418,7 @@ def encode_image(self, image, device, num_images_per_prompt): if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) image_embeds = self.image_encoder(image).image_embeds image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) return image_embeds diff --git a/src/diffusers/pipelines/flux/pipeline_flux_inpaint.py b/src/diffusers/pipelines/flux/pipeline_flux_inpaint.py index 4c35ffefe088..5844e50eb929 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_inpaint.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_inpaint.py @@ -275,7 +275,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0] + model_device = self.text_encoder_2.device + prompt_embeds = self.text_encoder_2(text_input_ids.to(model_device), output_hidden_states=False)[0] dtype = self.text_encoder_2.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -321,7 +322,8 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False) + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), output_hidden_states=False) # Use pooled output of CLIPTextModel prompt_embeds = prompt_embeds.pooler_output @@ -420,7 +422,7 @@ def encode_image(self, image, device, num_images_per_prompt): if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) image_embeds = self.image_encoder(image).image_embeds image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) return image_embeds diff --git a/src/diffusers/pipelines/flux/pipeline_flux_kontext.py b/src/diffusers/pipelines/flux/pipeline_flux_kontext.py index 849d9686de62..c7a5bf45ae1b 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_kontext.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_kontext.py @@ -296,7 +296,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0] + model_device = self.text_encoder_2.device + prompt_embeds = self.text_encoder_2(text_input_ids.to(model_device), output_hidden_states=False)[0] dtype = self.text_encoder_2.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -342,7 +343,8 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False) + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), output_hidden_states=False) # Use pooled output of CLIPTextModel prompt_embeds = prompt_embeds.pooler_output @@ -441,7 +443,7 @@ def encode_image(self, image, device, num_images_per_prompt): if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) image_embeds = self.image_encoder(image).image_embeds image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) return image_embeds diff --git a/src/diffusers/pipelines/flux/pipeline_flux_kontext_inpaint.py b/src/diffusers/pipelines/flux/pipeline_flux_kontext_inpaint.py index 982581c01b3c..dff9a16ee15b 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_kontext_inpaint.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_kontext_inpaint.py @@ -329,7 +329,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0] + model_device = self.text_encoder_2.device + prompt_embeds = self.text_encoder_2(text_input_ids.to(model_device), output_hidden_states=False)[0] dtype = self.text_encoder_2.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -375,7 +376,8 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False) + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), output_hidden_states=False) # Use pooled output of CLIPTextModel prompt_embeds = prompt_embeds.pooler_output @@ -474,7 +476,7 @@ def encode_image(self, image, device, num_images_per_prompt): if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) image_embeds = self.image_encoder(image).image_embeds image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) return image_embeds diff --git a/src/diffusers/pipelines/flux/pipeline_flux_prior_redux.py b/src/diffusers/pipelines/flux/pipeline_flux_prior_redux.py index f173fdef88c6..08861437f99f 100644 --- a/src/diffusers/pipelines/flux/pipeline_flux_prior_redux.py +++ b/src/diffusers/pipelines/flux/pipeline_flux_prior_redux.py @@ -234,7 +234,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0] + model_device = self.text_encoder_2.device + prompt_embeds = self.text_encoder_2(text_input_ids.to(model_device), output_hidden_states=False)[0] dtype = self.text_encoder_2.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -280,7 +281,8 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False) + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), output_hidden_states=False) # Use pooled output of CLIPTextModel prompt_embeds = prompt_embeds.pooler_output diff --git a/src/diffusers/pipelines/flux2/pipeline_flux2.py b/src/diffusers/pipelines/flux2/pipeline_flux2.py index b1645b4ae244..04e7d9abe37c 100644 --- a/src/diffusers/pipelines/flux2/pipeline_flux2.py +++ b/src/diffusers/pipelines/flux2/pipeline_flux2.py @@ -661,7 +661,7 @@ def prepare_image_latents( ): image_latents = [] for image in images: - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.vae.device, dtype=dtype) imagge_latent = self._encode_vae_image(image=image, generator=generator) image_latents.append(imagge_latent) # (1, 128, 32, 32) @@ -680,6 +680,7 @@ def prepare_image_latents( image_latents = image_latents.unsqueeze(0) # (1, N*1024, 128) image_latents = image_latents.repeat(batch_size, 1, 1) + image_latents = image_latents.to(device) image_latent_ids = image_latent_ids.repeat(batch_size, 1, 1) image_latent_ids = image_latent_ids.to(device) @@ -1021,6 +1022,7 @@ def __call__( ) latents = latents * latents_bn_std + latents_bn_mean latents = self._unpatchify_latents(latents) + latents = latents.to(self.vae.device) image = self.vae.decode(latents, return_dict=False)[0] image = self.image_processor.postprocess(image, output_type=output_type) diff --git a/src/diffusers/pipelines/flux2/pipeline_flux2_klein.py b/src/diffusers/pipelines/flux2/pipeline_flux2_klein.py index d768e6127f26..d85d40321689 100644 --- a/src/diffusers/pipelines/flux2/pipeline_flux2_klein.py +++ b/src/diffusers/pipelines/flux2/pipeline_flux2_klein.py @@ -241,8 +241,9 @@ def _get_qwen3_prompt_embeds( all_input_ids.append(inputs["input_ids"]) all_attention_masks.append(inputs["attention_mask"]) - input_ids = torch.cat(all_input_ids, dim=0).to(device) - attention_mask = torch.cat(all_attention_masks, dim=0).to(device) + model_device = text_encoder.device + input_ids = torch.cat(all_input_ids, dim=0).to(model_device) + attention_mask = torch.cat(all_attention_masks, dim=0).to(model_device) # Forward pass through the model output = text_encoder( @@ -520,7 +521,7 @@ def prepare_image_latents( ): image_latents = [] for image in images: - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.vae.device, dtype=dtype) imagge_latent = self._encode_vae_image(image=image, generator=generator) image_latents.append(imagge_latent) # (1, 128, 32, 32) @@ -539,6 +540,7 @@ def prepare_image_latents( image_latents = image_latents.unsqueeze(0) # (1, N*1024, 128) image_latents = image_latents.repeat(batch_size, 1, 1) + image_latents = image_latents.to(device) image_latent_ids = image_latent_ids.repeat(batch_size, 1, 1) image_latent_ids = image_latent_ids.to(device) @@ -915,6 +917,7 @@ def __call__( if output_type == "latent": image = latents else: + latents = latents.to(self.vae.device) image = self.vae.decode(latents, return_dict=False)[0] image = self.image_processor.postprocess(image, output_type=output_type) diff --git a/src/diffusers/pipelines/flux2/pipeline_flux2_klein_inpaint.py b/src/diffusers/pipelines/flux2/pipeline_flux2_klein_inpaint.py index fd9467003a71..e11c159807aa 100644 --- a/src/diffusers/pipelines/flux2/pipeline_flux2_klein_inpaint.py +++ b/src/diffusers/pipelines/flux2/pipeline_flux2_klein_inpaint.py @@ -288,8 +288,9 @@ def _get_qwen3_prompt_embeds( all_input_ids.append(inputs["input_ids"]) all_attention_masks.append(inputs["attention_mask"]) - input_ids = torch.cat(all_input_ids, dim=0).to(device) - attention_mask = torch.cat(all_attention_masks, dim=0).to(device) + model_device = text_encoder.device + input_ids = torch.cat(all_input_ids, dim=0).to(model_device) + attention_mask = torch.cat(all_attention_masks, dim=0).to(model_device) # Forward pass through the model output = text_encoder( @@ -584,7 +585,7 @@ def prepare_latents( latent_image_ids = self._prepare_latent_ids(dummy_latents) latent_image_ids = latent_image_ids.to(device) - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.vae.device, dtype=dtype) if image.shape[1] != self.latent_channels * 4: image_latents = self._encode_vae_image(image=image, generator=generator) else: @@ -595,6 +596,8 @@ def prepare_latents( ) image_latents = (image_latents - latents_bn_mean) / latents_bn_std + image_latents = image_latents.to(device) + if batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] == 0: # expand init_latents for batch_size additional_image_per_prompt = batch_size // image_latents.shape[0] @@ -626,7 +629,7 @@ def prepare_image_latents( ): image_latents = [] for image in images: - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.vae.device, dtype=dtype) if image.shape[1] != self.latent_channels * 4: image_latent = self._encode_vae_image(image=image, generator=generator) @@ -655,6 +658,7 @@ def prepare_image_latents( final_latents.append(packed) image_latents = torch.cat(final_latents, dim=1) # (batch_size, total_seq_len, 128) + image_latents = image_latents.to(device) image_latent_ids = image_latent_ids.to(device) diff --git a/src/diffusers/pipelines/flux2/pipeline_flux2_klein_kv.py b/src/diffusers/pipelines/flux2/pipeline_flux2_klein_kv.py index 78ed42f20afb..d273346bcb0a 100644 --- a/src/diffusers/pipelines/flux2/pipeline_flux2_klein_kv.py +++ b/src/diffusers/pipelines/flux2/pipeline_flux2_klein_kv.py @@ -529,7 +529,7 @@ def prepare_image_latents( ): image_latents = [] for image in images: - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.vae.device, dtype=dtype) imagge_latent = self._encode_vae_image(image=image, generator=generator) image_latents.append(imagge_latent) # (1, 128, 32, 32) @@ -548,6 +548,7 @@ def prepare_image_latents( image_latents = image_latents.unsqueeze(0) # (1, N*1024, 128) image_latents = image_latents.repeat(batch_size, 1, 1) + image_latents = image_latents.to(device) image_latent_ids = image_latent_ids.repeat(batch_size, 1, 1) image_latent_ids = image_latent_ids.to(device) diff --git a/src/diffusers/pipelines/hunyuandit/pipeline_hunyuandit.py b/src/diffusers/pipelines/hunyuandit/pipeline_hunyuandit.py index 5d656a3c370a..12353bc9d334 100644 --- a/src/diffusers/pipelines/hunyuandit/pipeline_hunyuandit.py +++ b/src/diffusers/pipelines/hunyuandit/pipeline_hunyuandit.py @@ -419,7 +419,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/kolors/pipeline_kolors.py b/src/diffusers/pipelines/kolors/pipeline_kolors.py index 1e11faf8b9b6..958c886c9433 100644 --- a/src/diffusers/pipelines/kolors/pipeline_kolors.py +++ b/src/diffusers/pipelines/kolors/pipeline_kolors.py @@ -367,7 +367,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/kolors/pipeline_kolors_img2img.py b/src/diffusers/pipelines/kolors/pipeline_kolors_img2img.py index d9b519267216..037a3331e325 100644 --- a/src/diffusers/pipelines/kolors/pipeline_kolors_img2img.py +++ b/src/diffusers/pipelines/kolors/pipeline_kolors_img2img.py @@ -387,7 +387,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py b/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py index 424a2c46e06b..92cfd67cfe98 100644 --- a/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py +++ b/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py @@ -323,17 +323,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -393,13 +394,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -427,7 +429,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -500,7 +502,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py b/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py index 60f59ec7f9d3..424c8b6a6691 100644 --- a/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py +++ b/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py @@ -308,17 +308,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -378,13 +379,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -412,7 +414,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -485,7 +487,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py b/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py index c6cdb127309b..14a2e6db74ab 100644 --- a/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py +++ b/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py @@ -417,7 +417,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/lucy/pipeline_lucy_edit.py b/src/diffusers/pipelines/lucy/pipeline_lucy_edit.py index 69eb2a02be5c..b9c9a8fb74f0 100644 --- a/src/diffusers/pipelines/lucy/pipeline_lucy_edit.py +++ b/src/diffusers/pipelines/lucy/pipeline_lucy_edit.py @@ -221,7 +221,8 @@ def _get_t5_prompt_embeds( text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask seq_lens = mask.gt(0).sum(dim=1).long() - prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), mask.to(model_device)).last_hidden_state prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] prompt_embeds = torch.stack( diff --git a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd.py b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd.py index 3a88272f24f4..805ed27c59ac 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd.py @@ -357,17 +357,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -427,13 +428,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -461,7 +463,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -534,7 +536,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_inpaint.py b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_inpaint.py index 98221e4c30ba..7f0d4b5b2f18 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_inpaint.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_inpaint.py @@ -333,17 +333,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -403,13 +404,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -437,7 +439,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -510,7 +512,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl.py b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl.py index 9ba42eb0352f..9ac20834c4ef 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl.py @@ -425,7 +425,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -486,7 +486,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) @@ -548,7 +548,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl_img2img.py b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl_img2img.py index bfe45761d5a0..c61919026338 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl_img2img.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl_img2img.py @@ -416,7 +416,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -477,7 +477,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) @@ -539,7 +539,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/pag/pipeline_pag_hunyuandit.py b/src/diffusers/pipelines/pag/pipeline_pag_hunyuandit.py index a443a19bd952..2ae2b4788b70 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_hunyuandit.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_hunyuandit.py @@ -429,7 +429,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/pag/pipeline_pag_kolors.py b/src/diffusers/pipelines/pag/pipeline_pag_kolors.py index 4f138d91d9c6..dc095d1b148d 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_kolors.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_kolors.py @@ -384,7 +384,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd.py b/src/diffusers/pipelines/pag/pipeline_pag_sd.py index b12597460f65..88c1e50f6ded 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd.py @@ -386,17 +386,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -456,13 +457,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -490,7 +492,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -563,7 +565,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_3.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_3.py index d86adccc2ccf..9dd76aa2ed13 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_3.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_3.py @@ -262,7 +262,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_3(text_input_ids.to(device))[0] + model_device = self.text_encoder_3.device + prompt_embeds = self.text_encoder_3(text_input_ids.to(model_device))[0] dtype = self.text_encoder_3.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -311,7 +312,7 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) pooled_prompt_embeds = prompt_embeds[0] if clip_skip is None: diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_3_img2img.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_3_img2img.py index 24f3d828bd81..2e78f9af040e 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_3_img2img.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_3_img2img.py @@ -278,7 +278,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_3(text_input_ids.to(device))[0] + model_device = self.text_encoder_3.device + prompt_embeds = self.text_encoder_3(text_input_ids.to(model_device))[0] dtype = self.text_encoder_3.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -327,7 +328,7 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) pooled_prompt_embeds = prompt_embeds[0] if clip_skip is None: diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_animatediff.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_animatediff.py index 2baeda5649ad..89dba211118b 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_animatediff.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_animatediff.py @@ -247,17 +247,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -317,13 +318,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -351,7 +353,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_img2img.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_img2img.py index de6dfbc585fa..86df5ec34026 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_img2img.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_img2img.py @@ -381,17 +381,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -451,13 +452,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -485,7 +487,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -558,7 +560,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_inpaint.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_inpaint.py index 426419f12f73..88cf5fc46369 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_inpaint.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_inpaint.py @@ -416,17 +416,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -486,13 +487,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -520,7 +522,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -593,7 +595,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_xl.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_xl.py index 1de57a922616..b2078206776e 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_xl.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_xl.py @@ -412,7 +412,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -473,7 +473,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) @@ -535,7 +535,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_img2img.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_img2img.py index 201d16a86f8a..60f538bc5ebb 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_img2img.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_img2img.py @@ -430,7 +430,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -491,7 +491,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) @@ -789,7 +789,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_inpaint.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_inpaint.py index 9e70a7779f1e..43108e86f4c0 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_inpaint.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_inpaint.py @@ -333,7 +333,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -520,7 +520,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -581,7 +581,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) diff --git a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2.py b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2.py index faad0fb14086..848cbd6d200c 100644 --- a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2.py +++ b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2.py @@ -179,7 +179,8 @@ def _get_t5_prompt_embeds( text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask seq_lens = mask.gt(0).sum(dim=1).long() - prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), mask.to(model_device)).last_hidden_state prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] prompt_embeds = torch.stack( diff --git a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing.py b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing.py index 8751240a1af9..fc4345fab832 100644 --- a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing.py +++ b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing.py @@ -200,7 +200,8 @@ def _get_t5_prompt_embeds( text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask seq_lens = mask.gt(0).sum(dim=1).long() - prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), mask.to(model_device)).last_hidden_state prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] prompt_embeds = torch.stack( diff --git a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_i2v.py b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_i2v.py index 335a38e53004..1171b21eb36b 100644 --- a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_i2v.py +++ b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_i2v.py @@ -205,7 +205,8 @@ def _get_t5_prompt_embeds( text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask seq_lens = mask.gt(0).sum(dim=1).long() - prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), mask.to(model_device)).last_hidden_state prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] prompt_embeds = torch.stack( diff --git a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_v2v.py b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_v2v.py index 810b0d019ddb..8701ab823889 100644 --- a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_v2v.py +++ b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_diffusion_forcing_v2v.py @@ -261,7 +261,8 @@ def _get_t5_prompt_embeds( text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask seq_lens = mask.gt(0).sum(dim=1).long() - prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), mask.to(model_device)).last_hidden_state prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] prompt_embeds = torch.stack( diff --git a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_i2v.py b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_i2v.py index 91c09a56fcfb..6727703e0a2b 100644 --- a/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_i2v.py +++ b/src/diffusers/pipelines/skyreels_v2/pipeline_skyreels_v2_i2v.py @@ -209,7 +209,8 @@ def _get_t5_prompt_embeds( text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask seq_lens = mask.gt(0).sum(dim=1).long() - prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), mask.to(model_device)).last_hidden_state prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] prompt_embeds = torch.stack( @@ -230,9 +231,9 @@ def encode_image( device: torch.device | None = None, ): device = device or self._execution_device - image = self.image_processor(images=image, return_tensors="pt").to(device) + image = self.image_processor(images=image, return_tensors="pt").to(self.image_encoder.device) image_embeds = self.image_encoder(**image, output_hidden_states=True) - return image_embeds.hidden_states[-2] + return image_embeds.hidden_states[-2].to(device) # Copied from diffusers.pipelines.wan.pipeline_wan_i2v.WanImageToVideoPipeline.encode_prompt def encode_prompt( diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index d28bb2a9fe59..b8e76973bdc5 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -414,17 +414,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -484,13 +485,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -517,7 +519,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -588,7 +590,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) @@ -1082,6 +1086,7 @@ def __call__( xm.mark_step() if not output_type == "latent": + latents = latents.to(self.vae.device) image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[ 0 ] diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py index 977de5d7fb39..02534c638318 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py @@ -286,17 +286,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -356,13 +357,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -392,7 +394,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py index 15b8daf334ed..a4c9acc0795b 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py @@ -175,7 +175,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index 719be9258341..09b1441f0b98 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -440,17 +440,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -510,13 +511,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -544,7 +546,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -617,7 +619,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index 96794eaa297a..e3c8e1e4f636 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -394,17 +394,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -464,13 +465,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -498,7 +500,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -571,7 +573,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py index 7a24e6008351..9d05ae6709f2 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py @@ -649,7 +649,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -735,7 +735,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py index 1a0a7412e5d7..8357745b6c91 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py @@ -299,17 +299,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -369,13 +370,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py index d2c9bf0c4162..26e23525dd75 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py @@ -379,17 +379,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -449,13 +450,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py index 059ae1e6fd4d..9ca7e252e389 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py @@ -341,17 +341,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -411,13 +412,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] diff --git a/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py b/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py index 5c05b469660f..387c29776207 100644 --- a/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py +++ b/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py @@ -273,7 +273,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_3(text_input_ids.to(device))[0] + model_device = self.text_encoder_3.device + prompt_embeds = self.text_encoder_3(text_input_ids.to(model_device))[0] dtype = self.text_encoder_3.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -321,7 +322,7 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) pooled_prompt_embeds = prompt_embeds[0] if clip_skip is None: @@ -708,7 +709,7 @@ def encode_image(self, image: PipelineImageInput, device: torch.device) -> torch if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=self.dtype) + image = image.to(device=self.image_encoder.device, dtype=self.dtype) return self.image_encoder(image, output_hidden_states=True).hidden_states[-2] @@ -1134,6 +1135,7 @@ def __call__( else: latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor + latents = latents.to(self.vae.device) image = self.vae.decode(latents, return_dict=False)[0] image = self.image_processor.postprocess(image, output_type=output_type) diff --git a/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py b/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py index c0ab805a4ef4..7ef3861cefde 100644 --- a/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py @@ -297,7 +297,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_3(text_input_ids.to(device))[0] + model_device = self.text_encoder_3.device + prompt_embeds = self.text_encoder_3(text_input_ids.to(model_device))[0] dtype = self.text_encoder_3.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -346,7 +347,7 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) pooled_prompt_embeds = prompt_embeds[0] if clip_skip is None: @@ -764,7 +765,7 @@ def encode_image(self, image: PipelineImageInput, device: torch.device) -> torch if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=self.dtype) + image = image.to(device=self.image_encoder.device, dtype=self.dtype) return self.image_encoder(image, output_hidden_states=True).hidden_states[-2] diff --git a/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_inpaint.py b/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_inpaint.py index 321e9f8dd80e..d750d6eed503 100644 --- a/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_inpaint.py @@ -303,7 +303,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_3(text_input_ids.to(device))[0] + model_device = self.text_encoder_3.device + prompt_embeds = self.text_encoder_3(text_input_ids.to(model_device))[0] dtype = self.text_encoder_3.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -352,7 +353,7 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) pooled_prompt_embeds = prompt_embeds[0] if clip_skip is None: @@ -856,7 +857,7 @@ def encode_image(self, image: PipelineImageInput, device: torch.device) -> torch if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=self.dtype) + image = image.to(device=self.image_encoder.device, dtype=self.dtype) return self.image_encoder(image, output_hidden_states=True).hidden_states[-2] diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py index ba94e3051fd3..901463fc41e9 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py @@ -402,7 +402,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -463,7 +463,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) @@ -525,7 +525,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -1284,6 +1284,7 @@ def __call__( else: latents = latents / self.vae.config.scaling_factor + latents = latents.to(self.vae.device) image = self.vae.decode(latents, return_dict=False)[0] # cast back to fp16 if needed diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py index c7a13ca02524..a78e193a5f27 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py @@ -420,7 +420,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -481,7 +481,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) @@ -776,7 +776,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py index 3f18cbe21d0f..502221152d40 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py @@ -337,7 +337,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) @@ -524,7 +524,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -585,7 +585,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) diff --git a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py index ffb877cfd0f6..ed0b1de6313d 100644 --- a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py +++ b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py @@ -393,17 +393,18 @@ def encode_prompt( f" {self.tokenizer.model_max_length} tokens: {removed_text}" ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = text_inputs.attention_mask.to(device) + attention_mask = text_inputs.attention_mask.to(model_device) else: attention_mask = None if clip_skip is None: - prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), attention_mask=attention_mask) prompt_embeds = prompt_embeds[0] else: prompt_embeds = self.text_encoder( - text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + text_input_ids.to(model_device), attention_mask=attention_mask, output_hidden_states=True ) # Access the `hidden_states` first, that contains a tuple of # all the hidden states from the encoder layers. Then index into @@ -463,13 +464,14 @@ def encode_prompt( return_tensors="pt", ) + model_device = self.text_encoder.device if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: - attention_mask = uncond_input.attention_mask.to(device) + attention_mask = uncond_input.attention_mask.to(model_device) else: attention_mask = None negative_prompt_embeds = self.text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(model_device), attention_mask=attention_mask, ) negative_prompt_embeds = negative_prompt_embeds[0] @@ -499,7 +501,9 @@ def run_safety_checker(self, image, device, dtype): feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") else: feature_extractor_input = self.image_processor.numpy_to_pil(image) - safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to( + self.safety_checker.device + ) image, has_nsfw_concept = self.safety_checker( images=image, clip_input=safety_checker_input.pixel_values.to(dtype) ) diff --git a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py index a6dd07847de2..77ec7cb047d3 100644 --- a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py +++ b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py @@ -430,7 +430,7 @@ def encode_prompt( f" {tokenizer.model_max_length} tokens: {removed_text}" ) - prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + prompt_embeds = text_encoder(text_input_ids.to(text_encoder.device), output_hidden_states=True) # We are only ALWAYS interested in the pooled output of the final text encoder if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2: @@ -491,7 +491,7 @@ def encode_prompt( ) negative_prompt_embeds = text_encoder( - uncond_input.input_ids.to(device), + uncond_input.input_ids.to(text_encoder.device), output_hidden_states=True, ) @@ -553,7 +553,7 @@ def encode_image(self, image, device, num_images_per_prompt, output_hidden_state if not isinstance(image, torch.Tensor): image = self.feature_extractor(image, return_tensors="pt").pixel_values - image = image.to(device=device, dtype=dtype) + image = image.to(device=self.image_encoder.device, dtype=dtype) if output_hidden_states: image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) diff --git a/src/diffusers/pipelines/visualcloze/pipeline_visualcloze_generation.py b/src/diffusers/pipelines/visualcloze/pipeline_visualcloze_generation.py index ed2cd519df25..5940d591166d 100644 --- a/src/diffusers/pipelines/visualcloze/pipeline_visualcloze_generation.py +++ b/src/diffusers/pipelines/visualcloze/pipeline_visualcloze_generation.py @@ -228,7 +228,8 @@ def _get_t5_prompt_embeds( f" {max_sequence_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0] + model_device = self.text_encoder_2.device + prompt_embeds = self.text_encoder_2(text_input_ids.to(model_device), output_hidden_states=False)[0] dtype = self.text_encoder_2.dtype prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) @@ -274,7 +275,8 @@ def _get_clip_prompt_embeds( "The following part of your input was truncated because CLIP can only handle sequences up to" f" {self.tokenizer_max_length} tokens: {removed_text}" ) - prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False) + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), output_hidden_states=False) # Use pooled output of CLIPTextModel prompt_embeds = prompt_embeds.pooler_output diff --git a/src/diffusers/pipelines/wan/pipeline_wan.py b/src/diffusers/pipelines/wan/pipeline_wan.py index be2d53f17932..73296b1d0b85 100644 --- a/src/diffusers/pipelines/wan/pipeline_wan.py +++ b/src/diffusers/pipelines/wan/pipeline_wan.py @@ -182,7 +182,8 @@ def _get_t5_prompt_embeds( text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask seq_lens = mask.gt(0).sum(dim=1).long() - prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), mask.to(model_device)).last_hidden_state prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] prompt_embeds = torch.stack( @@ -654,7 +655,7 @@ def __call__( self._current_timestep = None if not output_type == "latent": - latents = latents.to(self.vae.dtype) + latents = latents.to(self.vae.device, dtype=self.vae.dtype) latents_mean = ( torch.tensor(self.vae.config.latents_mean) .view(1, self.vae.config.z_dim, 1, 1, 1) @@ -664,6 +665,7 @@ def __call__( latents.device, latents.dtype ) latents = latents / latents_std + latents_mean + latents = latents.to(self.vae.device) video = self.vae.decode(latents, return_dict=False)[0] video = self.video_processor.postprocess_video(video, output_type=output_type) else: diff --git a/src/diffusers/pipelines/wan/pipeline_wan_animate.py b/src/diffusers/pipelines/wan/pipeline_wan_animate.py index 5806032c0142..baa270f10760 100644 --- a/src/diffusers/pipelines/wan/pipeline_wan_animate.py +++ b/src/diffusers/pipelines/wan/pipeline_wan_animate.py @@ -259,7 +259,8 @@ def _get_t5_prompt_embeds( text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask seq_lens = mask.gt(0).sum(dim=1).long() - prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), mask.to(model_device)).last_hidden_state prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] prompt_embeds = torch.stack( @@ -280,9 +281,9 @@ def encode_image( device: torch.device | None = None, ): device = device or self._execution_device - image = self.image_processor(images=image, return_tensors="pt").to(device) + image = self.image_processor(images=image, return_tensors="pt").to(self.image_encoder.device) image_embeds = self.image_encoder(**image, output_hidden_states=True) - return image_embeds.hidden_states[-2] + return image_embeds.hidden_states[-2].to(device) # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline.encode_prompt def encode_prompt( diff --git a/src/diffusers/pipelines/wan/pipeline_wan_i2v.py b/src/diffusers/pipelines/wan/pipeline_wan_i2v.py index 8061f67ab6b9..2959a3acf3c4 100644 --- a/src/diffusers/pipelines/wan/pipeline_wan_i2v.py +++ b/src/diffusers/pipelines/wan/pipeline_wan_i2v.py @@ -223,7 +223,8 @@ def _get_t5_prompt_embeds( text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask seq_lens = mask.gt(0).sum(dim=1).long() - prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), mask.to(model_device)).last_hidden_state prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] prompt_embeds = torch.stack( @@ -243,9 +244,9 @@ def encode_image( device: torch.device | None = None, ): device = device or self._execution_device - image = self.image_processor(images=image, return_tensors="pt").to(device) + image = self.image_processor(images=image, return_tensors="pt").to(self.image_encoder.device) image_embeds = self.image_encoder(**image, output_hidden_states=True) - return image_embeds.hidden_states[-2] + return image_embeds.hidden_states[-2].to(device) # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline.encode_prompt def encode_prompt( @@ -435,7 +436,7 @@ def prepare_latents( [image, image.new_zeros(image.shape[0], image.shape[1], num_frames - 2, height, width), last_image], dim=2, ) - video_condition = video_condition.to(device=device, dtype=self.vae.dtype) + video_condition = video_condition.to(device=self.vae.device, dtype=self.vae.dtype) latents_mean = ( torch.tensor(self.vae.config.latents_mean) @@ -455,7 +456,7 @@ def prepare_latents( latent_condition = retrieve_latents(self.vae.encode(video_condition), sample_mode="argmax") latent_condition = latent_condition.repeat(batch_size, 1, 1, 1, 1) - latent_condition = latent_condition.to(dtype) + latent_condition = latent_condition.to(device=device, dtype=dtype) latent_condition = (latent_condition - latents_mean) * latents_std if self.config.expand_timesteps: @@ -817,7 +818,7 @@ def __call__( latents = (1 - first_frame_mask) * condition + first_frame_mask * latents if not output_type == "latent": - latents = latents.to(self.vae.dtype) + latents = latents.to(device=self.vae.device, dtype=self.vae.dtype) latents_mean = ( torch.tensor(self.vae.config.latents_mean) .view(1, self.vae.config.z_dim, 1, 1, 1) diff --git a/src/diffusers/pipelines/wan/pipeline_wan_vace.py b/src/diffusers/pipelines/wan/pipeline_wan_vace.py index b0896d382d67..8c72adf09d6a 100644 --- a/src/diffusers/pipelines/wan/pipeline_wan_vace.py +++ b/src/diffusers/pipelines/wan/pipeline_wan_vace.py @@ -228,7 +228,8 @@ def _get_t5_prompt_embeds( text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask seq_lens = mask.gt(0).sum(dim=1).long() - prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), mask.to(model_device)).last_hidden_state prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] prompt_embeds = torch.stack( diff --git a/src/diffusers/pipelines/wan/pipeline_wan_video2video.py b/src/diffusers/pipelines/wan/pipeline_wan_video2video.py index 7780fc712227..f64f334c039e 100644 --- a/src/diffusers/pipelines/wan/pipeline_wan_video2video.py +++ b/src/diffusers/pipelines/wan/pipeline_wan_video2video.py @@ -246,7 +246,8 @@ def _get_t5_prompt_embeds( text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask seq_lens = mask.gt(0).sum(dim=1).long() - prompt_embeds = self.text_encoder(text_input_ids.to(device), mask.to(device)).last_hidden_state + model_device = self.text_encoder.device + prompt_embeds = self.text_encoder(text_input_ids.to(model_device), mask.to(model_device)).last_hidden_state prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] prompt_embeds = torch.stack(