[Modular] Qwen

docs/source/en/api/image_processor.md

@@ -20,6 +20,12 @@ All pipelines with [`VaeImageProcessor`] accept PIL Image, PyTorch tensor, or Nu
 
 [[autodoc]] image_processor.VaeImageProcessor
 
+## InpaintProcessor
+
+The [`InpaintProcessor`] accepts `mask` and `image` inputs and process them together. Optionally, it can accept padding_mask_crop and apply mask overlay.
+
+[[autodoc]] image_processor.InpaintProcessor
+
 ## VaeImageProcessorLDM3D
 
 The [`VaeImageProcessorLDM3D`] accepts RGB and depth inputs and returns RGB and depth outputs.

src/diffusers/__init__.py

@@ -372,6 +372,10 @@
  [
  "FluxAutoBlocks",
  "FluxModularPipeline",
+ "QwenImageAutoBlocks",
+ "QwenImageEditAutoBlocks",
+ "QwenImageEditModularPipeline",
+ "QwenImageModularPipeline",
  "StableDiffusionXLAutoBlocks",
  "StableDiffusionXLModularPipeline",
  "WanAutoBlocks",
@@ -1017,6 +1021,10 @@
  from .modular_pipelines import (
  FluxAutoBlocks,
  FluxModularPipeline,
+ QwenImageAutoBlocks,
+ QwenImageEditAutoBlocks,
+ QwenImageEditModularPipeline,
+ QwenImageModularPipeline,
  StableDiffusionXLAutoBlocks,
  StableDiffusionXLModularPipeline,
  WanAutoBlocks,

src/diffusers/hooks/_helpers.py

@@ -108,6 +108,7 @@ def _register_attention_processors_metadata():
  from ..models.attention_processor import AttnProcessor2_0
  from ..models.transformers.transformer_cogview4 import CogView4AttnProcessor
  from ..models.transformers.transformer_flux import FluxAttnProcessor
+ from ..models.transformers.transformer_qwenimage import QwenDoubleStreamAttnProcessor2_0
  from ..models.transformers.transformer_wan import WanAttnProcessor2_0
 
  # AttnProcessor2_0
@@ -140,6 +141,14 @@ def _register_attention_processors_metadata():
  metadata=AttentionProcessorMetadata(skip_processor_output_fn=_skip_proc_output_fn_Attention_FluxAttnProcessor),
  )
 
+ # QwenDoubleStreamAttnProcessor2
+ AttentionProcessorRegistry.register(
+ model_class=QwenDoubleStreamAttnProcessor2_0,
+ metadata=AttentionProcessorMetadata(
+ skip_processor_output_fn=_skip_proc_output_fn_Attention_QwenDoubleStreamAttnProcessor2_0
+ ),
+ )
+
 
 def _register_transformer_blocks_metadata():
  from ..models.attention import BasicTransformerBlock
@@ -298,4 +307,5 @@ def _skip_attention___ret___hidden_states___encoder_hidden_states(self, *args, *
 _skip_proc_output_fn_Attention_WanAttnProcessor2_0 = _skip_attention___ret___hidden_states
 # not sure what this is yet.
 _skip_proc_output_fn_Attention_FluxAttnProcessor = _skip_attention___ret___hidden_states
+_skip_proc_output_fn_Attention_QwenDoubleStreamAttnProcessor2_0 = _skip_attention___ret___hidden_states
 # fmt: on

src/diffusers/image_processor.py

@@ -523,6 +523,7 @@ def resize(
  size=(height, width),
  )
  image = self.pt_to_numpy(image)
+
  return image
 
  def binarize(self, image: PIL.Image.Image) -> PIL.Image.Image:
@@ -838,6 +839,137 @@ def apply_overlay(
  return image
 
 
+class InpaintProcessor(ConfigMixin):
+ """
+ Image processor for inpainting image and mask.
+ """
+
+ config_name = CONFIG_NAME
+
+ @register_to_config
+ def __init__(
+ self,
+ do_resize: bool = True,
+ vae_scale_factor: int = 8,
+ vae_latent_channels: int = 4,
+ resample: str = "lanczos",
+ reducing_gap: int = None,
+ do_normalize: bool = True,
+ do_binarize: bool = False,
+ do_convert_grayscale: bool = False,
+ mask_do_normalize: bool = False,
+ mask_do_binarize: bool = True,
+ mask_do_convert_grayscale: bool = True,
+ ):
+ super().__init__()
+
+ self._image_processor = VaeImageProcessor(
+ do_resize=do_resize,
+ vae_scale_factor=vae_scale_factor,
+ vae_latent_channels=vae_latent_channels,
+ resample=resample,
+ reducing_gap=reducing_gap,
+ do_normalize=do_normalize,
+ do_binarize=do_binarize,
+ do_convert_grayscale=do_convert_grayscale,
+ )
+ self._mask_processor = VaeImageProcessor(
+ do_resize=do_resize,
+ vae_scale_factor=vae_scale_factor,
+ vae_latent_channels=vae_latent_channels,
+ resample=resample,
+ reducing_gap=reducing_gap,
+ do_normalize=mask_do_normalize,
+ do_binarize=mask_do_binarize,
+ do_convert_grayscale=mask_do_convert_grayscale,
+ )
+
+ def preprocess(
+ self,
+ image: PIL.Image.Image,
+ mask: PIL.Image.Image = None,
+ height: int = None,
+ width: int = None,
+ padding_mask_crop: Optional[int] = None,
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ """
+ Preprocess the image and mask.
+ """
+ if mask is None and padding_mask_crop is not None:
+ raise ValueError("mask must be provided if padding_mask_crop is provided")
+
+ # if mask is None, same behavior as regular image processor
+ if mask is None:
+ return self._image_processor.preprocess(image, height=height, width=width)
+
+ if padding_mask_crop is not None:
+ crops_coords = self._image_processor.get_crop_region(mask, width, height, pad=padding_mask_crop)
+ resize_mode = "fill"
+ else:
+ crops_coords = None
+ resize_mode = "default"
+
+ processed_image = self._image_processor.preprocess(
+ image,
+ height=height,
+ width=width,
+ crops_coords=crops_coords,
+ resize_mode=resize_mode,
+ )
+
+ processed_mask = self._mask_processor.preprocess(
+ mask,
+ height=height,
+ width=width,
+ resize_mode=resize_mode,
+ crops_coords=crops_coords,
+ )
+
+ if crops_coords is not None:
+ postprocessing_kwargs = {
+ "crops_coords": crops_coords,
+ "original_image": image,
+ "original_mask": mask,
+ }
+ else:
+ postprocessing_kwargs = {
+ "crops_coords": None,
+ "original_image": None,
+ "original_mask": None,
+ }
+
+ return processed_image, processed_mask, postprocessing_kwargs
+
+ def postprocess(
+ self,
+ image: torch.Tensor,
+ output_type: str = "pil",
+ original_image: Optional[PIL.Image.Image] = None,
+ original_mask: Optional[PIL.Image.Image] = None,
+ crops_coords: Optional[Tuple[int, int, int, int]] = None,
+ ) -> Tuple[PIL.Image.Image, PIL.Image.Image]:
+ """
+ Postprocess the image, optionally apply mask overlay
+ """
+ image = self._image_processor.postprocess(
+ image,
+ output_type=output_type,
+ )
+ # optionally apply the mask overlay
+ if crops_coords is not None and (original_image is None or original_mask is None):
+ raise ValueError("original_image and original_mask must be provided if crops_coords is provided")
+
+ elif crops_coords is not None and output_type != "pil":
+ raise ValueError("output_type must be 'pil' if crops_coords is provided")
+
+ elif crops_coords is not None:
+ image = [
+ self._image_processor.apply_overlay(original_mask, original_image, i, crops_coords) for i in image
+ ]
+
+ return image
+
+
 class VaeImageProcessorLDM3D(VaeImageProcessor):
  """
  Image processor for VAE LDM3D.

src/diffusers/modular_pipelines/__init__.py

@@ -47,6 +47,12 @@
  _import_structure["stable_diffusion_xl"] = ["StableDiffusionXLAutoBlocks", "StableDiffusionXLModularPipeline"]
  _import_structure["wan"] = ["WanAutoBlocks", "WanModularPipeline"]
  _import_structure["flux"] = ["FluxAutoBlocks", "FluxModularPipeline"]
+ _import_structure["qwenimage"] = [
+ "QwenImageAutoBlocks",
+ "QwenImageModularPipeline",
+ "QwenImageEditModularPipeline",
+ "QwenImageEditAutoBlocks",
+ ]
  _import_structure["components_manager"] = ["ComponentsManager"]
 
 if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT:
@@ -68,6 +74,12 @@
  SequentialPipelineBlocks,
  )
  from .modular_pipeline_utils import ComponentSpec, ConfigSpec, InputParam, InsertableDict, OutputParam
+ from .qwenimage import (
+ QwenImageAutoBlocks,
+ QwenImageEditAutoBlocks,
+ QwenImageEditModularPipeline,
+ QwenImageModularPipeline,
+ )
  from .stable_diffusion_xl import StableDiffusionXLAutoBlocks, StableDiffusionXLModularPipeline
  from .wan import WanAutoBlocks, WanModularPipeline
 else:

src/diffusers/modular_pipelines/modular_pipeline.py

@@ -56,6 +56,8 @@
  ("stable-diffusion-xl", "StableDiffusionXLModularPipeline"),
  ("wan", "WanModularPipeline"),
  ("flux", "FluxModularPipeline"),
+ ("qwenimage", "QwenImageModularPipeline"),
+ ("qwenimage-edit", "QwenImageEditModularPipeline"),
  ]
 )
 
@@ -64,6 +66,8 @@
  ("StableDiffusionXLModularPipeline", "StableDiffusionXLAutoBlocks"),
  ("WanModularPipeline", "WanAutoBlocks"),
  ("FluxModularPipeline", "FluxAutoBlocks"),
+ ("QwenImageModularPipeline", "QwenImageAutoBlocks"),
+ ("QwenImageEditModularPipeline", "QwenImageEditAutoBlocks"),
  ]
 )
 
@@ -133,8 +137,8 @@ def __getattr__(self, name):
  Allow attribute access to intermediate values. If an attribute is not found in the object, look for it in the
  intermediates dict.
  """
- if name in self.intermediates:
- return self.intermediates[name]
+ if name in self.values:
+ return self.values[name]
  raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{name}'")
 
  def __repr__(self):
@@ -548,8 +552,11 @@ class AutoPipelineBlocks(ModularPipelineBlocks):
 
  def __init__(self):
  sub_blocks = InsertableDict()
- for block_name, block_cls in zip(self.block_names, self.block_classes):
- sub_blocks[block_name] = block_cls()
+ for block_name, block in zip(self.block_names, self.block_classes):
+ if inspect.isclass(block):
+ sub_blocks[block_name] = block()
+ else:
+ sub_blocks[block_name] = block
  self.sub_blocks = sub_blocks
  if not (len(self.block_classes) == len(self.block_names) == len(self.block_trigger_inputs)):
  raise ValueError(
@@ -830,7 +837,9 @@ def expected_configs(self):
  return expected_configs
 
  @classmethod
- def from_blocks_dict(cls, blocks_dict: Dict[str, Any]) -> "SequentialPipelineBlocks":
+ def from_blocks_dict(
+ cls, blocks_dict: Dict[str, Any], description: Optional[str] = None
+ ) -> "SequentialPipelineBlocks":
  """Creates a SequentialPipelineBlocks instance from a dictionary of blocks.
 
  Args:
@@ -852,12 +861,19 @@ def from_blocks_dict(cls, blocks_dict: Dict[str, Any]) -> "SequentialPipelineBlo
  instance.block_classes = [block.__class__ for block in sub_blocks.values()]
  instance.block_names = list(sub_blocks.keys())
  instance.sub_blocks = sub_blocks
+
+ if description is not None:
+ instance.description = description
+
  return instance
 
  def __init__(self):
  sub_blocks = InsertableDict()
- for block_name, block_cls in zip(self.block_names, self.block_classes):
- sub_blocks[block_name] = block_cls()
+ for block_name, block in zip(self.block_names, self.block_classes):
+ if inspect.isclass(block):
+ sub_blocks[block_name] = block()
+ else:
+ sub_blocks[block_name] = block
  self.sub_blocks = sub_blocks
 
  def _get_inputs(self):
@@ -1280,8 +1296,11 @@ def outputs(self) -> List[str]:
 
  def __init__(self):
  sub_blocks = InsertableDict()
- for block_name, block_cls in zip(self.block_names, self.block_classes):
- sub_blocks[block_name] = block_cls()
+ for block_name, block in zip(self.block_names, self.block_classes):
+ if inspect.isclass(block):
+ sub_blocks[block_name] = block()
+ else:
+ sub_blocks[block_name] = block
  self.sub_blocks = sub_blocks
 
  @classmethod