add flux inpaint + img2img + controlnet to auto pipeline

[yiyixuxu]

this PR

1. add the flux inpaint + img2img + controlnet to autopipeline
2. refactored AutoPipelineforInpaint.from_pretrained a little bit so it is clearer, and will work with controlnet when we have a controlnet +pag+inpaint pipeline available (Currently, we don't have one yet!). this is a follow-up for Fix StableDiffusionXLPAGInpaintPipeline #9128
3. similarly, made AutoPipelineForImage2Image work with the refiner checkpoint
4. add a little bit more test (mostly add a test to test AutoPipelineforInpaint.from_pretrained from inpaint-specific checkpoint, the only meaningful use case for that is to use with PAG) and also AutoPipelineforInpaint.from_pretrained from refiner checkpoint

testing script for everything I've added in this PR

fromdiffusersimportAutoPipelineForInpainting, AutoPipelineForImage2Image, AutoPipelineForText2Image, FluxControlNetModel, ControlNetModelimporttorchtorch.cuda.set_device(1)
# flux inpaintpipe=AutoPipelineForInpainting.from_pretrained("black-forest-labs/FLUX.1-schnell")
assertpipe.__class__.__name__=="FluxInpaintPipeline"# flux img2imgpipe=AutoPipelineForImage2Image.from_pretrained("black-forest-labs/FLUX.1-schnell")
assertpipe.__class__.__name__=="FluxImg2ImgPipeline"# Flux ControlNetcontrolnet_model="InstantX/FLUX.1-dev-controlnet-canny"controlnet=FluxControlNetModel.from_pretrained(controlnet_model, torch_dtype=torch.bfloat16)
pipe=AutoPipelineForText2Image.from_pretrained("black-forest-labs/FLUX.1-dev", controlnet=controlnet, torch_dtype=torch.bfloat16)
assertpipe.__class__.__name__=="FluxControlNetPipeline"# AutoPipelineForInpaintinginpaint_checkpoint="diffusers/stable-diffusion-xl-1.0-inpainting-0.1"base_checkpoint="stabilityai/stable-diffusion-xl-base-1.0"pipe=AutoPipelineForInpainting.from_pretrained(inpaint_checkpoint, enable_pag=True,torch_dtype=torch.float16)
assertpipe.__class__.__name__=="StableDiffusionXLPAGInpaintPipeline"pipe=AutoPipelineForInpainting.from_pretrained(base_checkpoint, enable_pag=True,torch_dtype=torch.float16)
assertpipe.__class__.__name__=="StableDiffusionXLPAGInpaintPipeline"pipe=AutoPipelineForInpainting.from_pretrained(inpaint_checkpoint, torch_dtype=torch.float16)
assertpipe.__class__.__name__=="StableDiffusionXLInpaintPipeline"pipe=AutoPipelineForInpainting.from_pretrained(base_checkpoint, torch_dtype=torch.float16)
assertpipe.__class__.__name__=="StableDiffusionXLInpaintPipeline"# test AutoPipelineForImage2Image.from_pretrained, make sure it can load both text2img checkpoint and img2img checkpoint (refiner)controlnet=ControlNetModel.from_pretrained(
"diffusers/controlnet-depth-sdxl-1.0-small",
variant="fp16",
torch_dtype=torch.float16,
)
# -> img2img + pagrefiner_checkpoint="stabilityai/stable-diffusion-xl-refiner-1.0"base_checkpoint="stabilityai/stable-diffusion-xl-base-1.0"## refiner checkpoint (img2img) -> img2img + pagpipe=AutoPipelineForImage2Image.from_pretrained(
refiner_checkpoint,
torch_dtype=torch.float16,
enable_pag=True,
)
assertpipe.__class__.__name__=="StableDiffusionXLPAGImg2ImgPipeline"## base checkpoint (text2img) -> img2img + pagpipe=AutoPipelineForImage2Image.from_pretrained(
base_checkpoint,
torch_dtype=torch.float16,
enable_pag=True,
)
assertpipe.__class__.__name__=="StableDiffusionXLPAGImg2ImgPipeline"# -> img2img + controlne## refiner checkpoint (img2img) -> img2img + controlnetpipe=AutoPipelineForImage2Image.from_pretrained(
refiner_checkpoint,
controlnet=controlnet,
torch_dtype=torch.float16,
)
assertpipe.__class__.__name__=="StableDiffusionXLControlNetImg2ImgPipeline"## base checkpoint (text2img) -> img2img + controlnetpipe=AutoPipelineForImage2Image.from_pretrained(
base_checkpoint,
controlnet=controlnet,
torch_dtype=torch.float16,
)
assertpipe.__class__.__name__=="StableDiffusionXLControlNetImg2ImgPipeline"# -> img2img + controlnet + pag## refiner checkpoint (img2img) -> img2img + controlnet + pagpipe=AutoPipelineForImage2Image.from_pretrained(
refiner_checkpoint,
controlnet=controlnet,
torch_dtype=torch.float16,
enable_pag=True,
)
assertpipe.__class__.__name__=="StableDiffusionXLControlNetPAGImg2ImgPipeline"## base checkpoint (text2img) -> img2img + controlnet + pagpipe=AutoPipelineForImage2Image.from_pretrained(
base_checkpoint,
controlnet=controlnet,
torch_dtype=torch.float16,
enable_pag=True,
)
assertpipe.__class__.__name__=="StableDiffusionXLControlNetPAGImg2ImgPipeline"

[HuggingFaceDocBuilderDev]

The docs for this PR live here. All of your documentation changes will be reflected on that endpoint. The docs are available until 30 days after the last update.