diff --git a/ldm/models/diffusion/ddim.py b/ldm/models/diffusion/ddim.py index 065b32986a5..ddf786b5a85 100644 --- a/ldm/models/diffusion/ddim.py +++ b/ldm/models/diffusion/ddim.py @@ -10,16 +10,17 @@ class DDIMSampler(object): - def __init__(self, model, schedule="linear", **kwargs): + def __init__(self, model, schedule="linear", device="cuda", **kwargs): super().__init__() self.model = model self.ddpm_num_timesteps = model.num_timesteps self.schedule = schedule + self.device = device def register_buffer(self, name, attr): if type(attr) == torch.Tensor: - if attr.device != torch.device("cuda"): - attr = attr.to(torch.device("cuda")) + if attr.device != torch.device(self.device): + attr = attr.to(torch.device(self.device)) setattr(self, name, attr) def make_schedule(self, ddim_num_steps, ddim_discretize="uniform", ddim_eta=0., verbose=True): diff --git a/ldm/models/diffusion/plms.py b/ldm/models/diffusion/plms.py index 5d09f023f3b..5eafe1d7ce5 100644 --- a/ldm/models/diffusion/plms.py +++ b/ldm/models/diffusion/plms.py @@ -9,13 +9,18 @@ class PLMSSampler(object): - def __init__(self, model, schedule="linear", **kwargs): + def __init__(self, model, schedule="linear", device="cuda", **kwargs): super().__init__() self.model = model self.ddpm_num_timesteps = model.num_timesteps self.schedule = schedule + self.device = device def register_buffer(self, name, attr): + if type(attr) == torch.Tensor: + if attr.device != torch.device(self.device): + attr = attr.to(torch.device(self.device)) + setattr(self, name, attr) def make_schedule(self, ddim_num_steps, ddim_discretize="uniform", ddim_eta=0., verbose=True): diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 0691cccb7ba..be6c05bdc95 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -58,7 +58,6 @@ import os from omegaconf import OmegaConf from PIL import Image -import PIL from tqdm import tqdm, trange from itertools import islice from einops import rearrange, repeat @@ -158,7 +157,8 @@ def __init__(self, @torch.no_grad() def txt2img(self,prompt,outdir=None,batch_size=None,iterations=None, steps=None,seed=None,grid=None,individual=None,width=None,height=None, - cfg_scale=None,ddim_eta=None,strength=None,embedding_path=None,init_img=None,skip_normalize=False): + cfg_scale=None,ddim_eta=None,strength=None,embedding_path=None,init_img=None, + skip_normalize=False,variants=None): """ Generate an image from the prompt, writing iteration images into the outdir The output is a list of lists in the format: [[filename1,seed1], [filename2,seed2],...] @@ -286,7 +286,8 @@ def txt2img(self,prompt,outdir=None,batch_size=None,iterations=None, @torch.no_grad() def img2img(self,prompt,outdir=None,init_img=None,batch_size=None,iterations=None, steps=None,seed=None,grid=None,individual=None,width=None,height=None, - cfg_scale=None,ddim_eta=None,strength=None,embedding_path=None,skip_normalize=False): + cfg_scale=None,ddim_eta=None,strength=None,embedding_path=None, + skip_normalize=False,variants=None): """ Generate an image from the prompt and the initial image, writing iteration images into the outdir The output is a list of lists in the format: [[filename1,seed1], [filename2,seed2],...] @@ -324,7 +325,7 @@ def img2img(self,prompt,outdir=None,init_img=None,batch_size=None,iterations=Non # PLMS sampler not supported yet, so ignore previous sampler if self.sampler_name!='ddim': print(f"sampler '{self.sampler_name}' is not yet supported. Using DDM sampler") - sampler = DDIMSampler(model) + sampler = DDIMSampler(model, device=self.device) else: sampler = self.sampler @@ -462,9 +463,9 @@ def load_model(self): msg = f'setting sampler to {self.sampler_name}' if self.sampler_name=='plms': - self.sampler = PLMSSampler(self.model) + self.sampler = PLMSSampler(self.model, device=self.device) elif self.sampler_name == 'ddim': - self.sampler = DDIMSampler(self.model) + self.sampler = DDIMSampler(self.model, device=self.device) elif self.sampler_name == 'k_dpm_2_a': self.sampler = KSampler(self.model,'dpm_2_ancestral') elif self.sampler_name == 'k_dpm_2': @@ -479,7 +480,7 @@ def load_model(self): self.sampler = KSampler(self.model,'lms') else: msg = f'unsupported sampler {self.sampler_name}, defaulting to plms' - self.sampler = PLMSSampler(self.model) + self.sampler = PLMSSampler(self.model, device=self.device) print(msg) @@ -506,7 +507,7 @@ def _load_img(self,path): w, h = image.size print(f"loaded input image of size ({w}, {h}) from {path}") w, h = map(lambda x: x - x % 32, (w, h)) # resize to integer multiple of 32 - image = image.resize((w, h), resample=PIL.Image.LANCZOS) + image = image.resize((w, h), resample=Image.Resampling.LANCZOS) image = np.array(image).astype(np.float32) / 255.0 image = image[None].transpose(0, 3, 1, 2) image = torch.from_numpy(image) diff --git a/scripts/dream.py b/scripts/dream.py index 7ba62056531..211813c4c2e 100755 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -6,6 +6,7 @@ import atexit import os import sys +import copy from PIL import Image,PngImagePlugin # readline unavailable on windows systems @@ -175,9 +176,32 @@ def main_loop(t2i,parser,log,infile): print(e) continue + + allVariantResults = [] + if opt.variants is not None: + print(f"Generating {opt.variants} variant(s)...") + newopt = copy.deepcopy(opt) + newopt.variants = None + for r in results: + newopt.init_img = r[0] + print(f"\t generating variant for {newopt.init_img}") + for j in range(0, opt.variants): + try: + variantResults = t2i.img2img(**vars(newopt)) + allVariantResults.append([newopt,variantResults]) + except AssertionError as e: + print(e) + continue + print(f"{opt.variants} Variants generated!") + print("Outputs:") write_log_message(t2i,opt,results,log) + if allVariantResults: + print("Variant outputs:") + for vr in allVariantResults: + write_log_message(t2i,vr[0],vr[1],log) + print("goodbye!") @@ -307,6 +331,7 @@ def create_cmd_parser(): parser.add_argument('-i','--individual',action='store_true',help="generate individual files (default)") parser.add_argument('-I','--init_img',type=str,help="path to input image (supersedes width and height)") parser.add_argument('-f','--strength',default=0.75,type=float,help="strength for noising/unnoising. 0.0 preserves image exactly, 1.0 replaces it completely") + parser.add_argument('-v','--variants',type=int,help="number of variants to generate of each image") parser.add_argument('-x','--skip_normalize',action='store_true',help="skip subprompt weight normalization") return parser @@ -315,7 +340,7 @@ def setup_readline(): readline.set_completer(Completer(['cd','pwd', '--steps','-s','--seed','-S','--iterations','-n','--batch_size','-b', '--width','-W','--height','-H','--cfg_scale','-C','--grid','-g', - '--individual','-i','--init_img','-I','--strength','-f']).complete) + '--individual','-i','--init_img','-I','--strength','-f','-v','--variants']).complete) readline.set_completer_delims(" ") readline.parse_and_bind('tab: complete') load_history() diff --git a/src/k-diffusion b/src/k-diffusion index db579906874..ef1bf07627c 160000 --- a/src/k-diffusion +++ b/src/k-diffusion @@ -1 +1 @@ -Subproject commit db5799068749bf3a6d5845120ed32df16b7d883b +Subproject commit ef1bf07627c9a10ba9137e68a0206b844544a7d9