diff --git a/dream-variations.py b/dream-variations.py new file mode 100644 index 00000000000..c5cb850db30 --- /dev/null +++ b/dream-variations.py @@ -0,0 +1,82 @@ +from ldm.simplet2i import T2I +import transformers +import numpy as np +import torch +from pytorch_lightning import seed_everything +import random + +t2i = T2I( + latent_diffusion_weights=False, + config = "configs/stable-diffusion/v1-inference.yaml" +) + +def slerp(t, v0, v1, DOT_THRESHOLD=0.9995): + """ helper function to spherically interpolate two arrays v1 v2 """ + + if not isinstance(v0, np.ndarray): + inputs_are_torch = True + input_device = v0.device + v0 = v0.cpu().numpy() + v1 = v1.cpu().numpy() + + dot = np.sum(v0 * v1 / (np.linalg.norm(v0) * np.linalg.norm(v1))) + if np.abs(dot) > DOT_THRESHOLD: + v2 = (1 - t) * v0 + t * v1 + else: + theta_0 = np.arccos(dot) + sin_theta_0 = np.sin(theta_0) + theta_t = theta_0 * t + sin_theta_t = np.sin(theta_t) + s0 = np.sin(theta_0 - theta_t) / sin_theta_0 + s1 = sin_theta_t / sin_theta_0 + v2 = s0 * v0 + s1 * v1 + + if inputs_are_torch: + v2 = torch.from_numpy(v2).to(input_device) + + return v2 + + +transformers.logging.set_verbosity_error() +seed = 3516972428 +seed_everything(seed) +init_code = torch.randn([t2i.batch_size, + t2i.latent_channels, + t2i.height // t2i.downsampling_factor, + t2i.width // t2i.downsampling_factor], + device=t2i.device) + + +print("loading model...") +t2i.load_model() + +##### interpolate +# noise = torch.randn([t2i.batch_size, +# t2i.latent_channels, +# t2i.height // t2i.downsampling_factor, +# t2i.width // t2i.downsampling_factor], +# device=t2i.device) +# for i in range(20): +# print("running generation " + str(i)) +# code = slerp(i / 20., init_code, noise) +# outputs = t2i.txt2img("elf queen with rainbow hair, golden hour. colored pencil drawing by rossdraws andrei riabovitchev trending on artstation", start_code=code, seed=seed) + +# generate variants +strength = 0.10 +prompt = "elf queen with rainbow hair, golden hour. colored pencil drawing by rossdraws andrei riabovitchev trending on artstation" + +print("generating base image") +t2i.txt2img(prompt, start_code=init_code, seed=seed) +for i in range(20): + random.seed() # reset RNG to an actually random state, so we can get a random seed + iter_seed = random.randrange(0,np.iinfo(np.uint32).max) + print("iteration " + str(i) + " running, seed = " + str(iter_seed)) + seed_everything(iter_seed) + noise = torch.randn([t2i.batch_size, + t2i.latent_channels, + t2i.height // t2i.downsampling_factor, + t2i.width // t2i.downsampling_factor], + device=t2i.device) + code = slerp(strength, init_code, noise) + seed_everything(iter_seed) + t2i.txt2img(prompt, start_code=code, seed=iter_seed) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 4737d90ba70..b16f54ab189 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -29,7 +29,7 @@ # do the slow model initialization t2i.load_model() -# Do the fast inference & image generation. Any options passed here +# Do the fast inference & image generation. Any options passed here # override the default values assigned during class initialization # Will call load_model() if the model was not previously loaded. # The method returns a list of images. Each row of the list is a sub-list of [filename,seed] @@ -45,7 +45,7 @@ results = t2i.img2img(prompt = "an astronaut riding a horse" outdir = "./outputs/img2img-samples" init_img = "./sketches/horse+rider.png") - + for row in results: print(f'filename={row[0]}') print(f'seed ={row[1]}') @@ -158,7 +158,7 @@ def __init__(self, def txt2img(self,prompt,outdir=None,batch_size=None,iterations=None, steps=None,seed=None,grid=None,individual=None,width=None,height=None, cfg_scale=None,ddim_eta=None,strength=None,embedding_path=None,init_img=None, - skip_normalize=False,variants=None): # note the "variants" option is an unused hack caused by how options are passed + skip_normalize=False,variants=None,start_code=None): # note the "variants" option is an unused hack caused by how options are passed """ Generate an image from the prompt, writing iteration images into the outdir The output is a list of lists in the format: [[filename1,seed1], [filename2,seed2],...] @@ -186,19 +186,19 @@ def txt2img(self,prompt,outdir=None,batch_size=None,iterations=None, grid = self.grid if individual: grid = False - + data = [batch_size * [prompt]] # make directories and establish names for the output files os.makedirs(outdir, exist_ok=True) - start_code = None - if self.fixed_code: - start_code = torch.randn([batch_size, - self.latent_channels, - height // self.downsampling_factor, - width // self.downsampling_factor], - device=self.device) + + # if self.fixed_code: + # start_code = torch.randn([batch_size, + # self.latent_channels, + # height // self.downsampling_factor, + # width // self.downsampling_factor], + # device=self.device) precision_scope = autocast if self.precision=="autocast" else nullcontext sampler = self.sampler @@ -281,7 +281,7 @@ def txt2img(self,prompt,outdir=None,batch_size=None,iterations=None, print(f'{image_count} images generated in',"%4.2fs"% (toc-tic)) return images - + # There is lots of shared code between this and txt2img and should be refactored. @torch.no_grad() def img2img(self,prompt,outdir=None,init_img=None,batch_size=None,iterations=None, @@ -319,7 +319,7 @@ def img2img(self,prompt,outdir=None,init_img=None,batch_size=None,iterations=Non grid = self.grid if individual: grid = False - + data = [batch_size * [prompt]] # PLMS sampler not supported yet, so ignore previous sampler @@ -345,7 +345,7 @@ def img2img(self,prompt,outdir=None,init_img=None,batch_size=None,iterations=Non except AssertionError: print(f"strength must be between 0.0 and 1.0, but received value {strength}") return [] - + t_enc = int(strength * steps) print(f"target t_enc is {t_enc} steps") @@ -485,7 +485,7 @@ def load_model(self): print(msg) return self.model - + def _load_model_from_config(self, config, ckpt): print(f"Loading model from {ckpt}") pl_sd = torch.load(ckpt, map_location="cpu") @@ -530,7 +530,7 @@ def _unique_filename(self,outdir,previousname=None,seed=0,isbatch=False,grid_cou filename = f'{basecount:06}.{seed}.01.png' else: filename = f'{basecount:06}.{seed}.png' - + return os.path.join(outdir,filename) else: @@ -540,7 +540,7 @@ def _unique_filename(self,outdir,previousname=None,seed=0,isbatch=False,grid_cou return self._unique_filename(outdir,previousname,seed) basecount = int(x.groups()[0]) - series = 0 + series = 0 finished = False while not finished: series += 1 @@ -552,7 +552,7 @@ def _unique_filename(self,outdir,previousname=None,seed=0,isbatch=False,grid_cou def _split_weighted_subprompts(text): """ - grabs all text up to the first occurrence of ':' + grabs all text up to the first occurrence of ':' uses the grabbed text as a sub-prompt, and takes the value following ':' as weight if ':' has no value defined, defaults to 1.0 repeats until no text remaining @@ -568,7 +568,7 @@ def _split_weighted_subprompts(text): remaining -= idx # remove from main text text = text[idx+1:] - # find value for weight + # find value for weight if " " in text: idx = text.index(" ") # first occurence else: # no space, read to end