From 387a48c4ecf24a49999cfd592f16bed14b956c35 Mon Sep 17 00:00:00 2001 From: xra Date: Mon, 29 Aug 2022 23:12:16 +0900 Subject: [PATCH 01/11] seed fuzzing adds 2 parameters for generating variations of a seed: -z optional 0-1 value to slerp from -S noise to random noise (allows variations on an image) -Z optional target seed that -S noise is slerped to (interpolate one image to another) based on https://github.com/bakkot/stable-diffusion/tree/noise --- ldm/models/diffusion/ksampler.py | 2 +- ldm/simplet2i.py | 108 +++++++++++++++++++++++++++++++ scripts/dream.py | 14 ++++ 3 files changed, 123 insertions(+), 1 deletion(-) diff --git a/ldm/models/diffusion/ksampler.py b/ldm/models/diffusion/ksampler.py index 1da81eee5a7..5d2756fd973 100644 --- a/ldm/models/diffusion/ksampler.py +++ b/ldm/models/diffusion/ksampler.py @@ -63,7 +63,7 @@ def sample( ): sigmas = self.model.get_sigmas(S) - if x_T: + if x_T is not None: x = x_T else: x = ( diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index d2f10c4a817..671918bf169 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -212,6 +212,8 @@ def prompt2image( upscale=None, variants=None, sampler_name=None, + seed_fuzz=None, + seed_fuzz_target=None, **args, ): # eat up additional cruft """ @@ -231,6 +233,8 @@ def prompt2image( ddim_eta // image randomness (eta=0.0 means the same seed always produces the same image) variants // if >0, the 1st generated image will be passed back to img2img to generate the requested number of variants image_callback // a function or method that will be called each time an image is generated + seed_fuzz // optional 0-1 value to slerp from -S noise to random noise (allows variations on an image) + seed_fuzz_target // optional target seed that -S noise is slerped to (interpolate one image to another) To use the callback, define a function of method that receives two arguments, an Image object and the seed. You can then do whatever you like with the image, including converting it to @@ -304,6 +308,8 @@ def process_image(image,seed): skip_normalize=skip_normalize, width=width, height=height, + seed_fuzz=seed_fuzz, + seed_fuzz_target=seed_fuzz_target, ) with scope(self.device.type), self.model.ema_scope(): @@ -390,6 +396,8 @@ def _txt2img( skip_normalize, width, height, + seed_fuzz, + seed_fuzz_target, ): """ An infinite iterator of images from the prompt. @@ -397,6 +405,8 @@ def _txt2img( sampler = self.sampler + base_x_T, target_x_T = self._seed_fuzz(width, height, seed_fuzz, seed_fuzz_target) + while True: uc, c = self._get_uc_and_c(prompt, batch_size, skip_normalize) shape = [ @@ -404,6 +414,9 @@ def _txt2img( height // self.downsampling_factor, width // self.downsampling_factor, ] + + x_T = self._seed_fuzz_slerp(width, height, steps, seed_fuzz, seed_fuzz_target, base_x_T, target_x_T) + samples, _ = sampler.sample( S=steps, conditioning=c, @@ -413,6 +426,7 @@ def _txt2img( unconditional_guidance_scale=cfg_scale, unconditional_conditioning=uc, eta=ddim_eta, + x_T = x_T ) yield self._samples_to_images(samples) @@ -517,6 +531,54 @@ def _new_seed(self): self.seed = random.randrange(0, np.iinfo(np.uint32).max) return self.seed + def _seed_fuzz(self, width:int, height:int, seed_fuzz:float, seed_fuzz_target:int) -> "tuple[torch.Tensor,torch.Tensor]": + base_x_T = None + target_x_T = None + if seed_fuzz is not None: + seed_fuzz = max(0.0, min(1.0, seed_fuzz)) + # seed fuzz, base noise is made from seed provided with -S + base_x_T = torch.randn([self.batch_size, + self.latent_channels, + height // self.downsampling_factor, + width // self.downsampling_factor], + device=self.device) + if seed_fuzz_target is not None: + # has target seed, store initial seed + initialSeed = torch.initial_seed() + seed_everything(seed_fuzz_target) # seed with target + target_x_T = torch.randn([self.batch_size, + self.latent_channels, + height // self.downsampling_factor, + width // self.downsampling_factor], + device=self.device) + # back to our initialSeed (is this correct? it works...) + seed_everything(initialSeed) + return base_x_T, target_x_T + + def _seed_fuzz_slerp(self, + width:int, height:int, steps:int, + seed_fuzz:float, seed_fuzz_target:int, + base_x_T:torch.Tensor, target_x_T:torch.Tensor) -> torch.Tensor: + x_T = None + if seed_fuzz is not None: + seed_fuzz = max(0.0, min(1.0, seed_fuzz)) + # no target seed, get random noise + if seed_fuzz_target is None: + target_x_T = torch.randn([self.batch_size, + self.latent_channels, + height // self.downsampling_factor, + width // self.downsampling_factor], + device=self.device) + + # slerp base -> target using seed_fuzz amount + x_T = self.slerp(seed_fuzz, base_x_T, target_x_T) + + # only for ksampler! + if isinstance(self.sampler, KSampler): + # KSampler does not do it when x_T provided + x_T = x_T * self.sampler.model.get_sigmas(steps)[0] + return x_T + def _get_device(self): if torch.cuda.is_available(): return torch.device('cuda') @@ -661,3 +723,49 @@ def _split_weighted_subprompts(text): weights.append(1.0) remaining = 0 return prompts, weights + + def slerp(self, t, v0, v1, DOT_THRESHOLD=0.9995): + ''' + Spherical linear interpolation + Args: + t (float/np.ndarray): Float value between 0.0 and 1.0 + v0 (np.ndarray): Starting vector + v1 (np.ndarray): Final vector + DOT_THRESHOLD (float): Threshold for considering the two vectors as + colineal. Not recommended to alter this. + Returns: + v2 (np.ndarray): Interpolation vector between v0 and v1 + ''' + c = False + if not isinstance(v0,np.ndarray): + c = True + v0 = v0.detach().cpu().numpy() + if not isinstance(v1,np.ndarray): + c = True + v1 = v1.detach().cpu().numpy() + # Copy the vectors to reuse them later + v0_copy = np.copy(v0) + v1_copy = np.copy(v1) + # Normalize the vectors to get the directions and angles + v0 = v0 / np.linalg.norm(v0) + v1 = v1 / np.linalg.norm(v1) + # Dot product with the normalized vectors (can't use np.dot in W) + dot = np.sum(v0 * v1) + # If absolute value of dot product is almost 1, vectors are ~colineal, so use lerp + if np.abs(dot) > DOT_THRESHOLD: + return lerp(t, v0_copy, v1_copy) + # Calculate initial angle between v0 and v1 + theta_0 = np.arccos(dot) + sin_theta_0 = np.sin(theta_0) + # Angle at timestep t + theta_t = theta_0 * t + sin_theta_t = np.sin(theta_t) + # Finish the slerp algorithm + s0 = np.sin(theta_0 - theta_t) / sin_theta_0 + s1 = sin_theta_t / sin_theta_0 + v2 = s0 * v0_copy + s1 * v1_copy + if c: + res = torch.from_numpy(v2).to(self.device) + else: + res = v2 + return res \ No newline at end of file diff --git a/scripts/dream.py b/scripts/dream.py index 5c51c6e68bb..f7733c3d962 100755 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -462,6 +462,20 @@ def create_cmd_parser(): metavar='SAMPLER_NAME', help=f'Switch to a different sampler. Supported samplers: {", ".join(SAMPLER_CHOICES)}', ) + parser.add_argument( + '-z', + '--seed_fuzz', + default=None, + type=float, + help='optional 0-1 value to slerp from -S noise to random noise (allows variations on an image)' + ) + parser.add_argument( + '-Z', + '--seed_fuzz_target', + default=None, + type=int, + help='optional target seed that -S noise is slerped to (interpolate one image to another)' + ) return parser From c062f5800d213e06c59f1e7d4f5f2b5b03510285 Mon Sep 17 00:00:00 2001 From: xra Date: Tue, 30 Aug 2022 00:37:27 +0900 Subject: [PATCH 02/11] was missing lerp --- ldm/simplet2i.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 671918bf169..fd6f3374bde 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -753,7 +753,7 @@ def slerp(self, t, v0, v1, DOT_THRESHOLD=0.9995): dot = np.sum(v0 * v1) # If absolute value of dot product is almost 1, vectors are ~colineal, so use lerp if np.abs(dot) > DOT_THRESHOLD: - return lerp(t, v0_copy, v1_copy) + return v0_copy*(1.0-t)+v1_copy*t # lerp # Calculate initial angle between v0 and v1 theta_0 = np.arccos(dot) sin_theta_0 = np.sin(theta_0) From cfac71e7c010d3353488c94322b0737b8c2ae508 Mon Sep 17 00:00:00 2001 From: xra Date: Tue, 30 Aug 2022 00:59:30 +0900 Subject: [PATCH 03/11] fixed up slerp --- ldm/simplet2i.py | 45 ++++++++++++++++++--------------------------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index fd6f3374bde..d6d24a45468 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -736,36 +736,27 @@ def slerp(self, t, v0, v1, DOT_THRESHOLD=0.9995): Returns: v2 (np.ndarray): Interpolation vector between v0 and v1 ''' - c = False + inputs_are_torch = False if not isinstance(v0,np.ndarray): - c = True + inputs_are_torch = True v0 = v0.detach().cpu().numpy() if not isinstance(v1,np.ndarray): - c = True + inputs_are_torch = True v1 = v1.detach().cpu().numpy() - # Copy the vectors to reuse them later - v0_copy = np.copy(v0) - v1_copy = np.copy(v1) - # Normalize the vectors to get the directions and angles - v0 = v0 / np.linalg.norm(v0) - v1 = v1 / np.linalg.norm(v1) - # Dot product with the normalized vectors (can't use np.dot in W) - dot = np.sum(v0 * v1) - # If absolute value of dot product is almost 1, vectors are ~colineal, so use lerp + + dot = np.sum(v0 * v1 / (np.linalg.norm(v0) * np.linalg.norm(v1))) if np.abs(dot) > DOT_THRESHOLD: - return v0_copy*(1.0-t)+v1_copy*t # lerp - # Calculate initial angle between v0 and v1 - theta_0 = np.arccos(dot) - sin_theta_0 = np.sin(theta_0) - # Angle at timestep t - theta_t = theta_0 * t - sin_theta_t = np.sin(theta_t) - # Finish the slerp algorithm - s0 = np.sin(theta_0 - theta_t) / sin_theta_0 - s1 = sin_theta_t / sin_theta_0 - v2 = s0 * v0_copy + s1 * v1_copy - if c: - res = torch.from_numpy(v2).to(self.device) + v2 = (1 - t) * v0 + t * v1 else: - res = v2 - return res \ No newline at end of file + theta_0 = np.arccos(dot) + sin_theta_0 = np.sin(theta_0) + theta_t = theta_0 * t + sin_theta_t = np.sin(theta_t) + s0 = np.sin(theta_0 - theta_t) / sin_theta_0 + s1 = sin_theta_t / sin_theta_0 + v2 = s0 * v0 + s1 * v1 + + if inputs_are_torch: + v2 = torch.from_numpy(v2).to(self.device) + + return v2 From 89a003db0bd074d25b0c0da6e5b9bc64663aeebf Mon Sep 17 00:00:00 2001 From: xra Date: Tue, 30 Aug 2022 12:22:36 +0900 Subject: [PATCH 04/11] switched to -v -V arguments and clarified code --- ldm/simplet2i.py | 53 ++++++++++++++++++++++++------------------------ scripts/dream.py | 12 +++++------ 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index d6d24a45468..bd93da2c869 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -212,8 +212,8 @@ def prompt2image( upscale=None, variants=None, sampler_name=None, - seed_fuzz=None, - seed_fuzz_target=None, + variant_amount=None, + variant_seed=None, **args, ): # eat up additional cruft """ @@ -233,8 +233,8 @@ def prompt2image( ddim_eta // image randomness (eta=0.0 means the same seed always produces the same image) variants // if >0, the 1st generated image will be passed back to img2img to generate the requested number of variants image_callback // a function or method that will be called each time an image is generated - seed_fuzz // optional 0-1 value to slerp from -S noise to random noise (allows variations on an image) - seed_fuzz_target // optional target seed that -S noise is slerped to (interpolate one image to another) + variant_amount // optional 0-1 value to slerp from -S noise to random noise (allows variations on an image) + variant_seed // optional target seed that -S noise is slerped to (interpolate one image to another) To use the callback, define a function of method that receives two arguments, an Image object and the seed. You can then do whatever you like with the image, including converting it to @@ -308,8 +308,8 @@ def process_image(image,seed): skip_normalize=skip_normalize, width=width, height=height, - seed_fuzz=seed_fuzz, - seed_fuzz_target=seed_fuzz_target, + variant_amount=variant_amount, + variant_seed=variant_seed, ) with scope(self.device.type), self.model.ema_scope(): @@ -396,8 +396,8 @@ def _txt2img( skip_normalize, width, height, - seed_fuzz, - seed_fuzz_target, + variant_amount, + variant_seed, ): """ An infinite iterator of images from the prompt. @@ -405,7 +405,7 @@ def _txt2img( sampler = self.sampler - base_x_T, target_x_T = self._seed_fuzz(width, height, seed_fuzz, seed_fuzz_target) + base_x_T, target_x_T = self._get_variation_noise(width, height, variant_amount, variant_seed) while True: uc, c = self._get_uc_and_c(prompt, batch_size, skip_normalize) @@ -415,7 +415,7 @@ def _txt2img( width // self.downsampling_factor, ] - x_T = self._seed_fuzz_slerp(width, height, steps, seed_fuzz, seed_fuzz_target, base_x_T, target_x_T) + x_T = self._apply_variation_slerp(width, height, steps, variant_amount, variant_seed, base_x_T, target_x_T) samples, _ = sampler.sample( S=steps, @@ -531,47 +531,48 @@ def _new_seed(self): self.seed = random.randrange(0, np.iinfo(np.uint32).max) return self.seed - def _seed_fuzz(self, width:int, height:int, seed_fuzz:float, seed_fuzz_target:int) -> "tuple[torch.Tensor,torch.Tensor]": + def _get_variation_noise(self, width:int, height:int, variant_amount:float, variant_seed:int) -> "tuple[torch.Tensor,torch.Tensor]": base_x_T = None target_x_T = None - if seed_fuzz is not None: - seed_fuzz = max(0.0, min(1.0, seed_fuzz)) - # seed fuzz, base noise is made from seed provided with -S + if variant_amount is not None: + variant_amount = max(0.0, min(1.0, variant_amount)) + # base noise is made from our initial seed or seed provided with -S base_x_T = torch.randn([self.batch_size, self.latent_channels, height // self.downsampling_factor, width // self.downsampling_factor], device=self.device) - if seed_fuzz_target is not None: - # has target seed, store initial seed + if variant_seed is not None: + # store initial seed initialSeed = torch.initial_seed() - seed_everything(seed_fuzz_target) # seed with target + # generate target noise from the provided variant seed + seed_everything(variant_seed) target_x_T = torch.randn([self.batch_size, self.latent_channels, height // self.downsampling_factor, width // self.downsampling_factor], device=self.device) - # back to our initialSeed (is this correct? it works...) + # switch back to the initial seed seed_everything(initialSeed) return base_x_T, target_x_T - def _seed_fuzz_slerp(self, + def _apply_variation_slerp(self, width:int, height:int, steps:int, - seed_fuzz:float, seed_fuzz_target:int, + variant_amount:float, variant_seed:int, base_x_T:torch.Tensor, target_x_T:torch.Tensor) -> torch.Tensor: x_T = None - if seed_fuzz is not None: - seed_fuzz = max(0.0, min(1.0, seed_fuzz)) - # no target seed, get random noise - if seed_fuzz_target is None: + if variant_amount is not None: + variant_amount = max(0.0, min(1.0, variant_amount)) + # no variant seed specified, generate random noise + if variant_seed is None: target_x_T = torch.randn([self.batch_size, self.latent_channels, height // self.downsampling_factor, width // self.downsampling_factor], device=self.device) - # slerp base -> target using seed_fuzz amount - x_T = self.slerp(seed_fuzz, base_x_T, target_x_T) + # slerp base -> target using variant amount + x_T = self.slerp(variant_amount, base_x_T, target_x_T) # only for ksampler! if isinstance(self.sampler, KSampler): diff --git a/scripts/dream.py b/scripts/dream.py index f7733c3d962..66fe9c07b3b 100755 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -463,18 +463,18 @@ def create_cmd_parser(): help=f'Switch to a different sampler. Supported samplers: {", ".join(SAMPLER_CHOICES)}', ) parser.add_argument( - '-z', - '--seed_fuzz', + '-v', + '--variant_amount', default=None, type=float, - help='optional 0-1 value to slerp from -S noise to random noise (allows variations on an image)' + help='0.0 to 1.0 value controlling percentage of variation noise applied' ) parser.add_argument( - '-Z', - '--seed_fuzz_target', + '-V', + '--variant_seed', default=None, type=int, - help='optional target seed that -S noise is slerped to (interpolate one image to another)' + help='manually seed the variation noise, instead of using randomly generated noise' ) return parser From 771ae051ba4fd5fe7816564b242ed045b0e285d2 Mon Sep 17 00:00:00 2001 From: xra Date: Tue, 30 Aug 2022 12:32:50 +0900 Subject: [PATCH 05/11] duplicate parameter during merge --- ldm/simplet2i.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index cac8aba909f..616c3977cdc 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -215,8 +215,6 @@ def prompt2image( upscale=None, variants=None, sampler_name=None, - variant_amount=None, - variant_seed=None, log_tokenization=False, variant_amount=None, variant_seed=None, From b2825d1bdcc609340bf4cbb2ac281417e5de00e3 Mon Sep 17 00:00:00 2001 From: xra Date: Tue, 30 Aug 2022 12:40:31 +0900 Subject: [PATCH 06/11] added assert for variant_amount in range of 0.0-1.0 --- ldm/simplet2i.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 616c3977cdc..3fae64ec931 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -274,6 +274,9 @@ def process_image(image,seed): assert ( 0.0 <= strength <= 1.0 ), 'can only work with strength in [0.0, 1.0]' + assert ( + 0.0 <= variant_amount <= 1.0 + ), '-v --variant_amount must be in 0.0 to 1.0 range' w = int(width / 64) * 64 h = int(height / 64) * 64 if h != height or w != width: From b7ef0544b467fdb520f2cf61ebfe1dd32156363b Mon Sep 17 00:00:00 2001 From: xra Date: Tue, 30 Aug 2022 18:09:27 +0900 Subject: [PATCH 07/11] fixed NoneType type error code gremlins... --- ldm/simplet2i.py | 7 ++++--- scripts/dream.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 3fae64ec931..0dd153a1fb4 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -216,7 +216,7 @@ def prompt2image( variants=None, sampler_name=None, log_tokenization=False, - variant_amount=None, + variant_amount=0.0, variant_seed=None, **args, ): # eat up additional cruft @@ -554,7 +554,7 @@ def _new_seed(self): def _get_variation_noise(self, width:int, height:int, variant_amount:float, variant_seed:int) -> "tuple[torch.Tensor,torch.Tensor]": base_x_T = None target_x_T = None - if variant_amount is not None: + if variant_amount != 0.0: variant_amount = max(0.0, min(1.0, variant_amount)) # base noise is made from our initial seed or seed provided with -S base_x_T = torch.randn([self.batch_size, @@ -581,10 +581,11 @@ def _apply_variation_slerp(self, variant_amount:float, variant_seed:int, base_x_T:torch.Tensor, target_x_T:torch.Tensor) -> torch.Tensor: x_T = None - if variant_amount is not None: + if variant_amount != 0.0: variant_amount = max(0.0, min(1.0, variant_amount)) # no variant seed specified, generate random noise if variant_seed is None: + # important note, target_x_T = torch.randn([self.batch_size, self.latent_channels, height // self.downsampling_factor, diff --git a/scripts/dream.py b/scripts/dream.py index d160fbbc694..e919f241367 100755 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -517,7 +517,7 @@ def create_cmd_parser(): parser.add_argument( '-v', '--variant_amount', - default=None, + default=0.0, type=float, help='0.0 to 1.0 value controlling percentage of variation noise applied' ) From 4aec5361a35642a1335a9d76d57fddf322802394 Mon Sep 17 00:00:00 2001 From: xra Date: Wed, 31 Aug 2022 00:10:02 +0900 Subject: [PATCH 08/11] removed special case for KSampler --- ldm/models/diffusion/ksampler.py | 2 +- ldm/simplet2i.py | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ldm/models/diffusion/ksampler.py b/ldm/models/diffusion/ksampler.py index a974d2fec92..cab1c519af2 100644 --- a/ldm/models/diffusion/ksampler.py +++ b/ldm/models/diffusion/ksampler.py @@ -67,7 +67,7 @@ def route_callback(k_callback_values): sigmas = self.model.get_sigmas(S) if x_T is not None: - x = x_T + x = x_T * sigmas[0] else: x = ( torch.randn([batch_size, *shape], device=self.device) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 0dd153a1fb4..e0707f0283f 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -431,7 +431,7 @@ def _txt2img( width // self.downsampling_factor, ] - x_T = self._apply_variation_slerp(width, height, steps, variant_amount, variant_seed, base_x_T, target_x_T) + x_T = self._apply_variation_slerp(width, height, variant_amount, variant_seed, base_x_T, target_x_T) samples, _ = sampler.sample( S=steps, @@ -577,7 +577,7 @@ def _get_variation_noise(self, width:int, height:int, variant_amount:float, vari return base_x_T, target_x_T def _apply_variation_slerp(self, - width:int, height:int, steps:int, + width:int, height:int, variant_amount:float, variant_seed:int, base_x_T:torch.Tensor, target_x_T:torch.Tensor) -> torch.Tensor: x_T = None @@ -594,11 +594,7 @@ def _apply_variation_slerp(self, # slerp base -> target using variant amount x_T = self.slerp(variant_amount, base_x_T, target_x_T) - - # only for ksampler! - if isinstance(self.sampler, KSampler): - # KSampler does not do it when x_T provided - x_T = x_T * self.sampler.model.get_sigmas(steps)[0] + return x_T def _get_device(self): From 18305ec3e1fff8cef40f725c60cb1a13c04c6f84 Mon Sep 17 00:00:00 2001 From: xra Date: Wed, 31 Aug 2022 01:48:01 +0900 Subject: [PATCH 09/11] fixed up Variations generating the same sequence of results using -v in combination with -S -n would re-generate the same sequence of variations, because seed_everything "restarts" the seed sequence in prompt2image, this is fixed now specifically for variants you can still use -S -n (without -v) and it will produce the same sequence of images starting from the given seed --- ldm/simplet2i.py | 71 ++++++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index e0707f0283f..156424ae10a 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -329,8 +329,15 @@ def process_image(image,seed): with scope(self.device.type), self.model.ema_scope(): for n in trange(iterations, desc='Generating'): - seed_everything(seed) - iter_images = next(images_iterator) + seed_everything(seed) + + iter_images,iter_seed = next(images_iterator) + # image iterator can modify the seed (during variations) + # this is a workaround until we have objects for containing gen info + # also wanted to keep it so self.seed is only modified by _new_seed() + if iter_seed is not None: + seed = iter_seed + for image in iter_images: results.append([image, seed]) if image_callback is not None: @@ -420,8 +427,8 @@ def _txt2img( """ sampler = self.sampler - - base_x_T, target_x_T = self._get_variation_noise(width, height, variant_amount, variant_seed) + + base_x_T = self._get_base_noise(width, height, variant_amount) while True: uc, c = self._get_uc_and_c(prompt, batch_size, skip_normalize) @@ -431,7 +438,7 @@ def _txt2img( width // self.downsampling_factor, ] - x_T = self._apply_variation_slerp(width, height, variant_amount, variant_seed, base_x_T, target_x_T) + x_T,seed = self._apply_variation_slerp(width, height, variant_amount, variant_seed, base_x_T) samples, _ = sampler.sample( S=steps, @@ -445,7 +452,7 @@ def _txt2img( img_callback=callback, x_T = x_T ) - yield self._samples_to_images(samples) + yield self._samples_to_images(samples),seed @torch.no_grad() def _img2img( @@ -503,7 +510,7 @@ def _img2img( unconditional_guidance_scale=cfg_scale, unconditional_conditioning=uc, ) - yield self._samples_to_images(samples) + yield self._samples_to_images(samples),None # TODO: does this actually need to run every loop? does anything in it vary by random seed? def _get_uc_and_c(self, prompt, batch_size, skip_normalize): @@ -551,51 +558,49 @@ def _new_seed(self): self.seed = random.randrange(0, np.iinfo(np.uint32).max) return self.seed - def _get_variation_noise(self, width:int, height:int, variant_amount:float, variant_seed:int) -> "tuple[torch.Tensor,torch.Tensor]": + def _get_base_noise(self, width:int, height:int, variant_amount:float) -> torch.Tensor: base_x_T = None - target_x_T = None if variant_amount != 0.0: variant_amount = max(0.0, min(1.0, variant_amount)) - # base noise is made from our initial seed or seed provided with -S + # base noise is made from whatever our seed currently is base_x_T = torch.randn([self.batch_size, self.latent_channels, height // self.downsampling_factor, width // self.downsampling_factor], device=self.device) - if variant_seed is not None: - # store initial seed - initialSeed = torch.initial_seed() - # generate target noise from the provided variant seed - seed_everything(variant_seed) - target_x_T = torch.randn([self.batch_size, - self.latent_channels, - height // self.downsampling_factor, - width // self.downsampling_factor], - device=self.device) - # switch back to the initial seed - seed_everything(initialSeed) - return base_x_T, target_x_T + return base_x_T def _apply_variation_slerp(self, - width:int, height:int, + width:int, height:int, variant_amount:float, variant_seed:int, - base_x_T:torch.Tensor, target_x_T:torch.Tensor) -> torch.Tensor: + base_x_T:torch.Tensor) -> "[torch.Tensor,int]": x_T = None + seed = None if variant_amount != 0.0: variant_amount = max(0.0, min(1.0, variant_amount)) # no variant seed specified, generate random noise if variant_seed is None: - # important note, - target_x_T = torch.randn([self.batch_size, - self.latent_channels, - height // self.downsampling_factor, - width // self.downsampling_factor], - device=self.device) + # TODO refactor seed overall? + # for now I use uptime to seed variations, + # otherwise you get the same set of variations + # from a seed provided with -S + # as seed_everything is used in prompt2image, + # which "restarts" the sequence of seeds. + seed = time.monotonic_ns() % np.iinfo(np.uint32).max + else: # use variant seed for noise + seed = variant_seed + + seed_everything(seed) + + target_x_T = torch.randn([self.batch_size, + self.latent_channels, + height // self.downsampling_factor, + width // self.downsampling_factor], + device=self.device) # slerp base -> target using variant amount x_T = self.slerp(variant_amount, base_x_T, target_x_T) - - return x_T + return x_T, seed def _get_device(self): if torch.cuda.is_available(): From 0581c5056fa7f957738c73b614e51fcf3dbf3965 Mon Sep 17 00:00:00 2001 From: xra Date: Thu, 1 Sep 2022 00:35:40 +0900 Subject: [PATCH 10/11] fixed batch size in variation noise --- ldm/simplet2i.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 067149986ec..59648000e66 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -714,7 +714,7 @@ def _get_base_noise(self, width:int, height:int, variant_amount:float) -> torch. if variant_amount != 0.0: variant_amount = max(0.0, min(1.0, variant_amount)) # base noise is made from whatever our seed currently is - base_x_T = torch.randn([self.batch_size, + base_x_T = torch.randn([1, self.latent_channels, height // self.downsampling_factor, width // self.downsampling_factor], @@ -743,7 +743,7 @@ def _apply_variation_slerp(self, seed_everything(seed) - target_x_T = torch.randn([self.batch_size, + target_x_T = torch.randn([1, self.latent_channels, height // self.downsampling_factor, width // self.downsampling_factor], From d6f54d3bfc57c1492505b17410265a366dc0a319 Mon Sep 17 00:00:00 2001 From: xra Date: Thu, 1 Sep 2022 01:30:39 +0900 Subject: [PATCH 11/11] reset seed to random state thanks to bakkot's suggestion --- ldm/simplet2i.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 59648000e66..b72fe3e312a 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -731,13 +731,8 @@ def _apply_variation_slerp(self, variant_amount = max(0.0, min(1.0, variant_amount)) # no variant seed specified, generate random noise if variant_seed is None: - # TODO refactor seed overall? - # for now I use uptime to seed variations, - # otherwise you get the same set of variations - # from a seed provided with -S - # as seed_everything is used in prompt2image, - # which "restarts" the sequence of seeds. - seed = time.monotonic_ns() % np.iinfo(np.uint32).max + random.seed() # reset RNG to an actually random state, so we can get a random seed + seed = random.randrange(0,np.iinfo(np.uint32).max) else: # use variant seed for noise seed = variant_seed