Skip to content

ability to generate variations in the space of seeds instead of latent space #81

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
bakkot wants to merge 2 commits into from
Closed

ability to generate variations in the space of seeds instead of latent space #81

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
239b3d3
add variation script
bakkot Aug 25, 2022
4213c62
more-vary
bakkot Aug 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 82 additions & 0 deletions dream-variations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
from ldm.simplet2i import T2I
import transformers
import numpy as np
import torch
from pytorch_lightning import seed_everything
import random

t2i = T2I(
latent_diffusion_weights=False,
config = "configs/stable-diffusion/v1-inference.yaml"
)

def slerp(t, v0, v1, DOT_THRESHOLD=0.9995):
""" helper function to spherically interpolate two arrays v1 v2 """

if not isinstance(v0, np.ndarray):
inputs_are_torch = True
input_device = v0.device
v0 = v0.cpu().numpy()
v1 = v1.cpu().numpy()

dot = np.sum(v0 * v1 / (np.linalg.norm(v0) * np.linalg.norm(v1)))
if np.abs(dot) > DOT_THRESHOLD:
v2 = (1 - t) * v0 + t * v1
else:
theta_0 = np.arccos(dot)
sin_theta_0 = np.sin(theta_0)
theta_t = theta_0 * t
sin_theta_t = np.sin(theta_t)
s0 = np.sin(theta_0 - theta_t) / sin_theta_0
s1 = sin_theta_t / sin_theta_0
v2 = s0 * v0 + s1 * v1

if inputs_are_torch:
v2 = torch.from_numpy(v2).to(input_device)

return v2


transformers.logging.set_verbosity_error()
seed = 3516972428
seed_everything(seed)
init_code = torch.randn([t2i.batch_size,
t2i.latent_channels,
t2i.height // t2i.downsampling_factor,
t2i.width // t2i.downsampling_factor],
device=t2i.device)


print("loading model...")
t2i.load_model()

##### interpolate
# noise = torch.randn([t2i.batch_size,
# t2i.latent_channels,
# t2i.height // t2i.downsampling_factor,
# t2i.width // t2i.downsampling_factor],
# device=t2i.device)
# for i in range(20):
# print("running generation " + str(i))
# code = slerp(i / 20., init_code, noise)
# outputs = t2i.txt2img("elf queen with rainbow hair, golden hour. colored pencil drawing by rossdraws andrei riabovitchev trending on artstation", start_code=code, seed=seed)

# generate variants
strength = 0.10
prompt = "elf queen with rainbow hair, golden hour. colored pencil drawing by rossdraws andrei riabovitchev trending on artstation"

print("generating base image")
t2i.txt2img(prompt, start_code=init_code, seed=seed)
for i in range(20):
random.seed() # reset RNG to an actually random state, so we can get a random seed
iter_seed = random.randrange(0,np.iinfo(np.uint32).max)
print("iteration " + str(i) + " running, seed = " + str(iter_seed))
seed_everything(iter_seed)
noise = torch.randn([t2i.batch_size,
t2i.latent_channels,
t2i.height // t2i.downsampling_factor,
t2i.width // t2i.downsampling_factor],
device=t2i.device)
code = slerp(strength, init_code, noise)
seed_everything(iter_seed)
t2i.txt2img(prompt, start_code=code, seed=iter_seed)
38 changes: 19 additions & 19 deletions ldm/simplet2i.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
# do the slow model initialization
t2i.load_model()

# Do the fast inference & image generation. Any options passed here
# Do the fast inference & image generation. Any options passed here
# override the default values assigned during class initialization
# Will call load_model() if the model was not previously loaded.
# The method returns a list of images. Each row of the list is a sub-list of [filename,seed]
Expand All @@ -45,7 +45,7 @@
results = t2i.img2img(prompt = "an astronaut riding a horse"
outdir = "./outputs/img2img-samples"
init_img = "./sketches/horse+rider.png")

for row in results:
print(f'filename={row[0]}')
print(f'seed ={row[1]}')
Expand Down Expand Up @@ -158,7 +158,7 @@ def __init__(self,
def txt2img(self,prompt,outdir=None,batch_size=None,iterations=None,
steps=None,seed=None,grid=None,individual=None,width=None,height=None,
cfg_scale=None,ddim_eta=None,strength=None,embedding_path=None,init_img=None,
skip_normalize=False,variants=None): # note the "variants" option is an unused hack caused by how options are passed
skip_normalize=False,variants=None,start_code=None): # note the "variants" option is an unused hack caused by how options are passed
"""
Generate an image from the prompt, writing iteration images into the outdir
The output is a list of lists in the format: [[filename1,seed1], [filename2,seed2],...]
Expand Down Expand Up @@ -186,19 +186,19 @@ def txt2img(self,prompt,outdir=None,batch_size=None,iterations=None,
grid = self.grid
if individual:
grid = False

data = [batch_size * [prompt]]

# make directories and establish names for the output files
os.makedirs(outdir, exist_ok=True)

start_code = None
if self.fixed_code:
start_code = torch.randn([batch_size,
self.latent_channels,
height // self.downsampling_factor,
width // self.downsampling_factor],
device=self.device)

# if self.fixed_code:
# start_code = torch.randn([batch_size,
# self.latent_channels,
# height // self.downsampling_factor,
# width // self.downsampling_factor],
# device=self.device)

precision_scope = autocast if self.precision=="autocast" else nullcontext
sampler = self.sampler
Expand Down Expand Up @@ -281,7 +281,7 @@ def txt2img(self,prompt,outdir=None,batch_size=None,iterations=None,
print(f'{image_count} images generated in',"%4.2fs"% (toc-tic))

return images

# There is lots of shared code between this and txt2img and should be refactored.
@torch.no_grad()
def img2img(self,prompt,outdir=None,init_img=None,batch_size=None,iterations=None,
Expand Down Expand Up @@ -319,7 +319,7 @@ def img2img(self,prompt,outdir=None,init_img=None,batch_size=None,iterations=Non
grid = self.grid
if individual:
grid = False

data = [batch_size * [prompt]]

# PLMS sampler not supported yet, so ignore previous sampler
Expand All @@ -345,7 +345,7 @@ def img2img(self,prompt,outdir=None,init_img=None,batch_size=None,iterations=Non
except AssertionError:
print(f"strength must be between 0.0 and 1.0, but received value {strength}")
return []

t_enc = int(strength * steps)
print(f"target t_enc is {t_enc} steps")

Expand Down Expand Up @@ -485,7 +485,7 @@ def load_model(self):
print(msg)

return self.model

def _load_model_from_config(self, config, ckpt):
print(f"Loading model from {ckpt}")
pl_sd = torch.load(ckpt, map_location="cpu")
Expand Down Expand Up @@ -530,7 +530,7 @@ def _unique_filename(self,outdir,previousname=None,seed=0,isbatch=False,grid_cou
filename = f'{basecount:06}.{seed}.01.png'
else:
filename = f'{basecount:06}.{seed}.png'

return os.path.join(outdir,filename)

else:
Expand All @@ -540,7 +540,7 @@ def _unique_filename(self,outdir,previousname=None,seed=0,isbatch=False,grid_cou
return self._unique_filename(outdir,previousname,seed)

basecount = int(x.groups()[0])
series = 0
series = 0
finished = False
while not finished:
series += 1
Expand All @@ -552,7 +552,7 @@ def _unique_filename(self,outdir,previousname=None,seed=0,isbatch=False,grid_cou

def _split_weighted_subprompts(text):
"""
grabs all text up to the first occurrence of ':'
grabs all text up to the first occurrence of ':'
uses the grabbed text as a sub-prompt, and takes the value following ':' as weight
if ':' has no value defined, defaults to 1.0
repeats until no text remaining
Expand All @@ -568,7 +568,7 @@ def _split_weighted_subprompts(text):
remaining -= idx
# remove from main text
text = text[idx+1:]
# find value for weight
# find value for weight
if " " in text:
idx = text.index(" ") # first occurence
else: # no space, read to end
Expand Down