From 7c95baabc0f0b77e0df3915387f4ad2ed4a92378 Mon Sep 17 00:00:00 2001 From: James Reynolds Date: Sun, 28 Aug 2022 22:22:15 -0600 Subject: [PATCH 1/7] Apple MPS Support --- .gitignore | 3 + README.md | 152 ++++++++++++++++++++++++++++++++ environment-mac.yaml | 32 +++++++ ldm/models/diffusion/ddim.py | 14 ++- ldm/models/diffusion/plms.py | 12 ++- ldm/modules/attention.py | 1 + ldm/modules/encoders/modules.py | 30 +++++-- notebook_helpers.py | 8 +- scripts/inpaint.py | 7 +- scripts/orig_scripts/img2img.py | 16 +++- scripts/orig_scripts/txt2img.py | 16 +++- 11 files changed, 271 insertions(+), 20 deletions(-) create mode 100644 environment-mac.yaml diff --git a/.gitignore b/.gitignore index ffdc22fed45..a1a568c36da 100644 --- a/.gitignore +++ b/.gitignore @@ -176,3 +176,6 @@ src logs/ **/__pycache__/ outputs + +# If it's a Mac +.DS_Store diff --git a/README.md b/README.md index c18b43d8fe7..e32a5d23acf 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,155 @@ +# Apple Silicon Mac Users + +Several people have gotten Stable Diffusion to work on Apple Silicon Macs using Anaconda. I've gathered up most of their instructions and put them in this fork (and readme). I haven't tested anything besides Anaconda, and I've read about issues with things like miniforge, so if you have an issue that isn't dealt with in this fork then head on over to the [Apple Silicon](https://github.com/CompVis/stable-diffusion/issues/25) issue on GitHub (that page is so long that GitHub hides most of it by default, so you need to find the hidden part and expand it to view the whole thing). This fork would not have been possible without the work done by the people on that issue. + +You have to have macOS 12.3 Monterey or later. Anything earlier than that won't work. + +BTW, I haven't tested any of this on Intel Macs. + +How to: + +``` +git clone https://github.com/lstein/stable-diffusion.git +cd stable-diffusion +git checkout apple-mps-support + +mkdir -p models/ldm/stable-diffusion-v1/ +ln -s /path/to/ckpt/sd-v1-1.ckpt models/ldm/stable-diffusion-v1/model.ckpt + +conda env create -f environment-mac.yaml +conda activate ldm +``` + +These instructions are identical to the main repo except I added environment-mac.yaml because Mac doesn't have cudatoolkit. + +After you follow all the instructions and run txt2img.py you might get several errors. Here's the errors I've seen and found solutions for. + +### Doesn't work anymore? + +We are using PyTorch nightly, which includes support for MPS. I don't know exactly how Anaconda does updates, but I woke up one morning and Stable Diffusion crashed and I couldn't think of anything I did that would've changed anything the night before, when it worked. A day and a half later I finally got it working again. I don't know what changed overnight. PyTorch-nightly changes overnight but I'm pretty sure I didn't manually update it. Either way, things are probably going to be bumpy on Apple Silicon until PyTorch releases a firm version that we can lock to. + +To manually update to the latest version of PyTorch nightly (which could fix issues), run this command. + + conda install pytorch torchvision torchaudio -c pytorch-nightly + +## Debugging? + +Tired of waiting for your renders to finish before you can see if it works? Reduce the steps! The picture wont look like anything but if it finishes, hey, it works! This could also help you figure out if you've got a memory problem, because I'm betting 1 step doesn't use much memory. + + python ./scripts/txt2img.py --prompt "ocean" --ddim_steps 1 + +### "No module named cv2" (or some other module) + +Did you remember to `conda activate ldm`? If your terminal prompt begins with "(ldm)" then you activated it. If it begins with "(base)" or something else you haven't. + +If it says you're missing taming you need to rebuild your virtual environment. + + conda env remove -n ldm + conda env create -f environment-mac.yaml + +If you have activated the ldm virtual environment and tried rebuilding it, maybe the problem could be that I have something installed that you don't and you'll just need to manually install it. Make sure you activate the virtual environment so it installs there instead of globally. + + conda activate ldm + pip install *name* + +You might also need to install Rust (I mention this again below). + +### "The operator [name] is not current implemented for the MPS device." (sic) + +Example error. + +``` +... +NotImplementedError: The operator 'aten::index.Tensor' is not current implemented for the MPS device. If you want this op to be added in priority during the prototype phase of this feature, please comment on [https://github.com/pytorch/pytorch/issues/77764](https://github.com/pytorch/pytorch/issues/77764). As a temporary fix, you can set the environment variable `PYTORCH_ENABLE_MPS_FALLBACK=1` to use the CPU as a fallback for this op. WARNING: this will be slower than running natively on MPS. +``` + +Just do what it says: + + export PYTORCH_ENABLE_MPS_FALLBACK=1 + +### "Could not build wheels for tokenizers" + +I have not seen this error because I had Rust installed on my computer before I started playing with Stable Diffusion. The fix is to install Rust. + + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + +### How come `--seed` doesn't work? + +> Completely reproducible results are not guaranteed across PyTorch releases, individual commits, or different platforms. Furthermore, results may not be reproducible between CPU and GPU executions, even when using identical seeds. + +[PyTorch docs](https://pytorch.org/docs/stable/notes/randomness.html) + +There is an [open issue](https://github.com/pytorch/pytorch/issues/78035) (as of August 2022) in pytorch regarding gradient inconsistency. I am guessing that's what is causing this. + +### libiomp5.dylib error? + + OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized. + +There are several things you can do. First, you could use something besides Anaconda like miniforge. I read a lot of things online telling people to use something else, but I am stuck with Anaconda for other reasons. + +Or you can try this. + + export KMP_DUPLICATE_LIB_OK=True + +Or this (which takes forever on my computer and didn't work anyway). + + conda install nomkl + +This error happens with Anaconda on Macs, and [nomkl](https://stackoverflow.com/questions/66224879/what-is-the-nomkl-python-package-used-for) is supposed to fix the issue (it isn't a module but a fix of some sort). [There's more suggestions](https://stackoverflow.com/questions/53014306/error-15-initializing-libiomp5-dylib-but-found-libiomp5-dylib-already-initial), like uninstalling tensorflow and reinstalling. I haven't tried them. + +### Not enough memory. + +This seems to be a common problem and is probably the underlying problem for a lot of symptoms (listed below). The fix is to lower your image size or to add `model.half()` right after the model is loaded. I should probably test it out. I've read that the reason this fixes problems is because it converts the model from 32-bit to 16-bit and that leaves more RAM for other things. I have no idea how that would affect the quality of the images though. + +See [this issue](https://github.com/CompVis/stable-diffusion/issues/71). + +### "Error: product of dimension sizes > 2**31'" + +This error happens with img2img, which I haven't played with too much yet. But I know it's because your image is too big or the resolution isn't a multiple of 32x32. Because the stable-diffusion model was trained on images that were 512 x 512, it's always best to use that output size (which is the default). However, if you're using that size and you get the above error, try 256 x 256 or 512 x 256 or something as the source image. + +BTW, 2**31-1 = [2,147,483,647](https://en.wikipedia.org/wiki/2,147,483,647#In_computing), which is also 32-bit signed [LONG_MAX](https://en.wikipedia.org/wiki/C_data_types) in C. + +### I just got Rickrolled! Do I have a virus? + +You don't have a virus. It's part of the project. Here's [Rick](https://github.com/lstein/stable-diffusion/blob/main/assets/rick.jpeg) and here's [the code](https://github.com/lstein/stable-diffusion/blob/69ae4b35e0a0f6ee1af8bb9a5d0016ccb27e36dc/scripts/txt2img.py#L79) that swaps him in. It's a NSFW filter, which IMO, doesn't work very good (and we call this "computer vision", sheesh). + +Actually, this could be happening because there's not enough RAM. You could try the `model.half()` suggestion or specify smaller output images. + +### My images come out black + +I haven't solved this issue. I just throw away my black images. There's a [similar issue](https://github.com/CompVis/stable-diffusion/issues/69) on CUDA GPU's where the images come out green. Maybe it's the same issue? Someone in that issue says to use "--precision full", but this fork actually disables that flag. I don't know why, someone else provided that code and I don't know what it does. Maybe the `model.half()` suggestion above would fix this issue too. I should probably test it. + +### "view size is not compatible with input tensor's size and stride" + +``` + File "/opt/anaconda3/envs/ldm/lib/python3.10/site-packages/torch/nn/functional.py", line 2511, in layer_norm + return torch.layer_norm(input, normalized_shape, weight, bias, eps, torch.backends.cudnn.enabled) +RuntimeError: view size is not compatible with input tensor's size and stride (at least one dimension spans across two contiguous subspaces). Use .reshape(...) instead. +``` + +Update to the latest version of lstein/stable-diffusion. We were patching pytorch but we found a file in stable-diffusion that we could change instead. This is a 32-bit vs 16-bit problem. + +### The processor must support the Intel bla bla bla + +What? Intel? On an Apple Silicon? + + Intel MKL FATAL ERROR: This system does not meet the minimum requirements for use of the Intel(R) Math Kernel Library. + The processor must support the Intel(R) Supplemental Streaming SIMD Extensions 3 (Intel(R) SSSE3) instructions.██████████████| 50/50 [02:25<00:00, 2.53s/it] + The processor must support the Intel(R) Streaming SIMD Extensions 4.2 (Intel(R) SSE4.2) instructions. + The processor must support the Intel(R) Advanced Vector Extensions (Intel(R) AVX) instructions. + +This fixed it for me: + + conda clean --yes --all + +### Still slow? + +I changed the defaults of n_samples and n_iter to 1 so that it uses less RAM and makes less images so it will be faster the first time you use it. I don't actually know what n_samples does internally, but I know it consumes a lot more RAM. The n_iter flag just loops around the image creation code, so it shouldn't consume more RAM (it should be faster if you're going to do multiple images because the libraries and model will already be loaded--use a prompt file to get this speed boost). + +These flags are the default sample and iter settings in this fork/branch: + + python scripts/txt2img.py --prompt "ocean" --n_samples=1 --n_iter=1 + # Stable Diffusion Dream Script This is a fork of CompVis/stable-diffusion, the wonderful open source diff --git a/environment-mac.yaml b/environment-mac.yaml new file mode 100644 index 00000000000..42a5c359e33 --- /dev/null +++ b/environment-mac.yaml @@ -0,0 +1,32 @@ +name: ldm +channels: + - apple + - conda-forge + - pytorch-nightly + - defaults +dependencies: + - python=3.10.4 + - pip=22.1.2 + - pytorch + - torchvision + - numpy=1.23.1 + - pip: + - albumentations==0.4.6 + - opencv-python==4.6.0.66 + - pudb==2019.2 + - imageio==2.9.0 + - imageio-ffmpeg==0.4.2 + - pytorch-lightning==1.4.2 + - omegaconf==2.1.1 + - test-tube>=0.7.5 + - streamlit==1.12.0 + - pillow==9.2.0 + - einops==0.3.0 + - torch-fidelity==0.3.0 + - transformers==4.19.2 + - torchmetrics==0.6.0 + - kornia==0.6.0 + - -e git+https://github.com/openai/CLIP.git@main#egg=clip + - -e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers + - -e git+https://github.com/lstein/k-diffusion.git@master#egg=k-diffusion + - -e . diff --git a/ldm/models/diffusion/ddim.py b/ldm/models/diffusion/ddim.py index 3d9086eb1d6..b2f21749b9c 100644 --- a/ldm/models/diffusion/ddim.py +++ b/ldm/models/diffusion/ddim.py @@ -14,17 +14,25 @@ class DDIMSampler(object): - def __init__(self, model, schedule='linear', device='cuda', **kwargs): + def __init__(self, model, schedule='linear', device='', **kwargs): super().__init__() self.model = model self.ddpm_num_timesteps = model.num_timesteps self.schedule = schedule - self.device = device + if device == '': + if torch.cuda.is_available(): + self.device = 'cuda' + elif torch.backends.mps.is_available(): + self.device = 'mps' + else: + self.device = 'cpu' + else: + self.device = device def register_buffer(self, name, attr): if type(attr) == torch.Tensor: if attr.device != torch.device(self.device): - attr = attr.to(torch.device(self.device)) + attr = attr.to(torch.float32).to(torch.device(self.device)) setattr(self, name, attr) def make_schedule( diff --git a/ldm/models/diffusion/plms.py b/ldm/models/diffusion/plms.py index 7b9dc4706be..e1af4fe7ca8 100644 --- a/ldm/models/diffusion/plms.py +++ b/ldm/models/diffusion/plms.py @@ -13,12 +13,20 @@ class PLMSSampler(object): - def __init__(self, model, schedule='linear', device='cuda', **kwargs): + def __init__(self, model, schedule='linear', device='', **kwargs): super().__init__() self.model = model self.ddpm_num_timesteps = model.num_timesteps self.schedule = schedule - self.device = device + if device == '': + if torch.cuda.is_available(): + self.device = 'cuda' + elif torch.backends.mps.is_available(): + self.device = 'mps' + else: + self.device = 'cpu' + else: + self.device = device def register_buffer(self, name, attr): if type(attr) == torch.Tensor: diff --git a/ldm/modules/attention.py b/ldm/modules/attention.py index 960a112001d..8bbda3e2599 100644 --- a/ldm/modules/attention.py +++ b/ldm/modules/attention.py @@ -234,6 +234,7 @@ def forward(self, x, context=None): ) def _forward(self, x, context=None): + x = x.contiguous() x = self.attn1(self.norm1(x)) + x x = self.attn2(self.norm2(x), context=context) + x x = self.ff(self.norm3(x)) + x diff --git a/ldm/modules/encoders/modules.py b/ldm/modules/encoders/modules.py index 2c25948b5cd..07e1da87f90 100644 --- a/ldm/modules/encoders/modules.py +++ b/ldm/modules/encoders/modules.py @@ -12,6 +12,15 @@ ) # TODO: can we directly rely on lucidrains code and simply add this as a reuirement? --> test +def get_default_device_type(): + if torch.cuda.is_available(): + return 'cuda' + elif torch.backends.mps.is_available(): + return 'mps' + else: + return 'cpu' + + def _expand_mask(mask, dtype, tgt_len=None): """ Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. @@ -67,7 +76,12 @@ class TransformerEmbedder(AbstractEncoder): """Some transformer encoder layers""" def __init__( - self, n_embed, n_layer, vocab_size, max_seq_len=77, device='cuda' + self, + n_embed, + n_layer, + vocab_size, + max_seq_len=77, + device=get_default_device_type(), ): super().__init__() self.device = device @@ -89,7 +103,9 @@ def encode(self, x): class BERTTokenizer(AbstractEncoder): """Uses a pretrained BERT tokenizer by huggingface. Vocab size: 30522 (?)""" - def __init__(self, device='cuda', vq_interface=True, max_length=77): + def __init__( + self, device=get_default_device_type(), vq_interface=True, max_length=77 + ): super().__init__() from transformers import ( BertTokenizerFast, @@ -145,7 +161,7 @@ def __init__( n_layer, vocab_size=30522, max_seq_len=77, - device='cuda', + device=get_default_device_type(), use_tokenizer=True, embedding_dropout=0.0, ): @@ -230,7 +246,7 @@ class FrozenCLIPEmbedder(AbstractEncoder): def __init__( self, version='openai/clip-vit-large-patch14', - device='cuda', + device=get_default_device_type(), max_length=77, ): super().__init__() @@ -455,13 +471,13 @@ class FrozenCLIPTextEmbedder(nn.Module): def __init__( self, version='ViT-L/14', - device='cuda', + device=get_default_device_type(), max_length=77, n_repeat=1, normalize=True, ): super().__init__() - self.model, _ = clip.load(version, jit=False, device='cpu') + self.model, _ = clip.load(version, jit=False, device=device) self.device = device self.max_length = max_length self.n_repeat = n_repeat @@ -496,7 +512,7 @@ def __init__( self, model, jit=False, - device='cuda' if torch.cuda.is_available() else 'cpu', + device=get_default_device_type(), antialias=False, ): super().__init__() diff --git a/notebook_helpers.py b/notebook_helpers.py index 5d0ebd7e1f8..a2a25e1953b 100644 --- a/notebook_helpers.py +++ b/notebook_helpers.py @@ -117,7 +117,13 @@ def get_cond(mode, selected_path): c = rearrange(c, '1 c h w -> 1 h w c') c = 2. * c - 1. - c = c.to(torch.device("cuda")) + if torch.cuda.is_available(): + device = torch.device('cuda') + elif torch.backends.mps.is_available(): + device = torch.device('mps') + else: + device = torch.device('cpu') + c = c.to(device) example["LR_image"] = c example["image"] = c_up diff --git a/scripts/inpaint.py b/scripts/inpaint.py index d6e6387a9a3..20271e29f2b 100644 --- a/scripts/inpaint.py +++ b/scripts/inpaint.py @@ -61,7 +61,12 @@ def make_batch(image, mask, device): model.load_state_dict(torch.load("models/ldm/inpainting_big/last.ckpt")["state_dict"], strict=False) - device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") + if torch.cuda.is_available(): + device = torch.device("cuda") + elif torch.backends.mps.is_available(): + device = torch.device("mps") + else: + device = torch.device("cpu") model = model.to(device) sampler = DDIMSampler(model) diff --git a/scripts/orig_scripts/img2img.py b/scripts/orig_scripts/img2img.py index 421e2151d9e..40bced669cb 100644 --- a/scripts/orig_scripts/img2img.py +++ b/scripts/orig_scripts/img2img.py @@ -19,6 +19,14 @@ from ldm.models.diffusion.ddim import DDIMSampler from ldm.models.diffusion.plms import PLMSSampler +def get_device(): + if(torch.cuda.is_available()): + return torch.device("cuda") + elif(torch.backends.mps.is_available()): + return torch.device("mps") + else: + return torch.device("cpu") + def chunk(it, size): it = iter(it) @@ -40,7 +48,7 @@ def load_model_from_config(config, ckpt, verbose=False): print("unexpected keys:") print(u) - model.cuda() + model.to(get_device()) model.eval() return model @@ -199,7 +207,7 @@ def main(): config = OmegaConf.load(f"{opt.config}") model = load_model_from_config(config, f"{opt.ckpt}") - device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") + device = get_device() model = model.to(device) if opt.plms: @@ -241,8 +249,10 @@ def main(): print(f"target t_enc is {t_enc} steps") precision_scope = autocast if opt.precision == "autocast" else nullcontext + if device.type == 'mps': + precision_scope = nullcontext # have to use f32 on mps with torch.no_grad(): - with precision_scope("cuda"): + with precision_scope(device.type): with model.ema_scope(): tic = time.time() all_samples = list() diff --git a/scripts/orig_scripts/txt2img.py b/scripts/orig_scripts/txt2img.py index 1edc5313093..2c68fe95167 100644 --- a/scripts/orig_scripts/txt2img.py +++ b/scripts/orig_scripts/txt2img.py @@ -19,6 +19,14 @@ from ldm.models.diffusion.ddim import DDIMSampler from ldm.models.diffusion.plms import PLMSSampler +def get_device(): + if(torch.cuda.is_available()): + return 'cuda' + elif(torch.backends.mps.is_available()): + return 'mps' + else: + return 'cpu' + def chunk(it, size): it = iter(it) @@ -40,7 +48,7 @@ def load_model_from_config(config, ckpt, verbose=False): print("unexpected keys:") print(u) - model.cuda() + model.to(get_device()) model.eval() return model @@ -195,7 +203,7 @@ def main(): config = OmegaConf.load(f"{opt.config}") model = load_model_from_config(config, f"{opt.ckpt}") - device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") + device = torch.device(get_device()) model = model.to(device) #for klms @@ -243,8 +251,10 @@ def forward(self, x, sigma, uncond, cond, cond_scale): start_code = torch.randn([opt.n_samples, opt.C, opt.H // opt.f, opt.W // opt.f], device=device) precision_scope = autocast if opt.precision=="autocast" else nullcontext + if device.type == 'mps': + precision_scope = nullcontext # have to use f32 on mps with torch.no_grad(): - with precision_scope("cuda"): + with precision_scope(device.type): with model.ema_scope(): tic = time.time() all_samples = list() From d0b168d2f93831d7a817cc8a03e40620115450fd Mon Sep 17 00:00:00 2001 From: James Reynolds Date: Mon, 29 Aug 2022 00:40:00 -0600 Subject: [PATCH 2/7] Fix --fixed_code (fixes seed) --- scripts/orig_scripts/txt2img.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/orig_scripts/txt2img.py b/scripts/orig_scripts/txt2img.py index 2c68fe95167..18d0c426352 100644 --- a/scripts/orig_scripts/txt2img.py +++ b/scripts/orig_scripts/txt2img.py @@ -248,7 +248,9 @@ def forward(self, x, sigma, uncond, cond, cond_scale): start_code = None if opt.fixed_code: - start_code = torch.randn([opt.n_samples, opt.C, opt.H // opt.f, opt.W // opt.f], device=device) + start_code = torch.randn( + [opt.n_samples, opt.C, opt.H // opt.f, opt.W // opt.f], device="cpu" + ).to(torch.device(device)) precision_scope = autocast if opt.precision=="autocast" else nullcontext if device.type == 'mps': From 04456f3e5d5e56f1f1cb4833e288be75b7e1b741 Mon Sep 17 00:00:00 2001 From: James Reynolds Date: Mon, 29 Aug 2022 00:40:29 -0600 Subject: [PATCH 3/7] Fix one more MPS crash --- ldm/models/diffusion/plms.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ldm/models/diffusion/plms.py b/ldm/models/diffusion/plms.py index e1af4fe7ca8..e60f76892b9 100644 --- a/ldm/models/diffusion/plms.py +++ b/ldm/models/diffusion/plms.py @@ -31,8 +31,7 @@ def __init__(self, model, schedule='linear', device='', **kwargs): def register_buffer(self, name, attr): if type(attr) == torch.Tensor: if attr.device != torch.device(self.device): - attr = attr.to(torch.device(self.device)) - + attr = attr.to(torch.float32).to(torch.device(self.device)) setattr(self, name, attr) def make_schedule( From c33e086ce4558c4f1ff5fa8ea5c8288e9cbd4241 Mon Sep 17 00:00:00 2001 From: James Reynolds Date: Tue, 30 Aug 2022 08:33:32 -0600 Subject: [PATCH 4/7] Fix precision_scope for cuda, move seed_everything after load_model_from_config (might break all seeds?), fix torch.randn for cuda --- scripts/orig_scripts/img2img.py | 2 +- scripts/orig_scripts/txt2img.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/orig_scripts/img2img.py b/scripts/orig_scripts/img2img.py index 40bced669cb..f36144ac138 100644 --- a/scripts/orig_scripts/img2img.py +++ b/scripts/orig_scripts/img2img.py @@ -249,7 +249,7 @@ def main(): print(f"target t_enc is {t_enc} steps") precision_scope = autocast if opt.precision == "autocast" else nullcontext - if device.type == 'mps': + if device.type in ['mps', 'cpu']: precision_scope = nullcontext # have to use f32 on mps with torch.no_grad(): with precision_scope(device.type): diff --git a/scripts/orig_scripts/txt2img.py b/scripts/orig_scripts/txt2img.py index 18d0c426352..8520a0d32bb 100644 --- a/scripts/orig_scripts/txt2img.py +++ b/scripts/orig_scripts/txt2img.py @@ -198,11 +198,12 @@ def main(): opt.ckpt = "models/ldm/text2img-large/model.ckpt" opt.outdir = "outputs/txt2img-samples-laion400m" - seed_everything(opt.seed) config = OmegaConf.load(f"{opt.config}") model = load_model_from_config(config, f"{opt.ckpt}") + seed_everything(opt.seed) + device = torch.device(get_device()) model = model.to(device) @@ -248,12 +249,14 @@ def forward(self, x, sigma, uncond, cond, cond_scale): start_code = None if opt.fixed_code: - start_code = torch.randn( - [opt.n_samples, opt.C, opt.H // opt.f, opt.W // opt.f], device="cpu" - ).to(torch.device(device)) + shape = [opt.C, opt.H // opt.f, opt.W // opt.f] + if device.type == 'mps': + start_code = torch.randn(rand_size, device='cpu').to(device) + else: + torch.randn(rand_size, device=device) precision_scope = autocast if opt.precision=="autocast" else nullcontext - if device.type == 'mps': + if device.type in ['mps', 'cpu']: precision_scope = nullcontext # have to use f32 on mps with torch.no_grad(): with precision_scope(device.type): From a936f6d44d9de44fb906c9487ea2ae89cf7a6e51 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Tue, 30 Aug 2022 16:55:34 -0400 Subject: [PATCH 5/7] Update ddim.py Checking variables for None is more idiomatic than checking for empty strings. --- ldm/models/diffusion/ddim.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ldm/models/diffusion/ddim.py b/ldm/models/diffusion/ddim.py index b2f21749b9c..5c7bd660e11 100644 --- a/ldm/models/diffusion/ddim.py +++ b/ldm/models/diffusion/ddim.py @@ -14,12 +14,12 @@ class DDIMSampler(object): - def __init__(self, model, schedule='linear', device='', **kwargs): + def __init__(self, model, schedule='linear', device=None, **kwargs): super().__init__() self.model = model self.ddpm_num_timesteps = model.num_timesteps self.schedule = schedule - if device == '': + if not device: if torch.cuda.is_available(): self.device = 'cuda' elif torch.backends.mps.is_available(): From 403151534f58b48bc6187ae3322f9981672fefe8 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Tue, 30 Aug 2022 20:26:51 -0400 Subject: [PATCH 6/7] Update ldm/models/diffusion/ddim.py Co-authored-by: Benjamin Warner --- ldm/models/diffusion/ddim.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ldm/models/diffusion/ddim.py b/ldm/models/diffusion/ddim.py index 5c7bd660e11..83a54e0e115 100644 --- a/ldm/models/diffusion/ddim.py +++ b/ldm/models/diffusion/ddim.py @@ -32,7 +32,7 @@ def __init__(self, model, schedule='linear', device=None, **kwargs): def register_buffer(self, name, attr): if type(attr) == torch.Tensor: if attr.device != torch.device(self.device): - attr = attr.to(torch.float32).to(torch.device(self.device)) + attr = attr.to(dtype=torch.float32, device=self.device) setattr(self, name, attr) def make_schedule( From 10619ae4e011ef6378150b44b8b4dd0712e82eff Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Tue, 30 Aug 2022 23:39:33 -0400 Subject: [PATCH 7/7] Update scripts/orig_scripts/txt2img.py Make --fixed-code work again Co-authored-by: Beau Gunderson --- scripts/orig_scripts/txt2img.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/orig_scripts/txt2img.py b/scripts/orig_scripts/txt2img.py index 8520a0d32bb..e3aacd0dcdb 100644 --- a/scripts/orig_scripts/txt2img.py +++ b/scripts/orig_scripts/txt2img.py @@ -249,11 +249,11 @@ def forward(self, x, sigma, uncond, cond, cond_scale): start_code = None if opt.fixed_code: - shape = [opt.C, opt.H // opt.f, opt.W // opt.f] + shape = [opt.n_samples, opt.C, opt.H // opt.f, opt.W // opt.f] if device.type == 'mps': - start_code = torch.randn(rand_size, device='cpu').to(device) + start_code = torch.randn(shape, device='cpu').to(device) else: - torch.randn(rand_size, device=device) + torch.randn(shape, device=device) precision_scope = autocast if opt.precision=="autocast" else nullcontext if device.type in ['mps', 'cpu']: