stable-diffusion-cli-on-modal/app/stable_diffusion_xl.py

from __future__ import annotations

import io
import os

import PIL.Image
from modal import Secret, enter, method
from setup import BASE_CACHE_PATH, app


@app.cls(
    gpu="A10G",
    secrets=[Secret.from_dotenv(__file__)],
)
class SDXLTxt2Img:
    """
    A class that wraps the Stable Diffusion pipeline and scheduler.
    """

    @enter()
    def _setup(self):
        import diffusers
        import torch
        import yaml

        config = {}
        with open("/config.yml", "r") as file:
            config = yaml.safe_load(file)
        self.cache_path = os.path.join(BASE_CACHE_PATH, config["model"]["name"])
        if os.path.exists(self.cache_path):
            print(f"The directory '{self.cache_path}' exists.")
        else:
            print(f"The directory '{self.cache_path}' does not exist.")

        self.pipe = diffusers.AutoPipelineForText2Image.from_pretrained(
            self.cache_path,
            torch_dtype=torch.float16,
            use_safetensors=True,
            variant="fp16",
        )

        self.refiner_cache_path = self.cache_path + "-refiner"
        self.refiner = diffusers.StableDiffusionXLImg2ImgPipeline.from_pretrained(
            self.refiner_cache_path,
            torch_dtype=torch.float16,
            use_safetensors=True,
            variant="fp16",
        )

        # controlnets = config.get("controlnets")
        # if controlnets is not None:
        #     for controlnet in controlnets:
        #         path = os.path.join(BASE_CACHE_PATH_CONTROLNET, controlnet["name"])
        #         controlnet = diffusers.ControlNetModel.from_pretrained(path, torch_dtype=torch.float16)
        #         self.controlnet_pipe = diffusers.StableDiffusionControlNetPipeline.from_pretrained(
        #             self.cache_path,
        #             controlnet=controlnet,
        #             custom_pipeline="lpw_stable_diffusion",
        #             scheduler=self.pipe.scheduler,
        #             vae=self.pipe.vae,
        #             torch_dtype=torch.float16,
        #             use_safetensors=True,
        #         )

    def _count_token(self, p: str, n: str) -> int:
        """
        Count the number of tokens in the prompt and negative prompt.
        """
        from transformers import CLIPTokenizer

        tokenizer = CLIPTokenizer.from_pretrained(
            self.cache_path,
            subfolder="tokenizer",
        )
        token_size_p = len(tokenizer.tokenize(p))
        token_size_n = len(tokenizer.tokenize(n))
        token_size = token_size_p
        if token_size_p <= token_size_n:
            token_size = token_size_n

        max_embeddings_multiples = 1
        max_length = tokenizer.model_max_length - 2
        if token_size > max_length:
            max_embeddings_multiples = token_size // max_length + 1

        print(f"token_size: {token_size}, max_embeddings_multiples: {max_embeddings_multiples}")

        return max_embeddings_multiples

    @method()
    def run_inference(
        self,
        prompt: str,
        n_prompt: str,
        height: int = 1024,
        width: int = 1024,
        steps: int = 30,
        seed: int = 1,
        use_upscaler: bool = False,
        output_format: str = "png",
    ) -> list[bytes]:
        """
        Runs the Stable Diffusion pipeline on the given prompt and outputs images.
        """
        import pillow_avif  # noqa
        import torch

        generator = torch.Generator("cuda").manual_seed(seed)
        self.pipe.to("cuda")
        generated_images = self.pipe(
            prompt=prompt,
            negative_prompt=n_prompt,
            height=height,
            width=width,
            generator=generator,
        ).images
        base_images = generated_images

        for image in base_images:
            image = self._resize_image(image=image, scale_factor=2)
            self.refiner.to("cuda")
            refined_images = self.refiner(
                prompt=prompt,
                negative_prompt=n_prompt,
                num_inference_steps=steps,
                strength=0.1,
                # guidance_scale=7.5,
                generator=generator,
                image=image,
            ).images
        generated_images.extend(refined_images)
        base_images = refined_images

        """
        Fix the generated images by the control_v11f1e_sd15_tile when `fix_by_controlnet_tile` is `True`.
        https://huggingface.co/lllyasviel/control_v11f1e_sd15_tile
        """
        # if fix_by_controlnet_tile:
        #     max_embeddings_multiples = self._count_token(p=prompt, n=n_prompt)
        #     self.controlnet_pipe.to("cuda")
        #     self.controlnet_pipe.enable_vae_tiling()
        #     self.controlnet_pipe.enable_xformers_memory_efficient_attention()
        #     for image in base_images:
        #         image = self._resize_image(image=image, scale_factor=2)
        #         with torch.autocast("cuda"):
        #             fixed_by_controlnet = self.controlnet_pipe(
        #                 prompt=prompt * batch_size,
        #                 negative_prompt=n_prompt * batch_size,
        #                 num_inference_steps=steps,
        #                 strength=0.3,
        #                 guidance_scale=7.5,
        #                 max_embeddings_multiples=max_embeddings_multiples,
        #                 generator=generator,
        #                 image=image,
        #             ).images
        #     generated_images.extend(fixed_by_controlnet)
        #     base_images = fixed_by_controlnet

        # if use_upscaler:
        #     upscaled = self._upscale(
        #         base_images=base_images,
        #         half_precision=False,
        #         tile=700,
        #         upscaler=upscaler,
        #     )
        #     generated_images.extend(upscaled)

        image_output = []
        for image in generated_images:
            with io.BytesIO() as buf:
                image.save(buf, format=output_format)
                image_output.append(buf.getvalue())

        return image_output

    def _resize_image(self, image: PIL.Image.Image, scale_factor: int) -> PIL.Image.Image:
        image = image.convert("RGB")
        width, height = image.size
        img = image.resize((width * scale_factor, height * scale_factor), resample=PIL.Image.LANCZOS)
        return img