ella: refresh README and mkdocs
Some checks are pending
CI / lint_and_typecheck (push) Waiting to run
Deploy docs to GitHub Pages / Deploy docs (push) Waiting to run
Spell checker / Spell check (push) Waiting to run

Follow up of #389
This commit is contained in:
Cédric Deltheil 2024-09-04 11:38:22 +00:00 committed by Cédric Deltheil
parent 277b0fd837
commit 428c2ddcec
4 changed files with 18 additions and 0 deletions

View file

@ -31,6 +31,7 @@ ______________________________________________________________________
## Latest News 🔥 ## Latest News 🔥
- Added [ELLA](https://arxiv.org/abs/2403.05135) for better prompts handling (contributed by [@ily-R](https://github.com/ily-R))
- Added the Box Segmenter all-in-one solution ([model](https://huggingface.co/finegrain/finegrain-box-segmenter), [HF Space](https://huggingface.co/spaces/finegrain/finegrain-object-cutter)) - Added the Box Segmenter all-in-one solution ([model](https://huggingface.co/finegrain/finegrain-box-segmenter), [HF Space](https://huggingface.co/spaces/finegrain/finegrain-object-cutter))
- Added [MVANet](https://arxiv.org/abs/2404.07445) for high resolution segmentation - Added [MVANet](https://arxiv.org/abs/2404.07445) for high resolution segmentation
- Added [IC-Light](https://github.com/lllyasviel/IC-Light) to manipulate the illumination of images - Added [IC-Light](https://github.com/lllyasviel/IC-Light) to manipulate the illumination of images

View file

@ -15,3 +15,5 @@
::: refiners.foundationals.latent_diffusion.style_aligned ::: refiners.foundationals.latent_diffusion.style_aligned
::: refiners.foundationals.latent_diffusion.multi_diffusion ::: refiners.foundationals.latent_diffusion.multi_diffusion
::: refiners.foundationals.latent_diffusion.ella_adapter

View file

@ -210,6 +210,11 @@ class PerceiverResampler(fl.Chain):
class ELLA(fl.Passthrough): class ELLA(fl.Passthrough):
"""ELLA latents encoder.
See [[arXiv:2403.05135] ELLA: Equip Diffusion Models with LLM for Enhanced Semantic Alignment](https://arxiv.org/abs/2403.05135) for more details.
"""
def __init__( def __init__(
self, self,
time_channel: int, time_channel: int,
@ -249,6 +254,8 @@ class ELLACrossAttentionAdapter(fl.Chain, Adapter[fl.UseContext]):
class ELLAAdapter(Generic[T], fl.Chain, Adapter[T]): class ELLAAdapter(Generic[T], fl.Chain, Adapter[T]):
"""Adapter for [`ELLA`][refiners.foundationals.latent_diffusion.ella_adapter.ELLA]."""
def __init__(self, target: T, latents_encoder: ELLA, weights: dict[str, Tensor] | None = None) -> None: def __init__(self, target: T, latents_encoder: ELLA, weights: dict[str, Tensor] | None = None) -> None:
if weights is not None: if weights is not None:
latents_encoder.load_state_dict(weights) latents_encoder.load_state_dict(weights)

View file

@ -5,7 +5,15 @@ from refiners.foundationals.latent_diffusion.stable_diffusion_1.unet import SD1U
class SD1ELLAAdapter(ELLAAdapter[SD1UNet]): class SD1ELLAAdapter(ELLAAdapter[SD1UNet]):
"""[`ELLA`][refiners.foundationals.latent_diffusion.ella_adapter.ELLA] adapter for Stable Diffusion 1.5."""
def __init__(self, target: SD1UNet, weights: dict[str, Tensor] | None = None) -> None: def __init__(self, target: SD1UNet, weights: dict[str, Tensor] | None = None) -> None:
"""Initialize the adapter.
Args:
target: The target model to adapt.
weights: The weights of the ELLA adapter (see `scripts/conversion/convert_ella_adapter.py`).
"""
latents_encoder = ELLA( latents_encoder = ELLA(
time_channel=320, time_channel=320,
timestep_embedding_dim=768, timestep_embedding_dim=768,