mirror of
https://github.com/finegrain-ai/refiners.git
synced 2024-11-21 21:58:47 +00:00
parent
277b0fd837
commit
428c2ddcec
|
@ -31,6 +31,7 @@ ______________________________________________________________________
|
||||||
|
|
||||||
## Latest News 🔥
|
## Latest News 🔥
|
||||||
|
|
||||||
|
- Added [ELLA](https://arxiv.org/abs/2403.05135) for better prompts handling (contributed by [@ily-R](https://github.com/ily-R))
|
||||||
- Added the Box Segmenter all-in-one solution ([model](https://huggingface.co/finegrain/finegrain-box-segmenter), [HF Space](https://huggingface.co/spaces/finegrain/finegrain-object-cutter))
|
- Added the Box Segmenter all-in-one solution ([model](https://huggingface.co/finegrain/finegrain-box-segmenter), [HF Space](https://huggingface.co/spaces/finegrain/finegrain-object-cutter))
|
||||||
- Added [MVANet](https://arxiv.org/abs/2404.07445) for high resolution segmentation
|
- Added [MVANet](https://arxiv.org/abs/2404.07445) for high resolution segmentation
|
||||||
- Added [IC-Light](https://github.com/lllyasviel/IC-Light) to manipulate the illumination of images
|
- Added [IC-Light](https://github.com/lllyasviel/IC-Light) to manipulate the illumination of images
|
||||||
|
|
|
@ -15,3 +15,5 @@
|
||||||
::: refiners.foundationals.latent_diffusion.style_aligned
|
::: refiners.foundationals.latent_diffusion.style_aligned
|
||||||
|
|
||||||
::: refiners.foundationals.latent_diffusion.multi_diffusion
|
::: refiners.foundationals.latent_diffusion.multi_diffusion
|
||||||
|
|
||||||
|
::: refiners.foundationals.latent_diffusion.ella_adapter
|
||||||
|
|
|
@ -210,6 +210,11 @@ class PerceiverResampler(fl.Chain):
|
||||||
|
|
||||||
|
|
||||||
class ELLA(fl.Passthrough):
|
class ELLA(fl.Passthrough):
|
||||||
|
"""ELLA latents encoder.
|
||||||
|
|
||||||
|
See [[arXiv:2403.05135] ELLA: Equip Diffusion Models with LLM for Enhanced Semantic Alignment](https://arxiv.org/abs/2403.05135) for more details.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
time_channel: int,
|
time_channel: int,
|
||||||
|
@ -249,6 +254,8 @@ class ELLACrossAttentionAdapter(fl.Chain, Adapter[fl.UseContext]):
|
||||||
|
|
||||||
|
|
||||||
class ELLAAdapter(Generic[T], fl.Chain, Adapter[T]):
|
class ELLAAdapter(Generic[T], fl.Chain, Adapter[T]):
|
||||||
|
"""Adapter for [`ELLA`][refiners.foundationals.latent_diffusion.ella_adapter.ELLA]."""
|
||||||
|
|
||||||
def __init__(self, target: T, latents_encoder: ELLA, weights: dict[str, Tensor] | None = None) -> None:
|
def __init__(self, target: T, latents_encoder: ELLA, weights: dict[str, Tensor] | None = None) -> None:
|
||||||
if weights is not None:
|
if weights is not None:
|
||||||
latents_encoder.load_state_dict(weights)
|
latents_encoder.load_state_dict(weights)
|
||||||
|
|
|
@ -5,7 +5,15 @@ from refiners.foundationals.latent_diffusion.stable_diffusion_1.unet import SD1U
|
||||||
|
|
||||||
|
|
||||||
class SD1ELLAAdapter(ELLAAdapter[SD1UNet]):
|
class SD1ELLAAdapter(ELLAAdapter[SD1UNet]):
|
||||||
|
"""[`ELLA`][refiners.foundationals.latent_diffusion.ella_adapter.ELLA] adapter for Stable Diffusion 1.5."""
|
||||||
|
|
||||||
def __init__(self, target: SD1UNet, weights: dict[str, Tensor] | None = None) -> None:
|
def __init__(self, target: SD1UNet, weights: dict[str, Tensor] | None = None) -> None:
|
||||||
|
"""Initialize the adapter.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
target: The target model to adapt.
|
||||||
|
weights: The weights of the ELLA adapter (see `scripts/conversion/convert_ella_adapter.py`).
|
||||||
|
"""
|
||||||
latents_encoder = ELLA(
|
latents_encoder = ELLA(
|
||||||
time_channel=320,
|
time_channel=320,
|
||||||
timestep_embedding_dim=768,
|
timestep_embedding_dim=768,
|
||||||
|
|
Loading…
Reference in a new issue