diff --git a/src/refiners/fluxion/layers/__init__.py b/src/refiners/fluxion/layers/__init__.py index 3cefd15..170bac6 100644 --- a/src/refiners/fluxion/layers/__init__.py +++ b/src/refiners/fluxion/layers/__init__.py @@ -1,4 +1,4 @@ -from refiners.fluxion.layers.activations import GLU, SiLU, ReLU, ApproximateGeLU, GeLU +from refiners.fluxion.layers.activations import GLU, SiLU, ReLU, ApproximateGeLU, GeLU, Sigmoid from refiners.fluxion.layers.norm import LayerNorm, GroupNorm, LayerNorm2d from refiners.fluxion.layers.attentions import Attention, SelfAttention, SelfAttention2d from refiners.fluxion.layers.basics import ( @@ -44,6 +44,7 @@ __all__ = [ "SiLU", "ReLU", "ApproximateGeLU", + "Sigmoid", "Attention", "SelfAttention", "SelfAttention2d", diff --git a/src/refiners/fluxion/layers/activations.py b/src/refiners/fluxion/layers/activations.py index b023199..eca9afd 100644 --- a/src/refiners/fluxion/layers/activations.py +++ b/src/refiners/fluxion/layers/activations.py @@ -46,6 +46,14 @@ class ApproximateGeLU(Activation): return x * sigmoid(1.702 * x) +class Sigmoid(Activation): + def __init__(self) -> None: + super().__init__() + + def forward(self, x: Tensor) -> Tensor: + return x.sigmoid() + + class GLU(Activation): """ Gated Linear Unit activation layer.