mirror of
https://github.com/finegrain-ai/refiners.git
synced 2024-11-09 23:12:02 +00:00
add Sigmoid activation
This commit is contained in:
parent
f49bb4f5fd
commit
bd49304fc8
|
@ -1,4 +1,4 @@
|
|||
from refiners.fluxion.layers.activations import GLU, SiLU, ReLU, ApproximateGeLU, GeLU
|
||||
from refiners.fluxion.layers.activations import GLU, SiLU, ReLU, ApproximateGeLU, GeLU, Sigmoid
|
||||
from refiners.fluxion.layers.norm import LayerNorm, GroupNorm, LayerNorm2d
|
||||
from refiners.fluxion.layers.attentions import Attention, SelfAttention, SelfAttention2d
|
||||
from refiners.fluxion.layers.basics import (
|
||||
|
@ -44,6 +44,7 @@ __all__ = [
|
|||
"SiLU",
|
||||
"ReLU",
|
||||
"ApproximateGeLU",
|
||||
"Sigmoid",
|
||||
"Attention",
|
||||
"SelfAttention",
|
||||
"SelfAttention2d",
|
||||
|
|
|
@ -46,6 +46,14 @@ class ApproximateGeLU(Activation):
|
|||
return x * sigmoid(1.702 * x)
|
||||
|
||||
|
||||
class Sigmoid(Activation):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
return x.sigmoid()
|
||||
|
||||
|
||||
class GLU(Activation):
|
||||
"""
|
||||
Gated Linear Unit activation layer.
|
||||
|
|
Loading…
Reference in a new issue