mirror of
https://github.com/finegrain-ai/refiners.git
synced 2024-11-22 14:18:46 +00:00
make clip g use quick gelu and pad_token_id 0
This commit is contained in:
parent
6594502c11
commit
b8e7179447
|
@ -247,11 +247,14 @@ class CLIPTextEncoderG(CLIPTextEncoder):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, device: Device | str | None = None, dtype: DType | None = None) -> None:
|
def __init__(self, device: Device | str | None = None, dtype: DType | None = None) -> None:
|
||||||
|
tokenizer = CLIPTokenizer(pad_token_id=0)
|
||||||
super().__init__(
|
super().__init__(
|
||||||
embedding_dim=1280,
|
embedding_dim=1280,
|
||||||
num_layers=32,
|
num_layers=32,
|
||||||
num_attention_heads=20,
|
num_attention_heads=20,
|
||||||
feedforward_dim=5120,
|
feedforward_dim=5120,
|
||||||
|
use_quick_gelu=True,
|
||||||
|
tokenizer=tokenizer,
|
||||||
device=device,
|
device=device,
|
||||||
dtype=dtype,
|
dtype=dtype,
|
||||||
)
|
)
|
||||||
|
|
Loading…
Reference in a new issue