Finegrain/refiners
1
0
Fork 0
mirror of https://github.com/finegrain-ai/refiners.git synced 2024-11-22 06:08:46 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

make clip g use quick gelu and pad_token_id 0

Browse source
This commit is contained in:
limiteinductive 2023-08-15 16:44:45 +02:00 committed by Benjamin Trom
parent 6594502c11
commit b8e7179447
1 changed files with 3 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/refiners/foundationals/clip/text_encoder.py
View file

@ -247,11 +247,14 @@ class CLIPTextEncoderG(CLIPTextEncoder):
""" """
def __init__(self, device: Device | str | None = None, dtype: DType | None = None) -> None: def __init__(self, device: Device | str | None = None, dtype: DType | None = None) -> None:
tokenizer = CLIPTokenizer(pad_token_id=0)
super().__init__( super().__init__(
embedding_dim=1280, embedding_dim=1280,
num_layers=32, num_layers=32,
num_attention_heads=20, num_attention_heads=20,
feedforward_dim=5120, feedforward_dim=5120,
use_quick_gelu=True,
tokenizer=tokenizer,
device=device, device=device,
dtype=dtype, dtype=dtype,
) )