From b8e7179447f51258c89af9355489b3aff879a334 Mon Sep 17 00:00:00 2001 From: limiteinductive Date: Tue, 15 Aug 2023 16:44:45 +0200 Subject: [PATCH] make clip g use quick gelu and pad_token_id 0 --- src/refiners/foundationals/clip/text_encoder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/refiners/foundationals/clip/text_encoder.py b/src/refiners/foundationals/clip/text_encoder.py index a3b551b..c35c912 100644 --- a/src/refiners/foundationals/clip/text_encoder.py +++ b/src/refiners/foundationals/clip/text_encoder.py @@ -247,11 +247,14 @@ class CLIPTextEncoderG(CLIPTextEncoder): """ def __init__(self, device: Device | str | None = None, dtype: DType | None = None) -> None: + tokenizer = CLIPTokenizer(pad_token_id=0) super().__init__( embedding_dim=1280, num_layers=32, num_attention_heads=20, feedforward_dim=5120, + use_quick_gelu=True, + tokenizer=tokenizer, device=device, dtype=dtype, )