From ff5341c85c48852fad19aa0138e8c64e8b6ccd80 Mon Sep 17 00:00:00 2001 From: limiteinductive Date: Tue, 12 Mar 2024 14:09:34 +0000 Subject: [PATCH] Change weight decay for Optimizer to normal PyTorch default --- src/refiners/training_utils/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/refiners/training_utils/config.py b/src/refiners/training_utils/config.py index 4098ea0..9c953a3 100644 --- a/src/refiners/training_utils/config.py +++ b/src/refiners/training_utils/config.py @@ -90,7 +90,7 @@ class OptimizerConfig(BaseModel): learning_rate: float = 1e-4 betas: tuple[float, float] = (0.9, 0.999) eps: float = 1e-8 - weight_decay: float = 0.0 + weight_decay: float = 1e-2 model_config = ConfigDict(extra="forbid")