Use bias correction on Prodigy

2024-11-22 06:08:46 +00:00 · 2023-09-11 23:14:56 +02:00 · 2023-09-11 23:14:56 +02:00 · 2f2510a9b1
parent 9364c0ea1c
commit 2f2510a9b1
1 changed files with 1 additions and 0 deletions
--- a/src/refiners/training_utils/config.py
+++ b/src/refiners/training_utils/config.py
@ -165,6 +165,7 @@ class OptimizerConfig(BaseModel):
                    betas=self.betas,
                    weight_decay=self.weight_decay,  # type: ignore
                    safeguard_warmup=True,
+                    use_bias_correction=True,  # recommended for diffusion models
                )