Finegrain/refiners
1
0
Fork 0
mirror of https://github.com/finegrain-ai/refiners.git synced 2024-11-12 16:18:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix clock step inconsistencies on batch end

Browse source
This commit is contained in:
hugojarkoff 2024-04-05 13:44:59 +00:00 committed by hugojarkoff
parent 09af570b23
commit bbb46e3fc7
2 changed files with 2 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/refiners/training_utils/clock.py
View file

@ -179,10 +179,8 @@ class TrainingClock(Callback["Trainer[BaseConfig, Any]"]):
self.log(f"Iteration {trainer.clock.iteration} started.") self.log(f"Iteration {trainer.clock.iteration} started.")
self.log(f"Step {trainer.clock.step} started.") self.log(f"Step {trainer.clock.step} started.")
def on_batch_end(self, trainer: "Trainer[BaseConfig, Any]") -> None:
self.log(f"Step {trainer.clock.step} ended.")
def on_backward_end(self, trainer: "Trainer[BaseConfig, Any]") -> None: def on_backward_end(self, trainer: "Trainer[BaseConfig, Any]") -> None:
self.log(f"Step {trainer.clock.step} ended.")
trainer.clock.step += 1 trainer.clock.step += 1
trainer.clock.num_batches_processed += 1 trainer.clock.num_batches_processed += 1
trainer.clock.num_minibatches_processed += 1 trainer.clock.num_minibatches_processed += 1

2
src/refiners/training_utils/common.py
View file

@ -66,7 +66,7 @@ def scoped_seed(seed: int | Callable[..., int] | None = None) -> Callable[..., C
actual_seed = seed(*args) if callable(seed) else seed actual_seed = seed(*args) if callable(seed) else seed
seed_everything(seed=actual_seed) seed_everything(seed=actual_seed)
result = func(*args, **kwargs) result = func(*args, **kwargs)
logger.debug(f"Restoring previous seed state") logger.trace(f"Restoring previous seed state")
random.setstate(random_state) random.setstate(random_state)
np.random.set_state(numpy_state) np.random.set_state(numpy_state)
torch.set_rng_state(torch_state) torch.set_rng_state(torch_state)