Laurent/REVA-QCAV
1
0
Fork 0
mirror of https://github.com/Laurent2916/REVA-QCAV.git synced 2024-11-09 23:12:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: validate every 100 steps

Browse source 
Former-commit-id: a3367de4ed56c5a708d66e7cd6be27d52bb92ccc [formerly 4606a91526eae57d56fc93df7ed34b867495e1c5]
Former-commit-id: 6584449cd25b18ddd46f6804c8b1653e1c72dda0
This commit is contained in:
Laurent Fainsin 2022-07-01 15:31:53 +02:00
parent cf8f52735a
commit 2571e5c6d3
1 changed files with 73 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
src/train.py
View file

@ -137,7 +137,7 @@ if __name__ == "__main__":
wandb.run.log_artifact(artifact) wandb.run.log_artifact(artifact)
# log gradients and weights four time per epoch # log gradients and weights four time per epoch
wandb.watch(net, log_freq=(len(train_loader) + len(val_loader)) // 4) wandb.watch(net, criterion, log_freq=100)
# print the config # print the config
logging.info(f"wandb config:\n{yaml.dump(wandb.config.as_dict())}") logging.info(f"wandb config:\n{yaml.dump(wandb.config.as_dict())}")
@ -198,13 +198,14 @@ if __name__ == "__main__":
} }
) )
if step and (step % 100 == 0 or step == len(train_loader)):
# Evaluation round # Evaluation round
net.eval() net.eval()
accuracy = 0 accuracy = 0
val_loss = 0 val_loss = 0
dice = 0 dice = 0
mae = 0 mae = 0
with tqdm(val_loader, total=len(ds_valid), desc="val", unit="img", leave=False) as pbar: with tqdm(val_loader, total=len(ds_valid), desc="val", unit="img", leave=False) as pbar2:
for images, masks_true in val_loader: for images, masks_true in val_loader:
# transfer images to device # transfer images to device
@ -223,7 +224,7 @@ if __name__ == "__main__":
dice += dice_coeff(masks_pred_bin, masks_true) dice += dice_coeff(masks_pred_bin, masks_true)
# update progress bar # update progress bar
pbar.update(images.shape[0]) pbar2.update(images.shape[0])
accuracy /= len(val_loader) accuracy /= len(val_loader)
val_loss /= len(val_loader) val_loss /= len(val_loader)
@ -272,17 +273,11 @@ if __name__ == "__main__":
net.train() net.train()
scheduler.step(dice) scheduler.step(dice)
# save weights when epoch end # export model to onnx format when validation ends
torch.save(net.state_dict(), f"checkpoints/model-{epoch}.pth")
artifact = wandb.Artifact("pth", type="model")
artifact.add_file(f"checkpoints/model-{epoch}.pth")
wandb.run.log_artifact(artifact)
# export model to onnx format
dummy_input = torch.randn(1, 3, 512, 512, requires_grad=True).to(device) dummy_input = torch.randn(1, 3, 512, 512, requires_grad=True).to(device)
torch.onnx.export(net, dummy_input, f"checkpoints/model-{epoch}.onnx") torch.onnx.export(net, dummy_input, f"checkpoints/model-{epoch}-{step}.onnx")
artifact = wandb.Artifact("onnx", type="model") artifact = wandb.Artifact("onnx", type="model")
artifact.add_file(f"checkpoints/model-{epoch}.onnx") artifact.add_file(f"checkpoints/model-{epoch}-{step}.onnx")
wandb.run.log_artifact(artifact) wandb.run.log_artifact(artifact)
# stop wandb # stop wandb