pbar on sec_evaluate

references/detection/train_pytorch.py

@@ -219,9 +219,13 @@ def sec_evaluate(model, val_loader, batch_transforms, val_metric, amp=False):
     model.eval()
     # Reset val metric
     val_metric.reset()
+    last_progress = 0
+    interval_progress = 5
+    pbar = tqdm(val_loader)
+    send_on_slack(str(pbar))
     # Validation loop
     val_loss, batch_cnt = 0, 0
-    for images, targets in tqdm(val_loader):
+    for images, targets in pbar:
         if torch.cuda.is_available():
             images = images.cuda()
         images = batch_transforms(images)
@@ -238,6 +242,10 @@ def sec_evaluate(model, val_loader, batch_transforms, val_metric, amp=False):
                 # Remove scores
                 val_metric.update(gts=boxes_gt, preds=boxes_pred[:, :-1])
 
+        current_progress = pbar.n / pbar.total * 100
+        if current_progress - last_progress > interval_progress:
+            send_on_slack(str(pbar))
+            last_progress = int(current_progress)
         val_loss += out["loss"].item()
         batch_cnt += 1