799 lines
32 KiB
Python
799 lines
32 KiB
Python
#
|
|
#
|
|
# 0=================================0
|
|
# | Kernel Point Convolutions |
|
|
# 0=================================0
|
|
#
|
|
#
|
|
# ----------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Class handling the test of any model
|
|
#
|
|
# ----------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Hugues THOMAS - 11/06/2018
|
|
#
|
|
|
|
|
|
# ----------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Imports and global variables
|
|
# \**********************************/
|
|
#
|
|
|
|
|
|
# Basic libs
|
|
import torch
|
|
import torch.nn as nn
|
|
import numpy as np
|
|
from os import makedirs, listdir
|
|
from os.path import exists, join
|
|
import time
|
|
import json
|
|
from sklearn.neighbors import KDTree
|
|
|
|
# PLY reader
|
|
from utils.ply import read_ply, write_ply
|
|
|
|
# Metrics
|
|
from utils.metrics import IoU_from_confusions, fast_confusion
|
|
from sklearn.metrics import confusion_matrix
|
|
|
|
#from utils.visualizer import show_ModelNet_models
|
|
|
|
# ----------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Tester Class
|
|
# \******************/
|
|
#
|
|
|
|
|
|
class ModelTester:
|
|
|
|
# Initialization methods
|
|
# ------------------------------------------------------------------------------------------------------------------
|
|
|
|
def __init__(self, net, chkp_path=None, on_gpu=True):
|
|
|
|
############
|
|
# Parameters
|
|
############
|
|
|
|
# Choose to train on CPU or GPU
|
|
if on_gpu and torch.cuda.is_available():
|
|
self.device = torch.device("cuda:0")
|
|
else:
|
|
self.device = torch.device("cpu")
|
|
net.to(self.device)
|
|
|
|
##########################
|
|
# Load previous checkpoint
|
|
##########################
|
|
|
|
checkpoint = torch.load(chkp_path)
|
|
net.load_state_dict(checkpoint['model_state_dict'])
|
|
self.epoch = checkpoint['epoch']
|
|
net.eval()
|
|
print("Model and training state restored.")
|
|
|
|
return
|
|
|
|
# Test main methods
|
|
# ------------------------------------------------------------------------------------------------------------------
|
|
|
|
def classification_test(self, net, test_loader, config, num_votes=100, debug=False):
|
|
|
|
############
|
|
# Initialize
|
|
############
|
|
|
|
# Choose test smoothing parameter (0 for no smothing, 0.99 for big smoothing)
|
|
softmax = torch.nn.Softmax(1)
|
|
|
|
# Number of classes including ignored labels
|
|
nc_tot = test_loader.dataset.num_classes
|
|
|
|
# Number of classes predicted by the model
|
|
nc_model = config.num_classes
|
|
|
|
# Initiate global prediction over test clouds
|
|
self.test_probs = np.zeros((test_loader.dataset.num_models, nc_model))
|
|
self.test_counts = np.zeros((test_loader.dataset.num_models, nc_model))
|
|
|
|
t = [time.time()]
|
|
mean_dt = np.zeros(1)
|
|
last_display = time.time()
|
|
while np.min(self.test_counts) < num_votes:
|
|
|
|
# Run model on all test examples
|
|
# ******************************
|
|
|
|
# Initiate result containers
|
|
probs = []
|
|
targets = []
|
|
obj_inds = []
|
|
|
|
# Start validation loop
|
|
for batch in test_loader:
|
|
|
|
# New time
|
|
t = t[-1:]
|
|
t += [time.time()]
|
|
|
|
if 'cuda' in self.device.type:
|
|
batch.to(self.device)
|
|
|
|
# Forward pass
|
|
outputs = net(batch, config)
|
|
|
|
# Get probs and labels
|
|
probs += [softmax(outputs).cpu().detach().numpy()]
|
|
targets += [batch.labels.cpu().numpy()]
|
|
obj_inds += [batch.model_inds.cpu().numpy()]
|
|
|
|
if 'cuda' in self.device.type:
|
|
torch.cuda.synchronize(self.device)
|
|
|
|
# Average timing
|
|
t += [time.time()]
|
|
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1]))
|
|
|
|
# Display
|
|
if (t[-1] - last_display) > 1.0:
|
|
last_display = t[-1]
|
|
message = 'Test vote {:.0f} : {:.1f}% (timings : {:4.2f} {:4.2f})'
|
|
print(message.format(np.min(self.test_counts),
|
|
100 * len(obj_inds) / config.validation_size,
|
|
1000 * (mean_dt[0]),
|
|
1000 * (mean_dt[1])))
|
|
# Stack all validation predictions
|
|
probs = np.vstack(probs)
|
|
targets = np.hstack(targets)
|
|
obj_inds = np.hstack(obj_inds)
|
|
|
|
if np.any(test_loader.dataset.input_labels[obj_inds] != targets):
|
|
raise ValueError('wrong object indices')
|
|
|
|
# Compute incremental average (predictions are always ordered)
|
|
self.test_counts[obj_inds] += 1
|
|
self.test_probs[obj_inds] += (probs - self.test_probs[obj_inds]) / (self.test_counts[obj_inds])
|
|
|
|
# Save/Display temporary results
|
|
# ******************************
|
|
|
|
test_labels = np.array(test_loader.dataset.label_values)
|
|
|
|
# Compute classification results
|
|
C1 = fast_confusion(test_loader.dataset.input_labels,
|
|
np.argmax(self.test_probs, axis=1),
|
|
test_labels)
|
|
|
|
ACC = 100 * np.sum(np.diag(C1)) / (np.sum(C1) + 1e-6)
|
|
print('Test Accuracy = {:.1f}%'.format(ACC))
|
|
|
|
return
|
|
|
|
def cloud_segmentation_test(self, net, test_loader, config, num_votes=100, debug=False):
|
|
"""
|
|
Test method for cloud segmentation models
|
|
"""
|
|
|
|
############
|
|
# Initialize
|
|
############
|
|
|
|
# Choose test smoothing parameter (0 for no smothing, 0.99 for big smoothing)
|
|
test_smooth = 0.95
|
|
test_radius_ratio = 0.7
|
|
softmax = torch.nn.Softmax(1)
|
|
|
|
# Number of classes including ignored labels
|
|
nc_tot = test_loader.dataset.num_classes
|
|
|
|
# Number of classes predicted by the model
|
|
nc_model = config.num_classes
|
|
|
|
# Initiate global prediction over test clouds
|
|
self.test_probs = [np.zeros((l.shape[0], nc_model)) for l in test_loader.dataset.input_labels]
|
|
|
|
# Test saving path
|
|
if config.saving:
|
|
test_path = join('test', config.saving_path.split('/')[-1])
|
|
if not exists(test_path):
|
|
makedirs(test_path)
|
|
if not exists(join(test_path, 'predictions')):
|
|
makedirs(join(test_path, 'predictions'))
|
|
if not exists(join(test_path, 'probs')):
|
|
makedirs(join(test_path, 'probs'))
|
|
if not exists(join(test_path, 'potentials')):
|
|
makedirs(join(test_path, 'potentials'))
|
|
else:
|
|
test_path = None
|
|
|
|
# If on validation directly compute score
|
|
if test_loader.dataset.set == 'validation':
|
|
val_proportions = np.zeros(nc_model, dtype=np.float32)
|
|
i = 0
|
|
for label_value in test_loader.dataset.label_values:
|
|
if label_value not in test_loader.dataset.ignored_labels:
|
|
val_proportions[i] = np.sum([np.sum(labels == label_value)
|
|
for labels in test_loader.dataset.validation_labels])
|
|
i += 1
|
|
else:
|
|
val_proportions = None
|
|
|
|
#####################
|
|
# Network predictions
|
|
#####################
|
|
|
|
test_epoch = 0
|
|
last_min = -0.5
|
|
|
|
t = [time.time()]
|
|
last_display = time.time()
|
|
mean_dt = np.zeros(1)
|
|
|
|
# Start test loop
|
|
while True:
|
|
print('Initialize workers')
|
|
for i, batch in enumerate(test_loader):
|
|
|
|
# New time
|
|
t = t[-1:]
|
|
t += [time.time()]
|
|
|
|
if i == 0:
|
|
print('Done in {:.1f}s'.format(t[1] - t[0]))
|
|
|
|
if 'cuda' in self.device.type:
|
|
batch.to(self.device)
|
|
|
|
# Forward pass
|
|
outputs = net(batch, config)
|
|
|
|
t += [time.time()]
|
|
|
|
# Get probs and labels
|
|
stacked_probs = softmax(outputs).cpu().detach().numpy()
|
|
s_points = batch.points[0].cpu().numpy()
|
|
lengths = batch.lengths[0].cpu().numpy()
|
|
in_inds = batch.input_inds.cpu().numpy()
|
|
cloud_inds = batch.cloud_inds.cpu().numpy()
|
|
torch.cuda.synchronize(self.device)
|
|
|
|
# Get predictions and labels per instance
|
|
# ***************************************
|
|
|
|
i0 = 0
|
|
for b_i, length in enumerate(lengths):
|
|
|
|
# Get prediction
|
|
points = s_points[i0:i0 + length]
|
|
probs = stacked_probs[i0:i0 + length]
|
|
inds = in_inds[i0:i0 + length]
|
|
c_i = cloud_inds[b_i]
|
|
|
|
if 0 < test_radius_ratio < 1:
|
|
mask = np.sum(points ** 2, axis=1) < (test_radius_ratio * config.in_radius) ** 2
|
|
inds = inds[mask]
|
|
probs = probs[mask]
|
|
|
|
# Update current probs in whole cloud
|
|
self.test_probs[c_i][inds] = test_smooth * self.test_probs[c_i][inds] + (1 - test_smooth) * probs
|
|
i0 += length
|
|
|
|
# Average timing
|
|
t += [time.time()]
|
|
if i < 2:
|
|
mean_dt = np.array(t[1:]) - np.array(t[:-1])
|
|
else:
|
|
mean_dt = 0.9 * mean_dt + 0.1 * (np.array(t[1:]) - np.array(t[:-1]))
|
|
|
|
# Display
|
|
if (t[-1] - last_display) > 1.0:
|
|
last_display = t[-1]
|
|
message = 'e{:03d}-i{:04d} => {:.1f}% (timings : {:4.2f} {:4.2f} {:4.2f})'
|
|
print(message.format(test_epoch, i,
|
|
100 * i / config.validation_size,
|
|
1000 * (mean_dt[0]),
|
|
1000 * (mean_dt[1]),
|
|
1000 * (mean_dt[2])))
|
|
|
|
# Update minimum od potentials
|
|
new_min = torch.min(test_loader.dataset.min_potentials)
|
|
print('Test epoch {:d}, end. Min potential = {:.1f}'.format(test_epoch, new_min))
|
|
#print([np.mean(pots) for pots in test_loader.dataset.potentials])
|
|
|
|
# Save predicted cloud
|
|
if last_min + 1 < new_min:
|
|
|
|
# Update last_min
|
|
last_min += 1
|
|
|
|
# Show vote results (On subcloud so it is not the good values here)
|
|
if test_loader.dataset.set == 'validation':
|
|
print('\nConfusion on sub clouds')
|
|
Confs = []
|
|
for i, file_path in enumerate(test_loader.dataset.files):
|
|
|
|
# Insert false columns for ignored labels
|
|
probs = np.array(self.test_probs[i], copy=True)
|
|
for l_ind, label_value in enumerate(test_loader.dataset.label_values):
|
|
if label_value in test_loader.dataset.ignored_labels:
|
|
probs = np.insert(probs, l_ind, 0, axis=1)
|
|
|
|
# Predicted labels
|
|
preds = test_loader.dataset.label_values[np.argmax(probs, axis=1)].astype(np.int32)
|
|
|
|
# Targets
|
|
targets = test_loader.dataset.input_labels[i]
|
|
|
|
# Confs
|
|
Confs += [fast_confusion(targets, preds, test_loader.dataset.label_values)]
|
|
|
|
# Regroup confusions
|
|
C = np.sum(np.stack(Confs), axis=0).astype(np.float32)
|
|
|
|
# Remove ignored labels from confusions
|
|
for l_ind, label_value in reversed(list(enumerate(test_loader.dataset.label_values))):
|
|
if label_value in test_loader.dataset.ignored_labels:
|
|
C = np.delete(C, l_ind, axis=0)
|
|
C = np.delete(C, l_ind, axis=1)
|
|
|
|
# Rescale with the right number of point per class
|
|
C *= np.expand_dims(val_proportions / (np.sum(C, axis=1) + 1e-6), 1)
|
|
|
|
# Compute IoUs
|
|
IoUs = IoU_from_confusions(C)
|
|
mIoU = np.mean(IoUs)
|
|
s = '{:5.2f} | '.format(100 * mIoU)
|
|
for IoU in IoUs:
|
|
s += '{:5.2f} '.format(100 * IoU)
|
|
print(s + '\n')
|
|
|
|
# Save real IoU once in a while
|
|
if int(np.ceil(new_min)) % 10 == 0:
|
|
|
|
# Project predictions
|
|
print('\nReproject Vote #{:d}'.format(int(np.floor(new_min))))
|
|
t1 = time.time()
|
|
proj_probs = []
|
|
for i, file_path in enumerate(test_loader.dataset.files):
|
|
|
|
print(i, file_path, test_loader.dataset.test_proj[i].shape, self.test_probs[i].shape)
|
|
|
|
print(test_loader.dataset.test_proj[i].dtype, np.max(test_loader.dataset.test_proj[i]))
|
|
print(test_loader.dataset.test_proj[i][:5])
|
|
|
|
# Reproject probs on the evaluations points
|
|
probs = self.test_probs[i][test_loader.dataset.test_proj[i], :]
|
|
proj_probs += [probs]
|
|
|
|
t2 = time.time()
|
|
print('Done in {:.1f} s\n'.format(t2 - t1))
|
|
|
|
# Show vote results
|
|
if test_loader.dataset.set == 'validation':
|
|
print('Confusion on full clouds')
|
|
t1 = time.time()
|
|
Confs = []
|
|
for i, file_path in enumerate(test_loader.dataset.files):
|
|
|
|
# Insert false columns for ignored labels
|
|
for l_ind, label_value in enumerate(test_loader.dataset.label_values):
|
|
if label_value in test_loader.dataset.ignored_labels:
|
|
proj_probs[i] = np.insert(proj_probs[i], l_ind, 0, axis=1)
|
|
|
|
# Get the predicted labels
|
|
preds = test_loader.dataset.label_values[np.argmax(proj_probs[i], axis=1)].astype(np.int32)
|
|
|
|
# Confusion
|
|
targets = test_loader.dataset.validation_labels[i]
|
|
Confs += [fast_confusion(targets, preds, test_loader.dataset.label_values)]
|
|
|
|
t2 = time.time()
|
|
print('Done in {:.1f} s\n'.format(t2 - t1))
|
|
|
|
# Regroup confusions
|
|
C = np.sum(np.stack(Confs), axis=0)
|
|
|
|
# Remove ignored labels from confusions
|
|
for l_ind, label_value in reversed(list(enumerate(test_loader.dataset.label_values))):
|
|
if label_value in test_loader.dataset.ignored_labels:
|
|
C = np.delete(C, l_ind, axis=0)
|
|
C = np.delete(C, l_ind, axis=1)
|
|
|
|
IoUs = IoU_from_confusions(C)
|
|
mIoU = np.mean(IoUs)
|
|
s = '{:5.2f} | '.format(100 * mIoU)
|
|
for IoU in IoUs:
|
|
s += '{:5.2f} '.format(100 * IoU)
|
|
print('-' * len(s))
|
|
print(s)
|
|
print('-' * len(s) + '\n')
|
|
|
|
# Save predictions
|
|
print('Saving clouds')
|
|
t1 = time.time()
|
|
for i, file_path in enumerate(test_loader.dataset.files):
|
|
|
|
# Get file
|
|
points = test_loader.dataset.load_evaluation_points(file_path)
|
|
|
|
# Get the predicted labels
|
|
preds = test_loader.dataset.label_values[np.argmax(proj_probs[i], axis=1)].astype(np.int32)
|
|
|
|
# Save plys
|
|
cloud_name = file_path.split('/')[-1]
|
|
test_name = join(test_path, 'predictions', cloud_name)
|
|
write_ply(test_name,
|
|
[points, preds],
|
|
['x', 'y', 'z', 'preds'])
|
|
test_name2 = join(test_path, 'probs', cloud_name)
|
|
prob_names = ['_'.join(test_loader.dataset.label_to_names[label].split())
|
|
for label in test_loader.dataset.label_values]
|
|
write_ply(test_name2,
|
|
[points, proj_probs[i]],
|
|
['x', 'y', 'z'] + prob_names)
|
|
|
|
# Save potentials
|
|
pot_points = np.array(test_loader.dataset.pot_trees[i].data, copy=False)
|
|
pot_name = join(test_path, 'potentials', cloud_name)
|
|
pots = test_loader.dataset.potentials[i].numpy().astype(np.float32)
|
|
write_ply(pot_name,
|
|
[pot_points.astype(np.float32), pots],
|
|
['x', 'y', 'z', 'pots'])
|
|
|
|
# Save ascii preds
|
|
if test_loader.dataset.set == 'test':
|
|
if test_loader.dataset.name.startswith('Semantic3D'):
|
|
ascii_name = join(test_path, 'predictions', test_loader.dataset.ascii_files[cloud_name])
|
|
else:
|
|
ascii_name = join(test_path, 'predictions', cloud_name[:-4] + '.txt')
|
|
np.savetxt(ascii_name, preds, fmt='%d')
|
|
|
|
t2 = time.time()
|
|
print('Done in {:.1f} s\n'.format(t2 - t1))
|
|
|
|
test_epoch += 1
|
|
|
|
# Break when reaching number of desired votes
|
|
if last_min > num_votes:
|
|
break
|
|
|
|
return
|
|
|
|
def slam_segmentation_test(self, net, test_loader, config, num_votes=100, debug=True):
|
|
"""
|
|
Test method for slam segmentation models
|
|
"""
|
|
|
|
############
|
|
# Initialize
|
|
############
|
|
|
|
# Choose validation smoothing parameter (0 for no smothing, 0.99 for big smoothing)
|
|
test_smooth = 0.5
|
|
last_min = -0.5
|
|
softmax = torch.nn.Softmax(1)
|
|
|
|
# Number of classes including ignored labels
|
|
nc_tot = test_loader.dataset.num_classes
|
|
nc_model = net.C
|
|
|
|
# Test saving path
|
|
test_path = None
|
|
report_path = None
|
|
if config.saving:
|
|
test_path = join('test', config.saving_path.split('/')[-1])
|
|
if not exists(test_path):
|
|
makedirs(test_path)
|
|
report_path = join(test_path, 'reports')
|
|
if not exists(report_path):
|
|
makedirs(report_path)
|
|
|
|
if test_loader.dataset.set == 'validation':
|
|
for folder in ['val_predictions', 'val_probs']:
|
|
if not exists(join(test_path, folder)):
|
|
makedirs(join(test_path, folder))
|
|
else:
|
|
for folder in ['predictions', 'probs']:
|
|
if not exists(join(test_path, folder)):
|
|
makedirs(join(test_path, folder))
|
|
|
|
# Init validation container
|
|
all_f_preds = []
|
|
all_f_labels = []
|
|
if test_loader.dataset.set == 'validation':
|
|
for i, seq_frames in enumerate(test_loader.dataset.frames):
|
|
all_f_preds.append([np.zeros((0,), dtype=np.int32) for _ in seq_frames])
|
|
all_f_labels.append([np.zeros((0,), dtype=np.int32) for _ in seq_frames])
|
|
|
|
#####################
|
|
# Network predictions
|
|
#####################
|
|
|
|
predictions = []
|
|
targets = []
|
|
test_epoch = 0
|
|
|
|
t = [time.time()]
|
|
last_display = time.time()
|
|
mean_dt = np.zeros(1)
|
|
|
|
# Start test loop
|
|
while True:
|
|
print('Initialize workers')
|
|
for i, batch in enumerate(test_loader):
|
|
|
|
# New time
|
|
t = t[-1:]
|
|
t += [time.time()]
|
|
|
|
if i == 0:
|
|
print('Done in {:.1f}s'.format(t[1] - t[0]))
|
|
|
|
if 'cuda' in self.device.type:
|
|
batch.to(self.device)
|
|
|
|
# Forward pass
|
|
outputs = net(batch, config)
|
|
|
|
# Get probs and labels
|
|
stk_probs = softmax(outputs).cpu().detach().numpy()
|
|
lengths = batch.lengths[0].cpu().numpy()
|
|
f_inds = batch.frame_inds.cpu().numpy()
|
|
r_inds_list = batch.reproj_inds
|
|
r_mask_list = batch.reproj_masks
|
|
labels_list = batch.val_labels
|
|
torch.cuda.synchronize(self.device)
|
|
|
|
t += [time.time()]
|
|
|
|
# Get predictions and labels per instance
|
|
# ***************************************
|
|
|
|
i0 = 0
|
|
for b_i, length in enumerate(lengths):
|
|
|
|
# Get prediction
|
|
probs = stk_probs[i0:i0 + length]
|
|
proj_inds = r_inds_list[b_i]
|
|
proj_mask = r_mask_list[b_i]
|
|
frame_labels = labels_list[b_i]
|
|
s_ind = f_inds[b_i, 0]
|
|
f_ind = f_inds[b_i, 1]
|
|
|
|
# Project predictions on the frame points
|
|
proj_probs = probs[proj_inds]
|
|
|
|
# Safe check if only one point:
|
|
if proj_probs.ndim < 2:
|
|
proj_probs = np.expand_dims(proj_probs, 0)
|
|
|
|
# Save probs in a binary file (uint8 format for lighter weight)
|
|
seq_name = test_loader.dataset.sequences[s_ind]
|
|
if test_loader.dataset.set == 'validation':
|
|
folder = 'val_probs'
|
|
pred_folder = 'val_predictions'
|
|
else:
|
|
folder = 'probs'
|
|
pred_folder = 'predictions'
|
|
filename = '{:s}_{:07d}.npy'.format(seq_name, f_ind)
|
|
filepath = join(test_path, folder, filename)
|
|
if exists(filepath):
|
|
frame_probs_uint8 = np.load(filepath)
|
|
else:
|
|
frame_probs_uint8 = np.zeros((proj_mask.shape[0], nc_model), dtype=np.uint8)
|
|
frame_probs = frame_probs_uint8[proj_mask, :].astype(np.float32) / 255
|
|
frame_probs = test_smooth * frame_probs + (1 - test_smooth) * proj_probs
|
|
frame_probs_uint8[proj_mask, :] = (frame_probs * 255).astype(np.uint8)
|
|
np.save(filepath, frame_probs_uint8)
|
|
|
|
# Save some prediction in ply format for visual
|
|
if test_loader.dataset.set == 'validation':
|
|
|
|
# Insert false columns for ignored labels
|
|
frame_probs_uint8_bis = frame_probs_uint8.copy()
|
|
for l_ind, label_value in enumerate(test_loader.dataset.label_values):
|
|
if label_value in test_loader.dataset.ignored_labels:
|
|
frame_probs_uint8_bis = np.insert(frame_probs_uint8_bis, l_ind, 0, axis=1)
|
|
|
|
# Predicted labels
|
|
frame_preds = test_loader.dataset.label_values[np.argmax(frame_probs_uint8_bis,
|
|
axis=1)].astype(np.int32)
|
|
|
|
# Save some of the frame pots
|
|
if f_ind % 20 == 0:
|
|
seq_path = join(test_loader.dataset.path, 'sequences', test_loader.dataset.sequences[s_ind])
|
|
velo_file = join(seq_path, 'velodyne', test_loader.dataset.frames[s_ind][f_ind] + '.bin')
|
|
frame_points = np.fromfile(velo_file, dtype=np.float32)
|
|
frame_points = frame_points.reshape((-1, 4))
|
|
predpath = join(test_path, pred_folder, filename[:-4] + '.ply')
|
|
#pots = test_loader.dataset.f_potentials[s_ind][f_ind]
|
|
pots = np.zeros((0,))
|
|
if pots.shape[0] > 0:
|
|
write_ply(predpath,
|
|
[frame_points[:, :3], frame_labels, frame_preds, pots],
|
|
['x', 'y', 'z', 'gt', 'pre', 'pots'])
|
|
else:
|
|
write_ply(predpath,
|
|
[frame_points[:, :3], frame_labels, frame_preds],
|
|
['x', 'y', 'z', 'gt', 'pre'])
|
|
|
|
# Also Save lbl probabilities
|
|
probpath = join(test_path, folder, filename[:-4] + '_probs.ply')
|
|
lbl_names = [test_loader.dataset.label_to_names[l]
|
|
for l in test_loader.dataset.label_values
|
|
if l not in test_loader.dataset.ignored_labels]
|
|
write_ply(probpath,
|
|
[frame_points[:, :3], frame_probs_uint8],
|
|
['x', 'y', 'z'] + lbl_names)
|
|
|
|
# keep frame preds in memory
|
|
all_f_preds[s_ind][f_ind] = frame_preds
|
|
all_f_labels[s_ind][f_ind] = frame_labels
|
|
|
|
else:
|
|
|
|
# Save some of the frame preds
|
|
if f_inds[b_i, 1] % 100 == 0:
|
|
|
|
# Insert false columns for ignored labels
|
|
for l_ind, label_value in enumerate(test_loader.dataset.label_values):
|
|
if label_value in test_loader.dataset.ignored_labels:
|
|
frame_probs_uint8 = np.insert(frame_probs_uint8, l_ind, 0, axis=1)
|
|
|
|
# Predicted labels
|
|
frame_preds = test_loader.dataset.label_values[np.argmax(frame_probs_uint8,
|
|
axis=1)].astype(np.int32)
|
|
|
|
# Load points
|
|
seq_path = join(test_loader.dataset.path, 'sequences', test_loader.dataset.sequences[s_ind])
|
|
velo_file = join(seq_path, 'velodyne', test_loader.dataset.frames[s_ind][f_ind] + '.bin')
|
|
frame_points = np.fromfile(velo_file, dtype=np.float32)
|
|
frame_points = frame_points.reshape((-1, 4))
|
|
predpath = join(test_path, pred_folder, filename[:-4] + '.ply')
|
|
#pots = test_loader.dataset.f_potentials[s_ind][f_ind]
|
|
pots = np.zeros((0,))
|
|
if pots.shape[0] > 0:
|
|
write_ply(predpath,
|
|
[frame_points[:, :3], frame_preds, pots],
|
|
['x', 'y', 'z', 'pre', 'pots'])
|
|
else:
|
|
write_ply(predpath,
|
|
[frame_points[:, :3], frame_preds],
|
|
['x', 'y', 'z', 'pre'])
|
|
|
|
# Stack all prediction for this epoch
|
|
i0 += length
|
|
|
|
# Average timing
|
|
t += [time.time()]
|
|
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1]))
|
|
|
|
# Display
|
|
if (t[-1] - last_display) > 1.0:
|
|
last_display = t[-1]
|
|
message = 'e{:03d}-i{:04d} => {:.1f}% (timings : {:4.2f} {:4.2f} {:4.2f}) / pots {:d} => {:.1f}%'
|
|
min_pot = int(torch.floor(torch.min(test_loader.dataset.potentials)))
|
|
pot_num = torch.sum(test_loader.dataset.potentials > min_pot + 0.5).type(torch.int32).item()
|
|
current_num = pot_num + (i + 1 - config.validation_size) * config.val_batch_num
|
|
print(message.format(test_epoch, i,
|
|
100 * i / config.validation_size,
|
|
1000 * (mean_dt[0]),
|
|
1000 * (mean_dt[1]),
|
|
1000 * (mean_dt[2]),
|
|
min_pot,
|
|
100.0 * current_num / len(test_loader.dataset.potentials)))
|
|
|
|
|
|
# Update minimum od potentials
|
|
new_min = torch.min(test_loader.dataset.potentials)
|
|
print('Test epoch {:d}, end. Min potential = {:.1f}'.format(test_epoch, new_min))
|
|
|
|
if last_min + 1 < new_min:
|
|
|
|
# Update last_min
|
|
last_min += 1
|
|
|
|
if test_loader.dataset.set == 'validation' and last_min % 1 == 0:
|
|
|
|
#####################################
|
|
# Results on the whole validation set
|
|
#####################################
|
|
|
|
# Confusions for our subparts of validation set
|
|
Confs = np.zeros((len(predictions), nc_tot, nc_tot), dtype=np.int32)
|
|
for i, (preds, truth) in enumerate(zip(predictions, targets)):
|
|
|
|
# Confusions
|
|
Confs[i, :, :] = fast_confusion(truth, preds, test_loader.dataset.label_values).astype(np.int32)
|
|
|
|
|
|
# Show vote results
|
|
print('\nCompute confusion')
|
|
|
|
val_preds = []
|
|
val_labels = []
|
|
t1 = time.time()
|
|
for i, seq_frames in enumerate(test_loader.dataset.frames):
|
|
val_preds += [np.hstack(all_f_preds[i])]
|
|
val_labels += [np.hstack(all_f_labels[i])]
|
|
val_preds = np.hstack(val_preds)
|
|
val_labels = np.hstack(val_labels)
|
|
t2 = time.time()
|
|
C_tot = fast_confusion(val_labels, val_preds, test_loader.dataset.label_values)
|
|
t3 = time.time()
|
|
print(' Stacking time : {:.1f}s'.format(t2 - t1))
|
|
print('Confusion time : {:.1f}s'.format(t3 - t2))
|
|
|
|
s1 = '\n'
|
|
for cc in C_tot:
|
|
for c in cc:
|
|
s1 += '{:7.0f} '.format(c)
|
|
s1 += '\n'
|
|
if debug:
|
|
print(s1)
|
|
|
|
# Remove ignored labels from confusions
|
|
for l_ind, label_value in reversed(list(enumerate(test_loader.dataset.label_values))):
|
|
if label_value in test_loader.dataset.ignored_labels:
|
|
C_tot = np.delete(C_tot, l_ind, axis=0)
|
|
C_tot = np.delete(C_tot, l_ind, axis=1)
|
|
|
|
# Objects IoU
|
|
val_IoUs = IoU_from_confusions(C_tot)
|
|
|
|
# Compute IoUs
|
|
mIoU = np.mean(val_IoUs)
|
|
s2 = '{:5.2f} | '.format(100 * mIoU)
|
|
for IoU in val_IoUs:
|
|
s2 += '{:5.2f} '.format(100 * IoU)
|
|
print(s2 + '\n')
|
|
|
|
# Save a report
|
|
report_file = join(report_path, 'report_{:04d}.txt'.format(int(np.floor(last_min))))
|
|
str = 'Report of the confusion and metrics\n'
|
|
str += '***********************************\n\n\n'
|
|
str += 'Confusion matrix:\n\n'
|
|
str += s1
|
|
str += '\nIoU values:\n\n'
|
|
str += s2
|
|
str += '\n\n'
|
|
with open(report_file, 'w') as f:
|
|
f.write(str)
|
|
|
|
test_epoch += 1
|
|
|
|
# Break when reaching number of desired votes
|
|
if last_min > num_votes:
|
|
break
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|