KPConv-PyTorch/utils/tester.py
2020-08-11 10:17:21 -04:00

799 lines
32 KiB
Python

#
#
# 0=================================0
# | Kernel Point Convolutions |
# 0=================================0
#
#
# ----------------------------------------------------------------------------------------------------------------------
#
# Class handling the test of any model
#
# ----------------------------------------------------------------------------------------------------------------------
#
# Hugues THOMAS - 11/06/2018
#
# ----------------------------------------------------------------------------------------------------------------------
#
# Imports and global variables
# \**********************************/
#
# Basic libs
import torch
import torch.nn as nn
import numpy as np
from os import makedirs, listdir
from os.path import exists, join
import time
import json
from sklearn.neighbors import KDTree
# PLY reader
from utils.ply import read_ply, write_ply
# Metrics
from utils.metrics import IoU_from_confusions, fast_confusion
from sklearn.metrics import confusion_matrix
#from utils.visualizer import show_ModelNet_models
# ----------------------------------------------------------------------------------------------------------------------
#
# Tester Class
# \******************/
#
class ModelTester:
# Initialization methods
# ------------------------------------------------------------------------------------------------------------------
def __init__(self, net, chkp_path=None, on_gpu=True):
############
# Parameters
############
# Choose to train on CPU or GPU
if on_gpu and torch.cuda.is_available():
self.device = torch.device("cuda:0")
else:
self.device = torch.device("cpu")
net.to(self.device)
##########################
# Load previous checkpoint
##########################
checkpoint = torch.load(chkp_path)
net.load_state_dict(checkpoint['model_state_dict'])
self.epoch = checkpoint['epoch']
net.eval()
print("Model and training state restored.")
return
# Test main methods
# ------------------------------------------------------------------------------------------------------------------
def classification_test(self, net, test_loader, config, num_votes=100, debug=False):
############
# Initialize
############
# Choose test smoothing parameter (0 for no smothing, 0.99 for big smoothing)
softmax = torch.nn.Softmax(1)
# Number of classes including ignored labels
nc_tot = test_loader.dataset.num_classes
# Number of classes predicted by the model
nc_model = config.num_classes
# Initiate global prediction over test clouds
self.test_probs = np.zeros((test_loader.dataset.num_models, nc_model))
self.test_counts = np.zeros((test_loader.dataset.num_models, nc_model))
t = [time.time()]
mean_dt = np.zeros(1)
last_display = time.time()
while np.min(self.test_counts) < num_votes:
# Run model on all test examples
# ******************************
# Initiate result containers
probs = []
targets = []
obj_inds = []
# Start validation loop
for batch in test_loader:
# New time
t = t[-1:]
t += [time.time()]
if 'cuda' in self.device.type:
batch.to(self.device)
# Forward pass
outputs = net(batch, config)
# Get probs and labels
probs += [softmax(outputs).cpu().detach().numpy()]
targets += [batch.labels.cpu().numpy()]
obj_inds += [batch.model_inds.cpu().numpy()]
if 'cuda' in self.device.type:
torch.cuda.synchronize(self.device)
# Average timing
t += [time.time()]
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1]))
# Display
if (t[-1] - last_display) > 1.0:
last_display = t[-1]
message = 'Test vote {:.0f} : {:.1f}% (timings : {:4.2f} {:4.2f})'
print(message.format(np.min(self.test_counts),
100 * len(obj_inds) / config.validation_size,
1000 * (mean_dt[0]),
1000 * (mean_dt[1])))
# Stack all validation predictions
probs = np.vstack(probs)
targets = np.hstack(targets)
obj_inds = np.hstack(obj_inds)
if np.any(test_loader.dataset.input_labels[obj_inds] != targets):
raise ValueError('wrong object indices')
# Compute incremental average (predictions are always ordered)
self.test_counts[obj_inds] += 1
self.test_probs[obj_inds] += (probs - self.test_probs[obj_inds]) / (self.test_counts[obj_inds])
# Save/Display temporary results
# ******************************
test_labels = np.array(test_loader.dataset.label_values)
# Compute classification results
C1 = fast_confusion(test_loader.dataset.input_labels,
np.argmax(self.test_probs, axis=1),
test_labels)
ACC = 100 * np.sum(np.diag(C1)) / (np.sum(C1) + 1e-6)
print('Test Accuracy = {:.1f}%'.format(ACC))
return
def cloud_segmentation_test(self, net, test_loader, config, num_votes=100, debug=False):
"""
Test method for cloud segmentation models
"""
############
# Initialize
############
# Choose test smoothing parameter (0 for no smothing, 0.99 for big smoothing)
test_smooth = 0.95
test_radius_ratio = 0.7
softmax = torch.nn.Softmax(1)
# Number of classes including ignored labels
nc_tot = test_loader.dataset.num_classes
# Number of classes predicted by the model
nc_model = config.num_classes
# Initiate global prediction over test clouds
self.test_probs = [np.zeros((l.shape[0], nc_model)) for l in test_loader.dataset.input_labels]
# Test saving path
if config.saving:
test_path = join('test', config.saving_path.split('/')[-1])
if not exists(test_path):
makedirs(test_path)
if not exists(join(test_path, 'predictions')):
makedirs(join(test_path, 'predictions'))
if not exists(join(test_path, 'probs')):
makedirs(join(test_path, 'probs'))
if not exists(join(test_path, 'potentials')):
makedirs(join(test_path, 'potentials'))
else:
test_path = None
# If on validation directly compute score
if test_loader.dataset.set == 'validation':
val_proportions = np.zeros(nc_model, dtype=np.float32)
i = 0
for label_value in test_loader.dataset.label_values:
if label_value not in test_loader.dataset.ignored_labels:
val_proportions[i] = np.sum([np.sum(labels == label_value)
for labels in test_loader.dataset.validation_labels])
i += 1
else:
val_proportions = None
#####################
# Network predictions
#####################
test_epoch = 0
last_min = -0.5
t = [time.time()]
last_display = time.time()
mean_dt = np.zeros(1)
# Start test loop
while True:
print('Initialize workers')
for i, batch in enumerate(test_loader):
# New time
t = t[-1:]
t += [time.time()]
if i == 0:
print('Done in {:.1f}s'.format(t[1] - t[0]))
if 'cuda' in self.device.type:
batch.to(self.device)
# Forward pass
outputs = net(batch, config)
t += [time.time()]
# Get probs and labels
stacked_probs = softmax(outputs).cpu().detach().numpy()
s_points = batch.points[0].cpu().numpy()
lengths = batch.lengths[0].cpu().numpy()
in_inds = batch.input_inds.cpu().numpy()
cloud_inds = batch.cloud_inds.cpu().numpy()
torch.cuda.synchronize(self.device)
# Get predictions and labels per instance
# ***************************************
i0 = 0
for b_i, length in enumerate(lengths):
# Get prediction
points = s_points[i0:i0 + length]
probs = stacked_probs[i0:i0 + length]
inds = in_inds[i0:i0 + length]
c_i = cloud_inds[b_i]
if 0 < test_radius_ratio < 1:
mask = np.sum(points ** 2, axis=1) < (test_radius_ratio * config.in_radius) ** 2
inds = inds[mask]
probs = probs[mask]
# Update current probs in whole cloud
self.test_probs[c_i][inds] = test_smooth * self.test_probs[c_i][inds] + (1 - test_smooth) * probs
i0 += length
# Average timing
t += [time.time()]
if i < 2:
mean_dt = np.array(t[1:]) - np.array(t[:-1])
else:
mean_dt = 0.9 * mean_dt + 0.1 * (np.array(t[1:]) - np.array(t[:-1]))
# Display
if (t[-1] - last_display) > 1.0:
last_display = t[-1]
message = 'e{:03d}-i{:04d} => {:.1f}% (timings : {:4.2f} {:4.2f} {:4.2f})'
print(message.format(test_epoch, i,
100 * i / config.validation_size,
1000 * (mean_dt[0]),
1000 * (mean_dt[1]),
1000 * (mean_dt[2])))
# Update minimum od potentials
new_min = torch.min(test_loader.dataset.min_potentials)
print('Test epoch {:d}, end. Min potential = {:.1f}'.format(test_epoch, new_min))
#print([np.mean(pots) for pots in test_loader.dataset.potentials])
# Save predicted cloud
if last_min + 1 < new_min:
# Update last_min
last_min += 1
# Show vote results (On subcloud so it is not the good values here)
if test_loader.dataset.set == 'validation':
print('\nConfusion on sub clouds')
Confs = []
for i, file_path in enumerate(test_loader.dataset.files):
# Insert false columns for ignored labels
probs = np.array(self.test_probs[i], copy=True)
for l_ind, label_value in enumerate(test_loader.dataset.label_values):
if label_value in test_loader.dataset.ignored_labels:
probs = np.insert(probs, l_ind, 0, axis=1)
# Predicted labels
preds = test_loader.dataset.label_values[np.argmax(probs, axis=1)].astype(np.int32)
# Targets
targets = test_loader.dataset.input_labels[i]
# Confs
Confs += [fast_confusion(targets, preds, test_loader.dataset.label_values)]
# Regroup confusions
C = np.sum(np.stack(Confs), axis=0).astype(np.float32)
# Remove ignored labels from confusions
for l_ind, label_value in reversed(list(enumerate(test_loader.dataset.label_values))):
if label_value in test_loader.dataset.ignored_labels:
C = np.delete(C, l_ind, axis=0)
C = np.delete(C, l_ind, axis=1)
# Rescale with the right number of point per class
C *= np.expand_dims(val_proportions / (np.sum(C, axis=1) + 1e-6), 1)
# Compute IoUs
IoUs = IoU_from_confusions(C)
mIoU = np.mean(IoUs)
s = '{:5.2f} | '.format(100 * mIoU)
for IoU in IoUs:
s += '{:5.2f} '.format(100 * IoU)
print(s + '\n')
# Save real IoU once in a while
if int(np.ceil(new_min)) % 10 == 0:
# Project predictions
print('\nReproject Vote #{:d}'.format(int(np.floor(new_min))))
t1 = time.time()
proj_probs = []
for i, file_path in enumerate(test_loader.dataset.files):
print(i, file_path, test_loader.dataset.test_proj[i].shape, self.test_probs[i].shape)
print(test_loader.dataset.test_proj[i].dtype, np.max(test_loader.dataset.test_proj[i]))
print(test_loader.dataset.test_proj[i][:5])
# Reproject probs on the evaluations points
probs = self.test_probs[i][test_loader.dataset.test_proj[i], :]
proj_probs += [probs]
t2 = time.time()
print('Done in {:.1f} s\n'.format(t2 - t1))
# Show vote results
if test_loader.dataset.set == 'validation':
print('Confusion on full clouds')
t1 = time.time()
Confs = []
for i, file_path in enumerate(test_loader.dataset.files):
# Insert false columns for ignored labels
for l_ind, label_value in enumerate(test_loader.dataset.label_values):
if label_value in test_loader.dataset.ignored_labels:
proj_probs[i] = np.insert(proj_probs[i], l_ind, 0, axis=1)
# Get the predicted labels
preds = test_loader.dataset.label_values[np.argmax(proj_probs[i], axis=1)].astype(np.int32)
# Confusion
targets = test_loader.dataset.validation_labels[i]
Confs += [fast_confusion(targets, preds, test_loader.dataset.label_values)]
t2 = time.time()
print('Done in {:.1f} s\n'.format(t2 - t1))
# Regroup confusions
C = np.sum(np.stack(Confs), axis=0)
# Remove ignored labels from confusions
for l_ind, label_value in reversed(list(enumerate(test_loader.dataset.label_values))):
if label_value in test_loader.dataset.ignored_labels:
C = np.delete(C, l_ind, axis=0)
C = np.delete(C, l_ind, axis=1)
IoUs = IoU_from_confusions(C)
mIoU = np.mean(IoUs)
s = '{:5.2f} | '.format(100 * mIoU)
for IoU in IoUs:
s += '{:5.2f} '.format(100 * IoU)
print('-' * len(s))
print(s)
print('-' * len(s) + '\n')
# Save predictions
print('Saving clouds')
t1 = time.time()
for i, file_path in enumerate(test_loader.dataset.files):
# Get file
points = test_loader.dataset.load_evaluation_points(file_path)
# Get the predicted labels
preds = test_loader.dataset.label_values[np.argmax(proj_probs[i], axis=1)].astype(np.int32)
# Save plys
cloud_name = file_path.split('/')[-1]
test_name = join(test_path, 'predictions', cloud_name)
write_ply(test_name,
[points, preds],
['x', 'y', 'z', 'preds'])
test_name2 = join(test_path, 'probs', cloud_name)
prob_names = ['_'.join(test_loader.dataset.label_to_names[label].split())
for label in test_loader.dataset.label_values]
write_ply(test_name2,
[points, proj_probs[i]],
['x', 'y', 'z'] + prob_names)
# Save potentials
pot_points = np.array(test_loader.dataset.pot_trees[i].data, copy=False)
pot_name = join(test_path, 'potentials', cloud_name)
pots = test_loader.dataset.potentials[i].numpy().astype(np.float32)
write_ply(pot_name,
[pot_points.astype(np.float32), pots],
['x', 'y', 'z', 'pots'])
# Save ascii preds
if test_loader.dataset.set == 'test':
if test_loader.dataset.name.startswith('Semantic3D'):
ascii_name = join(test_path, 'predictions', test_loader.dataset.ascii_files[cloud_name])
else:
ascii_name = join(test_path, 'predictions', cloud_name[:-4] + '.txt')
np.savetxt(ascii_name, preds, fmt='%d')
t2 = time.time()
print('Done in {:.1f} s\n'.format(t2 - t1))
test_epoch += 1
# Break when reaching number of desired votes
if last_min > num_votes:
break
return
def slam_segmentation_test(self, net, test_loader, config, num_votes=100, debug=True):
"""
Test method for slam segmentation models
"""
############
# Initialize
############
# Choose validation smoothing parameter (0 for no smothing, 0.99 for big smoothing)
test_smooth = 0.5
last_min = -0.5
softmax = torch.nn.Softmax(1)
# Number of classes including ignored labels
nc_tot = test_loader.dataset.num_classes
nc_model = net.C
# Test saving path
test_path = None
report_path = None
if config.saving:
test_path = join('test', config.saving_path.split('/')[-1])
if not exists(test_path):
makedirs(test_path)
report_path = join(test_path, 'reports')
if not exists(report_path):
makedirs(report_path)
if test_loader.dataset.set == 'validation':
for folder in ['val_predictions', 'val_probs']:
if not exists(join(test_path, folder)):
makedirs(join(test_path, folder))
else:
for folder in ['predictions', 'probs']:
if not exists(join(test_path, folder)):
makedirs(join(test_path, folder))
# Init validation container
all_f_preds = []
all_f_labels = []
if test_loader.dataset.set == 'validation':
for i, seq_frames in enumerate(test_loader.dataset.frames):
all_f_preds.append([np.zeros((0,), dtype=np.int32) for _ in seq_frames])
all_f_labels.append([np.zeros((0,), dtype=np.int32) for _ in seq_frames])
#####################
# Network predictions
#####################
predictions = []
targets = []
test_epoch = 0
t = [time.time()]
last_display = time.time()
mean_dt = np.zeros(1)
# Start test loop
while True:
print('Initialize workers')
for i, batch in enumerate(test_loader):
# New time
t = t[-1:]
t += [time.time()]
if i == 0:
print('Done in {:.1f}s'.format(t[1] - t[0]))
if 'cuda' in self.device.type:
batch.to(self.device)
# Forward pass
outputs = net(batch, config)
# Get probs and labels
stk_probs = softmax(outputs).cpu().detach().numpy()
lengths = batch.lengths[0].cpu().numpy()
f_inds = batch.frame_inds.cpu().numpy()
r_inds_list = batch.reproj_inds
r_mask_list = batch.reproj_masks
labels_list = batch.val_labels
torch.cuda.synchronize(self.device)
t += [time.time()]
# Get predictions and labels per instance
# ***************************************
i0 = 0
for b_i, length in enumerate(lengths):
# Get prediction
probs = stk_probs[i0:i0 + length]
proj_inds = r_inds_list[b_i]
proj_mask = r_mask_list[b_i]
frame_labels = labels_list[b_i]
s_ind = f_inds[b_i, 0]
f_ind = f_inds[b_i, 1]
# Project predictions on the frame points
proj_probs = probs[proj_inds]
# Safe check if only one point:
if proj_probs.ndim < 2:
proj_probs = np.expand_dims(proj_probs, 0)
# Save probs in a binary file (uint8 format for lighter weight)
seq_name = test_loader.dataset.sequences[s_ind]
if test_loader.dataset.set == 'validation':
folder = 'val_probs'
pred_folder = 'val_predictions'
else:
folder = 'probs'
pred_folder = 'predictions'
filename = '{:s}_{:07d}.npy'.format(seq_name, f_ind)
filepath = join(test_path, folder, filename)
if exists(filepath):
frame_probs_uint8 = np.load(filepath)
else:
frame_probs_uint8 = np.zeros((proj_mask.shape[0], nc_model), dtype=np.uint8)
frame_probs = frame_probs_uint8[proj_mask, :].astype(np.float32) / 255
frame_probs = test_smooth * frame_probs + (1 - test_smooth) * proj_probs
frame_probs_uint8[proj_mask, :] = (frame_probs * 255).astype(np.uint8)
np.save(filepath, frame_probs_uint8)
# Save some prediction in ply format for visual
if test_loader.dataset.set == 'validation':
# Insert false columns for ignored labels
frame_probs_uint8_bis = frame_probs_uint8.copy()
for l_ind, label_value in enumerate(test_loader.dataset.label_values):
if label_value in test_loader.dataset.ignored_labels:
frame_probs_uint8_bis = np.insert(frame_probs_uint8_bis, l_ind, 0, axis=1)
# Predicted labels
frame_preds = test_loader.dataset.label_values[np.argmax(frame_probs_uint8_bis,
axis=1)].astype(np.int32)
# Save some of the frame pots
if f_ind % 20 == 0:
seq_path = join(test_loader.dataset.path, 'sequences', test_loader.dataset.sequences[s_ind])
velo_file = join(seq_path, 'velodyne', test_loader.dataset.frames[s_ind][f_ind] + '.bin')
frame_points = np.fromfile(velo_file, dtype=np.float32)
frame_points = frame_points.reshape((-1, 4))
predpath = join(test_path, pred_folder, filename[:-4] + '.ply')
#pots = test_loader.dataset.f_potentials[s_ind][f_ind]
pots = np.zeros((0,))
if pots.shape[0] > 0:
write_ply(predpath,
[frame_points[:, :3], frame_labels, frame_preds, pots],
['x', 'y', 'z', 'gt', 'pre', 'pots'])
else:
write_ply(predpath,
[frame_points[:, :3], frame_labels, frame_preds],
['x', 'y', 'z', 'gt', 'pre'])
# Also Save lbl probabilities
probpath = join(test_path, folder, filename[:-4] + '_probs.ply')
lbl_names = [test_loader.dataset.label_to_names[l]
for l in test_loader.dataset.label_values
if l not in test_loader.dataset.ignored_labels]
write_ply(probpath,
[frame_points[:, :3], frame_probs_uint8],
['x', 'y', 'z'] + lbl_names)
# keep frame preds in memory
all_f_preds[s_ind][f_ind] = frame_preds
all_f_labels[s_ind][f_ind] = frame_labels
else:
# Save some of the frame preds
if f_inds[b_i, 1] % 100 == 0:
# Insert false columns for ignored labels
for l_ind, label_value in enumerate(test_loader.dataset.label_values):
if label_value in test_loader.dataset.ignored_labels:
frame_probs_uint8 = np.insert(frame_probs_uint8, l_ind, 0, axis=1)
# Predicted labels
frame_preds = test_loader.dataset.label_values[np.argmax(frame_probs_uint8,
axis=1)].astype(np.int32)
# Load points
seq_path = join(test_loader.dataset.path, 'sequences', test_loader.dataset.sequences[s_ind])
velo_file = join(seq_path, 'velodyne', test_loader.dataset.frames[s_ind][f_ind] + '.bin')
frame_points = np.fromfile(velo_file, dtype=np.float32)
frame_points = frame_points.reshape((-1, 4))
predpath = join(test_path, pred_folder, filename[:-4] + '.ply')
#pots = test_loader.dataset.f_potentials[s_ind][f_ind]
pots = np.zeros((0,))
if pots.shape[0] > 0:
write_ply(predpath,
[frame_points[:, :3], frame_preds, pots],
['x', 'y', 'z', 'pre', 'pots'])
else:
write_ply(predpath,
[frame_points[:, :3], frame_preds],
['x', 'y', 'z', 'pre'])
# Stack all prediction for this epoch
i0 += length
# Average timing
t += [time.time()]
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1]))
# Display
if (t[-1] - last_display) > 1.0:
last_display = t[-1]
message = 'e{:03d}-i{:04d} => {:.1f}% (timings : {:4.2f} {:4.2f} {:4.2f}) / pots {:d} => {:.1f}%'
min_pot = int(torch.floor(torch.min(test_loader.dataset.potentials)))
pot_num = torch.sum(test_loader.dataset.potentials > min_pot + 0.5).type(torch.int32).item()
current_num = pot_num + (i + 1 - config.validation_size) * config.val_batch_num
print(message.format(test_epoch, i,
100 * i / config.validation_size,
1000 * (mean_dt[0]),
1000 * (mean_dt[1]),
1000 * (mean_dt[2]),
min_pot,
100.0 * current_num / len(test_loader.dataset.potentials)))
# Update minimum od potentials
new_min = torch.min(test_loader.dataset.potentials)
print('Test epoch {:d}, end. Min potential = {:.1f}'.format(test_epoch, new_min))
if last_min + 1 < new_min:
# Update last_min
last_min += 1
if test_loader.dataset.set == 'validation' and last_min % 1 == 0:
#####################################
# Results on the whole validation set
#####################################
# Confusions for our subparts of validation set
Confs = np.zeros((len(predictions), nc_tot, nc_tot), dtype=np.int32)
for i, (preds, truth) in enumerate(zip(predictions, targets)):
# Confusions
Confs[i, :, :] = fast_confusion(truth, preds, test_loader.dataset.label_values).astype(np.int32)
# Show vote results
print('\nCompute confusion')
val_preds = []
val_labels = []
t1 = time.time()
for i, seq_frames in enumerate(test_loader.dataset.frames):
val_preds += [np.hstack(all_f_preds[i])]
val_labels += [np.hstack(all_f_labels[i])]
val_preds = np.hstack(val_preds)
val_labels = np.hstack(val_labels)
t2 = time.time()
C_tot = fast_confusion(val_labels, val_preds, test_loader.dataset.label_values)
t3 = time.time()
print(' Stacking time : {:.1f}s'.format(t2 - t1))
print('Confusion time : {:.1f}s'.format(t3 - t2))
s1 = '\n'
for cc in C_tot:
for c in cc:
s1 += '{:7.0f} '.format(c)
s1 += '\n'
if debug:
print(s1)
# Remove ignored labels from confusions
for l_ind, label_value in reversed(list(enumerate(test_loader.dataset.label_values))):
if label_value in test_loader.dataset.ignored_labels:
C_tot = np.delete(C_tot, l_ind, axis=0)
C_tot = np.delete(C_tot, l_ind, axis=1)
# Objects IoU
val_IoUs = IoU_from_confusions(C_tot)
# Compute IoUs
mIoU = np.mean(val_IoUs)
s2 = '{:5.2f} | '.format(100 * mIoU)
for IoU in val_IoUs:
s2 += '{:5.2f} '.format(100 * IoU)
print(s2 + '\n')
# Save a report
report_file = join(report_path, 'report_{:04d}.txt'.format(int(np.floor(last_min))))
str = 'Report of the confusion and metrics\n'
str += '***********************************\n\n\n'
str += 'Confusion matrix:\n\n'
str += s1
str += '\nIoU values:\n\n'
str += s2
str += '\n\n'
with open(report_file, 'w') as f:
f.write(str)
test_epoch += 1
# Break when reaching number of desired votes
if last_min > num_votes:
break
return