# # # 0=================================0 # | Kernel Point Convolutions | # 0=================================0 # # # ---------------------------------------------------------------------------------------------------------------------- # # Class handling the test of any model # # ---------------------------------------------------------------------------------------------------------------------- # # Hugues THOMAS - 11/06/2018 # # ---------------------------------------------------------------------------------------------------------------------- # # Imports and global variables # \**********************************/ # # Basic libs import torch import torch.nn as nn import numpy as np from os import makedirs, listdir from os.path import exists, join import time import json from sklearn.neighbors import KDTree # PLY reader from utils.ply import read_ply, write_ply # Metrics from utils.metrics import IoU_from_confusions, fast_confusion from sklearn.metrics import confusion_matrix #from utils.visualizer import show_ModelNet_models # ---------------------------------------------------------------------------------------------------------------------- # # Tester Class # \******************/ # class ModelTester: # Initialization methods # ------------------------------------------------------------------------------------------------------------------ def __init__(self, net, chkp_path=None, on_gpu=True): ############ # Parameters ############ # Choose to train on CPU or GPU if on_gpu and torch.cuda.is_available(): self.device = torch.device("cuda:0") else: self.device = torch.device("cpu") net.to(self.device) ########################## # Load previous checkpoint ########################## checkpoint = torch.load(chkp_path) net.load_state_dict(checkpoint['model_state_dict']) self.epoch = checkpoint['epoch'] net.eval() print("Model and training state restored.") return # Test main methods # ------------------------------------------------------------------------------------------------------------------ def classification_test(self, net, test_loader, config, num_votes=100, debug=False): ############ # Initialize ############ # Choose test smoothing parameter (0 for no smothing, 0.99 for big smoothing) softmax = torch.nn.Softmax(1) # Number of classes including ignored labels nc_tot = test_loader.dataset.num_classes # Number of classes predicted by the model nc_model = config.num_classes # Initiate global prediction over test clouds self.test_probs = np.zeros((test_loader.dataset.num_models, nc_model)) self.test_counts = np.zeros((test_loader.dataset.num_models, nc_model)) t = [time.time()] mean_dt = np.zeros(1) last_display = time.time() while np.min(self.test_counts) < num_votes: # Run model on all test examples # ****************************** # Initiate result containers probs = [] targets = [] obj_inds = [] # Start validation loop for batch in test_loader: # New time t = t[-1:] t += [time.time()] if 'cuda' in self.device.type: batch.to(self.device) # Forward pass outputs = net(batch, config) # Get probs and labels probs += [softmax(outputs).cpu().detach().numpy()] targets += [batch.labels.cpu().numpy()] obj_inds += [batch.model_inds.cpu().numpy()] if 'cuda' in self.device.type: torch.cuda.synchronize(self.device) # Average timing t += [time.time()] mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1])) # Display if (t[-1] - last_display) > 1.0: last_display = t[-1] message = 'Test vote {:.0f} : {:.1f}% (timings : {:4.2f} {:4.2f})' print(message.format(np.min(self.test_counts), 100 * len(obj_inds) / config.validation_size, 1000 * (mean_dt[0]), 1000 * (mean_dt[1]))) # Stack all validation predictions probs = np.vstack(probs) targets = np.hstack(targets) obj_inds = np.hstack(obj_inds) if np.any(test_loader.dataset.input_labels[obj_inds] != targets): raise ValueError('wrong object indices') # Compute incremental average (predictions are always ordered) self.test_counts[obj_inds] += 1 self.test_probs[obj_inds] += (probs - self.test_probs[obj_inds]) / (self.test_counts[obj_inds]) # Save/Display temporary results # ****************************** test_labels = np.array(test_loader.dataset.label_values) # Compute classification results C1 = fast_confusion(test_loader.dataset.input_labels, np.argmax(self.test_probs, axis=1), test_labels) ACC = 100 * np.sum(np.diag(C1)) / (np.sum(C1) + 1e-6) print('Test Accuracy = {:.1f}%'.format(ACC)) return def cloud_segmentation_test(self, net, test_loader, config, num_votes=100, debug=False): """ Test method for cloud segmentation models """ ############ # Initialize ############ # Choose test smoothing parameter (0 for no smothing, 0.99 for big smoothing) test_smooth = 0.95 test_radius_ratio = 0.7 softmax = torch.nn.Softmax(1) # Number of classes including ignored labels nc_tot = test_loader.dataset.num_classes # Number of classes predicted by the model nc_model = config.num_classes # Initiate global prediction over test clouds self.test_probs = [np.zeros((l.shape[0], nc_model)) for l in test_loader.dataset.input_labels] # Test saving path if config.saving: test_path = join('test', config.saving_path.split('/')[-1]) if not exists(test_path): makedirs(test_path) if not exists(join(test_path, 'predictions')): makedirs(join(test_path, 'predictions')) if not exists(join(test_path, 'probs')): makedirs(join(test_path, 'probs')) if not exists(join(test_path, 'potentials')): makedirs(join(test_path, 'potentials')) else: test_path = None # If on validation directly compute score if test_loader.dataset.set == 'validation': val_proportions = np.zeros(nc_model, dtype=np.float32) i = 0 for label_value in test_loader.dataset.label_values: if label_value not in test_loader.dataset.ignored_labels: val_proportions[i] = np.sum([np.sum(labels == label_value) for labels in test_loader.dataset.validation_labels]) i += 1 else: val_proportions = None ##################### # Network predictions ##################### test_epoch = 0 last_min = -0.5 t = [time.time()] last_display = time.time() mean_dt = np.zeros(1) # Start test loop while True: print('Initialize workers') for i, batch in enumerate(test_loader): # New time t = t[-1:] t += [time.time()] if i == 0: print('Done in {:.1f}s'.format(t[1] - t[0])) if 'cuda' in self.device.type: batch.to(self.device) # Forward pass outputs = net(batch, config) t += [time.time()] # Get probs and labels stacked_probs = softmax(outputs).cpu().detach().numpy() s_points = batch.points[0].cpu().numpy() lengths = batch.lengths[0].cpu().numpy() in_inds = batch.input_inds.cpu().numpy() cloud_inds = batch.cloud_inds.cpu().numpy() torch.cuda.synchronize(self.device) # Get predictions and labels per instance # *************************************** i0 = 0 for b_i, length in enumerate(lengths): # Get prediction points = s_points[i0:i0 + length] probs = stacked_probs[i0:i0 + length] inds = in_inds[i0:i0 + length] c_i = cloud_inds[b_i] if 0 < test_radius_ratio < 1: mask = np.sum(points ** 2, axis=1) < (test_radius_ratio * config.in_radius) ** 2 inds = inds[mask] probs = probs[mask] # Update current probs in whole cloud self.test_probs[c_i][inds] = test_smooth * self.test_probs[c_i][inds] + (1 - test_smooth) * probs i0 += length # Average timing t += [time.time()] if i < 2: mean_dt = np.array(t[1:]) - np.array(t[:-1]) else: mean_dt = 0.9 * mean_dt + 0.1 * (np.array(t[1:]) - np.array(t[:-1])) # Display if (t[-1] - last_display) > 1.0: last_display = t[-1] message = 'e{:03d}-i{:04d} => {:.1f}% (timings : {:4.2f} {:4.2f} {:4.2f})' print(message.format(test_epoch, i, 100 * i / config.validation_size, 1000 * (mean_dt[0]), 1000 * (mean_dt[1]), 1000 * (mean_dt[2]))) # Update minimum od potentials new_min = torch.min(test_loader.dataset.min_potentials) print('Test epoch {:d}, end. Min potential = {:.1f}'.format(test_epoch, new_min)) #print([np.mean(pots) for pots in test_loader.dataset.potentials]) # Save predicted cloud if last_min + 1 < new_min: # Update last_min last_min += 1 # Show vote results (On subcloud so it is not the good values here) if test_loader.dataset.set == 'validation': print('\nConfusion on sub clouds') Confs = [] for i, file_path in enumerate(test_loader.dataset.files): # Insert false columns for ignored labels probs = np.array(self.test_probs[i], copy=True) for l_ind, label_value in enumerate(test_loader.dataset.label_values): if label_value in test_loader.dataset.ignored_labels: probs = np.insert(probs, l_ind, 0, axis=1) # Predicted labels preds = test_loader.dataset.label_values[np.argmax(probs, axis=1)].astype(np.int32) # Targets targets = test_loader.dataset.input_labels[i] # Confs Confs += [fast_confusion(targets, preds, test_loader.dataset.label_values)] # Regroup confusions C = np.sum(np.stack(Confs), axis=0).astype(np.float32) # Remove ignored labels from confusions for l_ind, label_value in reversed(list(enumerate(test_loader.dataset.label_values))): if label_value in test_loader.dataset.ignored_labels: C = np.delete(C, l_ind, axis=0) C = np.delete(C, l_ind, axis=1) # Rescale with the right number of point per class C *= np.expand_dims(val_proportions / (np.sum(C, axis=1) + 1e-6), 1) # Compute IoUs IoUs = IoU_from_confusions(C) mIoU = np.mean(IoUs) s = '{:5.2f} | '.format(100 * mIoU) for IoU in IoUs: s += '{:5.2f} '.format(100 * IoU) print(s + '\n') # Save real IoU once in a while if int(np.ceil(new_min)) % 10 == 0: # Project predictions print('\nReproject Vote #{:d}'.format(int(np.floor(new_min)))) t1 = time.time() proj_probs = [] for i, file_path in enumerate(test_loader.dataset.files): # print(i, file_path, test_loader.dataset.test_proj[i].shape, self.test_probs[i].shape) # print(test_loader.dataset.test_proj[i].dtype, np.max(test_loader.dataset.test_proj[i])) # print(test_loader.dataset.test_proj[i][:5]) # Reproject probs on the evaluations points probs = self.test_probs[i][test_loader.dataset.test_proj[i], :] proj_probs += [probs] # Insert false columns for ignored labels for l_ind, label_value in enumerate(test_loader.dataset.label_values): if label_value in test_loader.dataset.ignored_labels: proj_probs[i] = np.insert(proj_probs[i], l_ind, 0, axis=1) t2 = time.time() print('Done in {:.1f} s\n'.format(t2 - t1)) # Show vote results if test_loader.dataset.set == 'validation': print('Confusion on full clouds') t1 = time.time() Confs = [] for i, file_path in enumerate(test_loader.dataset.files): # Get the predicted labels preds = test_loader.dataset.label_values[np.argmax(proj_probs[i], axis=1)].astype(np.int32) # Confusion targets = test_loader.dataset.validation_labels[i] Confs += [fast_confusion(targets, preds, test_loader.dataset.label_values)] t2 = time.time() print('Done in {:.1f} s\n'.format(t2 - t1)) # Regroup confusions C = np.sum(np.stack(Confs), axis=0) # Remove ignored labels from confusions for l_ind, label_value in reversed(list(enumerate(test_loader.dataset.label_values))): if label_value in test_loader.dataset.ignored_labels: C = np.delete(C, l_ind, axis=0) C = np.delete(C, l_ind, axis=1) IoUs = IoU_from_confusions(C) mIoU = np.mean(IoUs) s = '{:5.2f} | '.format(100 * mIoU) for IoU in IoUs: s += '{:5.2f} '.format(100 * IoU) print('-' * len(s)) print(s) print('-' * len(s) + '\n') # Save predictions print('Saving clouds') t1 = time.time() for i, file_path in enumerate(test_loader.dataset.files): # Get file points = test_loader.dataset.load_evaluation_points(file_path) # Get the predicted labels preds = test_loader.dataset.label_values[np.argmax(proj_probs[i], axis=1)].astype(np.int32) # Save plys cloud_name = file_path.split('/')[-1] test_name = join(test_path, 'predictions', cloud_name) write_ply(test_name, [points, preds], ['x', 'y', 'z', 'preds']) test_name2 = join(test_path, 'probs', cloud_name) prob_names = ['_'.join(test_loader.dataset.label_to_names[label].split()) for label in test_loader.dataset.label_values] write_ply(test_name2, [points, proj_probs[i]], ['x', 'y', 'z'] + prob_names) # Save potentials pot_points = np.array(test_loader.dataset.pot_trees[i].data, copy=False) pot_name = join(test_path, 'potentials', cloud_name) pots = test_loader.dataset.potentials[i].numpy().astype(np.float32) write_ply(pot_name, [pot_points.astype(np.float32), pots], ['x', 'y', 'z', 'pots']) # Save ascii preds if test_loader.dataset.set == 'test': if test_loader.dataset.name.startswith('Semantic3D'): ascii_name = join(test_path, 'predictions', test_loader.dataset.ascii_files[cloud_name]) else: ascii_name = join(test_path, 'predictions', cloud_name[:-4] + '.txt') np.savetxt(ascii_name, preds, fmt='%d') t2 = time.time() print('Done in {:.1f} s\n'.format(t2 - t1)) test_epoch += 1 # Break when reaching number of desired votes if last_min > num_votes: break return def slam_segmentation_test(self, net, test_loader, config, num_votes=100, debug=True): """ Test method for slam segmentation models """ ############ # Initialize ############ # Choose validation smoothing parameter (0 for no smothing, 0.99 for big smoothing) test_smooth = 0.5 last_min = -0.5 softmax = torch.nn.Softmax(1) # Number of classes including ignored labels nc_tot = test_loader.dataset.num_classes nc_model = net.C # Test saving path test_path = None report_path = None if config.saving: test_path = join('test', config.saving_path.split('/')[-1]) if not exists(test_path): makedirs(test_path) report_path = join(test_path, 'reports') if not exists(report_path): makedirs(report_path) if test_loader.dataset.set == 'validation': for folder in ['val_predictions', 'val_probs']: if not exists(join(test_path, folder)): makedirs(join(test_path, folder)) else: for folder in ['predictions', 'probs']: if not exists(join(test_path, folder)): makedirs(join(test_path, folder)) # Init validation container all_f_preds = [] all_f_labels = [] if test_loader.dataset.set == 'validation': for i, seq_frames in enumerate(test_loader.dataset.frames): all_f_preds.append([np.zeros((0,), dtype=np.int32) for _ in seq_frames]) all_f_labels.append([np.zeros((0,), dtype=np.int32) for _ in seq_frames]) ##################### # Network predictions ##################### predictions = [] targets = [] test_epoch = 0 t = [time.time()] last_display = time.time() mean_dt = np.zeros(1) # Start test loop while True: print('Initialize workers') for i, batch in enumerate(test_loader): # New time t = t[-1:] t += [time.time()] if i == 0: print('Done in {:.1f}s'.format(t[1] - t[0])) if 'cuda' in self.device.type: batch.to(self.device) # Forward pass outputs = net(batch, config) # Get probs and labels stk_probs = softmax(outputs).cpu().detach().numpy() lengths = batch.lengths[0].cpu().numpy() f_inds = batch.frame_inds.cpu().numpy() r_inds_list = batch.reproj_inds r_mask_list = batch.reproj_masks labels_list = batch.val_labels torch.cuda.synchronize(self.device) t += [time.time()] # Get predictions and labels per instance # *************************************** i0 = 0 for b_i, length in enumerate(lengths): # Get prediction probs = stk_probs[i0:i0 + length] proj_inds = r_inds_list[b_i] proj_mask = r_mask_list[b_i] frame_labels = labels_list[b_i] s_ind = f_inds[b_i, 0] f_ind = f_inds[b_i, 1] # Project predictions on the frame points proj_probs = probs[proj_inds] # Safe check if only one point: if proj_probs.ndim < 2: proj_probs = np.expand_dims(proj_probs, 0) # Save probs in a binary file (uint8 format for lighter weight) seq_name = test_loader.dataset.sequences[s_ind] if test_loader.dataset.set == 'validation': folder = 'val_probs' pred_folder = 'val_predictions' else: folder = 'probs' pred_folder = 'predictions' filename = '{:s}_{:07d}.npy'.format(seq_name, f_ind) filepath = join(test_path, folder, filename) if exists(filepath): frame_probs_uint8 = np.load(filepath) else: frame_probs_uint8 = np.zeros((proj_mask.shape[0], nc_model), dtype=np.uint8) frame_probs = frame_probs_uint8[proj_mask, :].astype(np.float32) / 255 frame_probs = test_smooth * frame_probs + (1 - test_smooth) * proj_probs frame_probs_uint8[proj_mask, :] = (frame_probs * 255).astype(np.uint8) np.save(filepath, frame_probs_uint8) # Save some prediction in ply format for visual if test_loader.dataset.set == 'validation': # Insert false columns for ignored labels frame_probs_uint8_bis = frame_probs_uint8.copy() for l_ind, label_value in enumerate(test_loader.dataset.label_values): if label_value in test_loader.dataset.ignored_labels: frame_probs_uint8_bis = np.insert(frame_probs_uint8_bis, l_ind, 0, axis=1) # Predicted labels frame_preds = test_loader.dataset.label_values[np.argmax(frame_probs_uint8_bis, axis=1)].astype(np.int32) # Save some of the frame pots if f_ind % 20 == 0: seq_path = join(test_loader.dataset.path, 'sequences', test_loader.dataset.sequences[s_ind]) velo_file = join(seq_path, 'velodyne', test_loader.dataset.frames[s_ind][f_ind] + '.bin') frame_points = np.fromfile(velo_file, dtype=np.float32) frame_points = frame_points.reshape((-1, 4)) predpath = join(test_path, pred_folder, filename[:-4] + '.ply') #pots = test_loader.dataset.f_potentials[s_ind][f_ind] pots = np.zeros((0,)) if pots.shape[0] > 0: write_ply(predpath, [frame_points[:, :3], frame_labels, frame_preds, pots], ['x', 'y', 'z', 'gt', 'pre', 'pots']) else: write_ply(predpath, [frame_points[:, :3], frame_labels, frame_preds], ['x', 'y', 'z', 'gt', 'pre']) # Also Save lbl probabilities probpath = join(test_path, folder, filename[:-4] + '_probs.ply') lbl_names = [test_loader.dataset.label_to_names[l] for l in test_loader.dataset.label_values if l not in test_loader.dataset.ignored_labels] write_ply(probpath, [frame_points[:, :3], frame_probs_uint8], ['x', 'y', 'z'] + lbl_names) # keep frame preds in memory all_f_preds[s_ind][f_ind] = frame_preds all_f_labels[s_ind][f_ind] = frame_labels else: # Save some of the frame preds if f_inds[b_i, 1] % 100 == 0: # Insert false columns for ignored labels for l_ind, label_value in enumerate(test_loader.dataset.label_values): if label_value in test_loader.dataset.ignored_labels: frame_probs_uint8 = np.insert(frame_probs_uint8, l_ind, 0, axis=1) # Predicted labels frame_preds = test_loader.dataset.label_values[np.argmax(frame_probs_uint8, axis=1)].astype(np.int32) # Load points seq_path = join(test_loader.dataset.path, 'sequences', test_loader.dataset.sequences[s_ind]) velo_file = join(seq_path, 'velodyne', test_loader.dataset.frames[s_ind][f_ind] + '.bin') frame_points = np.fromfile(velo_file, dtype=np.float32) frame_points = frame_points.reshape((-1, 4)) predpath = join(test_path, pred_folder, filename[:-4] + '.ply') #pots = test_loader.dataset.f_potentials[s_ind][f_ind] pots = np.zeros((0,)) if pots.shape[0] > 0: write_ply(predpath, [frame_points[:, :3], frame_preds, pots], ['x', 'y', 'z', 'pre', 'pots']) else: write_ply(predpath, [frame_points[:, :3], frame_preds], ['x', 'y', 'z', 'pre']) # Stack all prediction for this epoch i0 += length # Average timing t += [time.time()] mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1])) # Display if (t[-1] - last_display) > 1.0: last_display = t[-1] message = 'e{:03d}-i{:04d} => {:.1f}% (timings : {:4.2f} {:4.2f} {:4.2f}) / pots {:d} => {:.1f}%' min_pot = int(torch.floor(torch.min(test_loader.dataset.potentials))) pot_num = torch.sum(test_loader.dataset.potentials > min_pot + 0.5).type(torch.int32).item() current_num = pot_num + (i + 1 - config.validation_size) * config.val_batch_num print(message.format(test_epoch, i, 100 * i / config.validation_size, 1000 * (mean_dt[0]), 1000 * (mean_dt[1]), 1000 * (mean_dt[2]), min_pot, 100.0 * current_num / len(test_loader.dataset.potentials))) # Update minimum od potentials new_min = torch.min(test_loader.dataset.potentials) print('Test epoch {:d}, end. Min potential = {:.1f}'.format(test_epoch, new_min)) if last_min + 1 < new_min: # Update last_min last_min += 1 if test_loader.dataset.set == 'validation' and last_min % 1 == 0: ##################################### # Results on the whole validation set ##################################### # Confusions for our subparts of validation set Confs = np.zeros((len(predictions), nc_tot, nc_tot), dtype=np.int32) for i, (preds, truth) in enumerate(zip(predictions, targets)): # Confusions Confs[i, :, :] = fast_confusion(truth, preds, test_loader.dataset.label_values).astype(np.int32) # Show vote results print('\nCompute confusion') val_preds = [] val_labels = [] t1 = time.time() for i, seq_frames in enumerate(test_loader.dataset.frames): val_preds += [np.hstack(all_f_preds[i])] val_labels += [np.hstack(all_f_labels[i])] val_preds = np.hstack(val_preds) val_labels = np.hstack(val_labels) t2 = time.time() C_tot = fast_confusion(val_labels, val_preds, test_loader.dataset.label_values) t3 = time.time() print(' Stacking time : {:.1f}s'.format(t2 - t1)) print('Confusion time : {:.1f}s'.format(t3 - t2)) s1 = '\n' for cc in C_tot: for c in cc: s1 += '{:7.0f} '.format(c) s1 += '\n' if debug: print(s1) # Remove ignored labels from confusions for l_ind, label_value in reversed(list(enumerate(test_loader.dataset.label_values))): if label_value in test_loader.dataset.ignored_labels: C_tot = np.delete(C_tot, l_ind, axis=0) C_tot = np.delete(C_tot, l_ind, axis=1) # Objects IoU val_IoUs = IoU_from_confusions(C_tot) # Compute IoUs mIoU = np.mean(val_IoUs) s2 = '{:5.2f} | '.format(100 * mIoU) for IoU in val_IoUs: s2 += '{:5.2f} '.format(100 * IoU) print(s2 + '\n') # Save a report report_file = join(report_path, 'report_{:04d}.txt'.format(int(np.floor(last_min)))) str = 'Report of the confusion and metrics\n' str += '***********************************\n\n\n' str += 'Confusion matrix:\n\n' str += s1 str += '\nIoU values:\n\n' str += s2 str += '\n\n' with open(report_file, 'w') as f: f.write(str) test_epoch += 1 # Break when reaching number of desired votes if last_min > num_votes: break return