1910 lines
71 KiB
Python
1910 lines
71 KiB
Python
#
|
|
#
|
|
# 0=================================0
|
|
# | Kernel Point Convolutions |
|
|
# 0=================================0
|
|
#
|
|
#
|
|
# ----------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Class handling the training of any model
|
|
#
|
|
# ----------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Hugues THOMAS - 11/06/2018
|
|
#
|
|
|
|
|
|
# ----------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Imports and global variables
|
|
# \**********************************/
|
|
#
|
|
|
|
|
|
# Basic libs
|
|
import torch
|
|
import torch.nn as nn
|
|
import numpy as np
|
|
import pickle
|
|
import os
|
|
from os import makedirs, remove
|
|
from os.path import exists, join
|
|
import time
|
|
import sys
|
|
|
|
# PLY reader
|
|
from utils.ply import read_ply, write_ply
|
|
|
|
# Metrics
|
|
from utils.metrics import IoU_from_confusions
|
|
from utils.config import Config
|
|
from sklearn.metrics import confusion_matrix
|
|
from sklearn.neighbors import KDTree
|
|
|
|
from models.blocks import KPConv
|
|
|
|
# ----------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Trainer Class
|
|
# \*******************/
|
|
#
|
|
|
|
|
|
class ModelTrainer:
|
|
|
|
# Initialization methods
|
|
# ------------------------------------------------------------------------------------------------------------------
|
|
|
|
def __init__(self, net, config, chkp_path=None, finetune=False, on_gpu=True):
|
|
"""
|
|
Initialize training parameters and reload previous model for restore/finetune
|
|
:param net: network object
|
|
:param config: configuration object
|
|
:param chkp_path: path to the checkpoint that needs to be loaded (None for new training)
|
|
:param finetune: finetune from checkpoint (True) or restore training from checkpoint (False)
|
|
:param on_gpu: Train on GPU or CPU
|
|
"""
|
|
|
|
############
|
|
# Parameters
|
|
############
|
|
|
|
# Epoch index
|
|
self.epoch = 0
|
|
self.step = 0
|
|
|
|
# Optimizer
|
|
self.optimizer = torch.optim.SGD(net.parameters(),
|
|
lr=config.learning_rate,
|
|
momentum=config.momentum,
|
|
weight_decay=config.weight_decay)
|
|
|
|
# Choose to train on CPU or GPU
|
|
if on_gpu and torch.cuda.is_available():
|
|
self.device = torch.device("cuda:0")
|
|
else:
|
|
self.device = torch.device("cpu")
|
|
net.to(self.device)
|
|
|
|
##########################
|
|
# Load previous checkpoint
|
|
##########################
|
|
|
|
if (chkp_path is not None):
|
|
if finetune:
|
|
checkpoint = torch.load(chkp_path)
|
|
net.load_state_dict(checkpoint['model_state_dict'])
|
|
net.train()
|
|
print("Model restored and ready for finetuning.")
|
|
else:
|
|
checkpoint = torch.load(chkp_path)
|
|
net.load_state_dict(checkpoint['model_state_dict'])
|
|
self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
|
|
self.epoch = checkpoint['epoch']
|
|
net.train()
|
|
print("Model and training state restored.")
|
|
|
|
# Path of the result folder
|
|
if config.saving:
|
|
if config.saving_path is None:
|
|
config.saving_path = time.strftime('results/Log_%Y-%m-%d_%H-%M-%S', time.gmtime())
|
|
if not exists(config.saving_path):
|
|
makedirs(config.saving_path)
|
|
config.save()
|
|
|
|
return
|
|
|
|
# Training main method
|
|
# ------------------------------------------------------------------------------------------------------------------
|
|
|
|
def train(self, net, training_loader, val_loader, config):
|
|
"""
|
|
Train the model on a particular dataset.
|
|
"""
|
|
|
|
################
|
|
# Initialization
|
|
################
|
|
|
|
if config.saving:
|
|
# Training log file
|
|
with open(join(config.saving_path, 'training.txt'), "w") as file:
|
|
file.write('epochs steps out_loss offset_loss train_accuracy time\n')
|
|
|
|
# Killing file (simply delete this file when you want to stop the training)
|
|
PID_file = join(config.saving_path, 'running_PID.txt')
|
|
if not exists(PID_file):
|
|
with open(PID_file, "w") as file:
|
|
file.write('Launched with PyCharm')
|
|
|
|
# Checkpoints directory
|
|
checkpoint_directory = join(config.saving_path, 'checkpoints')
|
|
if not exists(checkpoint_directory):
|
|
makedirs(checkpoint_directory)
|
|
else:
|
|
checkpoint_directory = None
|
|
PID_file = None
|
|
|
|
# Loop variables
|
|
t0 = time.time()
|
|
t = [time.time()]
|
|
last_display = time.time()
|
|
mean_dt = np.zeros(1)
|
|
|
|
# Start training loop
|
|
for epoch in range(config.max_epoch):
|
|
|
|
# Remove File for kill signal
|
|
if epoch == config.max_epoch - 1 and exists(PID_file):
|
|
remove(PID_file)
|
|
|
|
self.step = 0
|
|
for batch in training_loader:
|
|
|
|
# Check kill signal (running_PID.txt deleted)
|
|
if config.saving and not exists(PID_file):
|
|
continue
|
|
|
|
##################
|
|
# Processing batch
|
|
##################
|
|
|
|
# New time
|
|
t = t[-1:]
|
|
t += [time.time()]
|
|
|
|
if 'cuda' in self.device.type:
|
|
batch.to(self.device)
|
|
|
|
# zero the parameter gradients
|
|
self.optimizer.zero_grad()
|
|
|
|
# Forward pass
|
|
outputs = net(batch, config)
|
|
loss = net.loss(outputs, batch.labels)
|
|
acc = net.accuracy(outputs, batch.labels)
|
|
|
|
t += [time.time()]
|
|
|
|
# Backward + optimize
|
|
loss.backward()
|
|
|
|
if config.grad_clip_norm > 0:
|
|
#torch.nn.utils.clip_grad_norm_(net.parameters(), config.grad_clip_norm)
|
|
torch.nn.utils.clip_grad_value_(net.parameters(), config.grad_clip_norm)
|
|
self.optimizer.step()
|
|
torch.cuda.synchronize(self.device)
|
|
|
|
t += [time.time()]
|
|
|
|
# Average timing
|
|
if self.step < 2:
|
|
mean_dt = np.array(t[1:]) - np.array(t[:-1])
|
|
else:
|
|
mean_dt = 0.9 * mean_dt + 0.1 * (np.array(t[1:]) - np.array(t[:-1]))
|
|
|
|
# Console display (only one per second)
|
|
if (t[-1] - last_display) > 1.0:
|
|
last_display = t[-1]
|
|
message = 'e{:03d}-i{:04d} => L={:.3f} acc={:3.0f}% / t(ms): {:5.1f} {:5.1f} {:5.1f})'
|
|
print(message.format(self.epoch, self.step,
|
|
loss.item(),
|
|
100*acc,
|
|
1000 * mean_dt[0],
|
|
1000 * mean_dt[1],
|
|
1000 * mean_dt[2]))
|
|
|
|
# Log file
|
|
if config.saving:
|
|
with open(join(config.saving_path, 'training.txt'), "a") as file:
|
|
message = '{:d} {:d} {:.3f} {:.3f} {:.3f} {:.3f}\n'
|
|
file.write(message.format(self.epoch,
|
|
self.step,
|
|
net.output_loss,
|
|
net.reg_loss,
|
|
acc,
|
|
t[-1] - t0))
|
|
|
|
|
|
self.step += 1
|
|
|
|
##############
|
|
# End of epoch
|
|
##############
|
|
|
|
# Check kill signal (running_PID.txt deleted)
|
|
if config.saving and not exists(PID_file):
|
|
break
|
|
|
|
# Update learning rate
|
|
if self.epoch in config.lr_decays:
|
|
for param_group in self.optimizer.param_groups:
|
|
param_group['lr'] *= config.lr_decays[self.epoch]
|
|
|
|
# Update epoch
|
|
self.epoch += 1
|
|
|
|
# Saving
|
|
if config.saving:
|
|
# Get current state dict
|
|
save_dict = {'epoch': self.epoch,
|
|
'model_state_dict': net.state_dict(),
|
|
'optimizer_state_dict': self.optimizer.state_dict(),
|
|
'saving_path': config.saving_path}
|
|
|
|
# Save current state of the network (for restoring purposes)
|
|
checkpoint_path = join(checkpoint_directory, 'current_chkp.tar')
|
|
torch.save(save_dict, checkpoint_path)
|
|
|
|
# Save checkpoints occasionally
|
|
if (self.epoch + 1) % config.checkpoint_gap == 0:
|
|
checkpoint_path = join(checkpoint_directory, 'chkp_{:04d}.tar'.format(self.epoch))
|
|
torch.save(save_dict, checkpoint_path)
|
|
|
|
# Validation
|
|
net.eval()
|
|
self.validation(net, val_loader, config)
|
|
net.train()
|
|
|
|
print('Finished Training')
|
|
return
|
|
|
|
# Validation methods
|
|
# ------------------------------------------------------------------------------------------------------------------
|
|
|
|
def validation(self, net, val_loader, config: Config):
|
|
|
|
if config.dataset_task == 'classification':
|
|
self.object_classification_validation(net, val_loader, config)
|
|
elif config.dataset_task == 'segmentation':
|
|
self.object_segmentation_validation(net, val_loader, config)
|
|
elif config.dataset_task == 'cloud_segmentation':
|
|
self.cloud_segmentation_validation(net, val_loader, config)
|
|
elif config.dataset_task == 'slam_segmentation':
|
|
self.slam_segmentation_validation(net, val_loader, config)
|
|
else:
|
|
raise ValueError('No validation method implemented for this network type')
|
|
|
|
def object_classification_validation(self, net, val_loader, config):
|
|
"""
|
|
Perform a round of validation and show/save results
|
|
:param net: network object
|
|
:param val_loader: data loader for validation set
|
|
:param config: configuration object
|
|
"""
|
|
|
|
############
|
|
# Initialize
|
|
############
|
|
|
|
# Choose validation smoothing parameter (0 for no smothing, 0.99 for big smoothing)
|
|
val_smooth = 0.95
|
|
|
|
# Number of classes predicted by the model
|
|
nc_model = config.num_classes
|
|
softmax = torch.nn.Softmax(1)
|
|
|
|
# Initialize global prediction over all models
|
|
if not hasattr(self, 'val_probs'):
|
|
self.val_probs = np.zeros((val_loader.dataset.num_models, nc_model))
|
|
|
|
#####################
|
|
# Network predictions
|
|
#####################
|
|
|
|
probs = []
|
|
targets = []
|
|
obj_inds = []
|
|
|
|
t = [time.time()]
|
|
last_display = time.time()
|
|
mean_dt = np.zeros(1)
|
|
|
|
# Start validation loop
|
|
for batch in val_loader:
|
|
|
|
# New time
|
|
t = t[-1:]
|
|
t += [time.time()]
|
|
|
|
if 'cuda' in self.device.type:
|
|
batch.to(self.device)
|
|
|
|
# Forward pass
|
|
outputs = net(batch, config)
|
|
|
|
# Get probs and labels
|
|
probs += [softmax(outputs).cpu().detach().numpy()]
|
|
targets += [batch.labels.cpu().numpy()]
|
|
obj_inds += [batch.model_inds.cpu().numpy()]
|
|
torch.cuda.synchronize(self.device)
|
|
|
|
# Average timing
|
|
t += [time.time()]
|
|
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1]))
|
|
|
|
# Display
|
|
if (t[-1] - last_display) > 1.0:
|
|
last_display = t[-1]
|
|
message = 'Validation : {:.1f}% (timings : {:4.2f} {:4.2f})'
|
|
print(message.format(100 * len(obj_inds) / config.validation_size,
|
|
1000 * (mean_dt[0]),
|
|
1000 * (mean_dt[1])))
|
|
|
|
# Stack all validation predictions
|
|
probs = np.vstack(probs)
|
|
targets = np.hstack(targets)
|
|
obj_inds = np.hstack(obj_inds)
|
|
|
|
###################
|
|
# Voting validation
|
|
###################
|
|
|
|
self.val_probs[obj_inds] = val_smooth * self.val_probs[obj_inds] + (1-val_smooth) * probs
|
|
|
|
############
|
|
# Confusions
|
|
############
|
|
|
|
validation_labels = np.array(val_loader.dataset.label_values)
|
|
|
|
# Compute classification results
|
|
C1 = confusion_matrix(targets,
|
|
np.argmax(probs, axis=1),
|
|
validation_labels)
|
|
|
|
# Compute votes confusion
|
|
C2 = confusion_matrix(val_loader.dataset.input_labels,
|
|
np.argmax(self.val_probs, axis=1),
|
|
validation_labels)
|
|
|
|
|
|
# Saving (optionnal)
|
|
if config.saving:
|
|
print("Save confusions")
|
|
conf_list = [C1, C2]
|
|
file_list = ['val_confs.txt', 'vote_confs.txt']
|
|
for conf, conf_file in zip(conf_list, file_list):
|
|
test_file = join(config.saving_path, conf_file)
|
|
if exists(test_file):
|
|
with open(test_file, "a") as text_file:
|
|
for line in conf:
|
|
for value in line:
|
|
text_file.write('%d ' % value)
|
|
text_file.write('\n')
|
|
else:
|
|
with open(test_file, "w") as text_file:
|
|
for line in conf:
|
|
for value in line:
|
|
text_file.write('%d ' % value)
|
|
text_file.write('\n')
|
|
|
|
val_ACC = 100 * np.sum(np.diag(C1)) / (np.sum(C1) + 1e-6)
|
|
vote_ACC = 100 * np.sum(np.diag(C2)) / (np.sum(C2) + 1e-6)
|
|
print('Accuracies : val = {:.1f}% / vote = {:.1f}%'.format(val_ACC, vote_ACC))
|
|
|
|
return C1
|
|
|
|
def cloud_segmentation_validation(self, net, val_loader, config):
|
|
"""
|
|
Validation method for cloud segmentation models
|
|
"""
|
|
|
|
############
|
|
# Initialize
|
|
############
|
|
|
|
# Choose validation smoothing parameter (0 for no smothing, 0.99 for big smoothing)
|
|
val_smooth = 0.95
|
|
softmax = torch.nn.Softmax(1)
|
|
|
|
# Do not validate if dataset has no validation cloud
|
|
if val_loader.dataset.validation_split not in val_loader.dataset.all_splits:
|
|
return
|
|
|
|
# Number of classes including ignored labels
|
|
nc_tot = val_loader.dataset.num_classes
|
|
|
|
# Number of classes predicted by the model
|
|
nc_model = config.num_classes
|
|
|
|
#print(nc_tot)
|
|
#print(nc_model)
|
|
|
|
# Initiate global prediction over validation clouds
|
|
if not hasattr(self, 'validation_probs'):
|
|
self.validation_probs = [np.zeros((l.shape[0], nc_model))
|
|
for l in val_loader.dataset.input_labels]
|
|
self.val_proportions = np.zeros(nc_model, dtype=np.float32)
|
|
i = 0
|
|
for label_value in val_loader.dataset.label_values:
|
|
if label_value not in val_loader.dataset.ignored_labels:
|
|
self.val_proportions[i] = np.sum([np.sum(labels == label_value)
|
|
for labels in val_loader.dataset.validation_labels])
|
|
i += 1
|
|
|
|
#####################
|
|
# Network predictions
|
|
#####################
|
|
|
|
predictions = []
|
|
targets = []
|
|
|
|
t = [time.time()]
|
|
last_display = time.time()
|
|
mean_dt = np.zeros(1)
|
|
|
|
# Start validation loop
|
|
for i, batch in enumerate(val_loader):
|
|
|
|
# New time
|
|
t = t[-1:]
|
|
t += [time.time()]
|
|
|
|
if 'cuda' in self.device.type:
|
|
batch.to(self.device)
|
|
|
|
# Forward pass
|
|
outputs = net(batch, config)
|
|
|
|
# Get probs and labels
|
|
stacked_probs = softmax(outputs).cpu().detach().numpy()
|
|
labels = batch.labels.cpu().numpy()
|
|
lengths = batch.lengths[0].cpu().numpy()
|
|
in_inds = batch.input_inds.cpu().numpy()
|
|
cloud_inds = batch.cloud_inds.cpu().numpy()
|
|
torch.cuda.synchronize(self.device)
|
|
|
|
# Get predictions and labels per instance
|
|
# ***************************************
|
|
|
|
i0 = 0
|
|
for b_i, length in enumerate(lengths):
|
|
|
|
# Get prediction
|
|
target = labels[i0:i0 + length]
|
|
probs = stacked_probs[i0:i0 + length]
|
|
inds = in_inds[i0:i0 + length]
|
|
c_i = cloud_inds[b_i]
|
|
|
|
# Update current probs in whole cloud
|
|
self.validation_probs[c_i][inds] = val_smooth * self.validation_probs[c_i][inds] \
|
|
+ (1 - val_smooth) * probs
|
|
|
|
# Stack all prediction for this epoch
|
|
predictions.append(probs)
|
|
targets.append(target)
|
|
i0 += length
|
|
|
|
# Average timing
|
|
t += [time.time()]
|
|
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1]))
|
|
|
|
# Display
|
|
if (t[-1] - last_display) > 1.0:
|
|
last_display = t[-1]
|
|
message = 'Validation : {:.1f}% (timings : {:4.2f} {:4.2f})'
|
|
print(message.format(100 * i / config.validation_size,
|
|
1000 * (mean_dt[0]),
|
|
1000 * (mean_dt[1])))
|
|
|
|
# Confusions for our subparts of validation set
|
|
Confs = np.zeros((len(predictions), nc_tot, nc_tot), dtype=np.int32)
|
|
for i, (probs, truth) in enumerate(zip(predictions, targets)):
|
|
|
|
# Insert false columns for ignored labels
|
|
for l_ind, label_value in enumerate(val_loader.dataset.label_values):
|
|
if label_value in val_loader.dataset.ignored_labels:
|
|
probs = np.insert(probs, l_ind, 0, axis=1)
|
|
|
|
# Predicted labels
|
|
preds = val_loader.dataset.label_values[np.argmax(probs, axis=1)]
|
|
|
|
# Confusions
|
|
Confs[i, :, :] = confusion_matrix(truth, preds, val_loader.dataset.label_values)
|
|
|
|
# Sum all confusions
|
|
C = np.sum(Confs, axis=0).astype(np.float32)
|
|
|
|
# Remove ignored labels from confusions
|
|
for l_ind, label_value in reversed(list(enumerate(val_loader.dataset.label_values))):
|
|
if label_value in val_loader.dataset.ignored_labels:
|
|
C = np.delete(C, l_ind, axis=0)
|
|
C = np.delete(C, l_ind, axis=1)
|
|
|
|
# Balance with real validation proportions
|
|
C *= np.expand_dims(self.val_proportions / (np.sum(C, axis=1) + 1e-6), 1)
|
|
|
|
# Objects IoU
|
|
IoUs = IoU_from_confusions(C)
|
|
|
|
# Saving (optionnal)
|
|
if config.saving:
|
|
|
|
# Name of saving file
|
|
test_file = join(config.saving_path, 'val_IoUs.txt')
|
|
|
|
# Line to write:
|
|
line = ''
|
|
for IoU in IoUs:
|
|
line += '{:.3f} '.format(IoU)
|
|
line = line + '\n'
|
|
|
|
# Write in file
|
|
if exists(test_file):
|
|
with open(test_file, "a") as text_file:
|
|
text_file.write(line)
|
|
else:
|
|
with open(test_file, "w") as text_file:
|
|
text_file.write(line)
|
|
|
|
# Save potentials
|
|
pot_path = join(config.saving_path, 'potentials')
|
|
if not exists(pot_path):
|
|
makedirs(pot_path)
|
|
files = val_loader.dataset.train_files
|
|
i_val = 0
|
|
for i, file_path in enumerate(files):
|
|
if val_loader.dataset.all_splits[i] == val_loader.dataset.validation_split:
|
|
pot_points = np.array(val_loader.dataset.pot_trees[i_val].data, copy=False)
|
|
cloud_name = file_path.split('/')[-1]
|
|
pot_name = join(pot_path, cloud_name)
|
|
pots = val_loader.dataset.potentials[i_val].numpy().astype(np.float32)
|
|
write_ply(pot_name,
|
|
[pot_points.astype(np.float32), pots],
|
|
['x', 'y', 'z', 'pots'])
|
|
|
|
# Print instance mean
|
|
mIoU = 100 * np.mean(IoUs)
|
|
print('{:s} mean IoU = {:.1f}%'.format(config.dataset, mIoU))
|
|
|
|
# Save predicted cloud occasionally
|
|
if config.saving and (self.epoch + 1) % config.checkpoint_gap == 0:
|
|
val_path = join(config.saving_path, 'val_preds_{:d}'.format(self.epoch))
|
|
if not exists(val_path):
|
|
makedirs(val_path)
|
|
files = val_loader.dataset.train_files
|
|
i_val = 0
|
|
for i, file_path in enumerate(files):
|
|
if val_loader.dataset.all_splits[i] == val_loader.dataset.validation_split:
|
|
|
|
# Get points
|
|
points = val_loader.dataset.load_evaluation_points(file_path)
|
|
|
|
# Get probs on our own ply points
|
|
sub_probs = self.validation_probs[i_val]
|
|
|
|
# Insert false columns for ignored labels
|
|
for l_ind, label_value in enumerate(val_loader.dataset.label_values):
|
|
if label_value in val_loader.dataset.ignored_labels:
|
|
sub_probs = np.insert(sub_probs, l_ind, 0, axis=1)
|
|
|
|
# Get the predicted labels
|
|
sub_preds = val_loader.dataset.label_values[np.argmax(sub_probs, axis=1).astype(np.int32)]
|
|
|
|
# Reproject preds on the evaluations points
|
|
preds = (sub_preds[val_loader.dataset.validation_proj[i_val]]).astype(np.int32)
|
|
|
|
# Path of saved validation file
|
|
cloud_name = file_path.split('/')[-1]
|
|
val_name = join(val_path, cloud_name)
|
|
|
|
# Save file
|
|
labels = val_loader.dataset.validation_labels[i_val].astype(np.int32)
|
|
write_ply(val_name,
|
|
[points, preds, labels],
|
|
['x', 'y', 'z', 'preds', 'class'])
|
|
|
|
i_val += 1
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
|
|
def validation_error(self, model, dataset):
|
|
"""
|
|
Validation method for classification models
|
|
"""
|
|
|
|
############
|
|
# Initialize
|
|
############
|
|
|
|
# Choose validation smoothing parameter (0 for no smothing, 0.99 for big smoothing)
|
|
val_smooth = 0.95
|
|
|
|
# Initialise iterator with train data
|
|
self.sess.run(dataset.val_init_op)
|
|
|
|
# Number of classes predicted by the model
|
|
nc_model = model.config.num_classes
|
|
|
|
# Initialize global prediction over all models
|
|
if not hasattr(self, 'val_probs'):
|
|
self.val_probs = np.zeros((len(dataset.input_labels['validation']), nc_model))
|
|
|
|
#####################
|
|
# Network predictions
|
|
#####################
|
|
|
|
probs = []
|
|
targets = []
|
|
obj_inds = []
|
|
|
|
mean_dt = np.zeros(2)
|
|
last_display = time.time()
|
|
while True:
|
|
try:
|
|
# Run one step of the model.
|
|
t = [time.time()]
|
|
ops = (self.prob_logits, model.labels, model.inputs['object_inds'])
|
|
prob, labels, inds = self.sess.run(ops, {model.dropout_prob: 1.0})
|
|
t += [time.time()]
|
|
|
|
# Get probs and labels
|
|
probs += [prob]
|
|
targets += [labels]
|
|
obj_inds += [inds]
|
|
|
|
# Average timing
|
|
t += [time.time()]
|
|
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1]))
|
|
|
|
# Display
|
|
if (t[-1] - last_display) > 1.0:
|
|
last_display = t[-1]
|
|
message = 'Validation : {:.1f}% (timings : {:4.2f} {:4.2f})'
|
|
print(message.format(100 * len(obj_inds) / model.config.validation_size,
|
|
1000 * (mean_dt[0]),
|
|
1000 * (mean_dt[1])))
|
|
|
|
except tf.errors.OutOfRangeError:
|
|
break
|
|
|
|
# Stack all validation predictions
|
|
probs = np.vstack(probs)
|
|
targets = np.hstack(targets)
|
|
obj_inds = np.hstack(obj_inds)
|
|
|
|
###################
|
|
# Voting validation
|
|
###################
|
|
|
|
self.val_probs[obj_inds] = val_smooth * self.val_probs[obj_inds] + (1-val_smooth) * probs
|
|
|
|
############
|
|
# Confusions
|
|
############
|
|
|
|
validation_labels = np.array(dataset.label_values)
|
|
|
|
# Compute classification results
|
|
C1 = confusion_matrix(targets,
|
|
np.argmax(probs, axis=1),
|
|
validation_labels)
|
|
|
|
# Compute training confusion
|
|
C2 = confusion_matrix(self.training_labels,
|
|
self.training_preds,
|
|
validation_labels)
|
|
|
|
# Compute votes confusion
|
|
C3 = confusion_matrix(dataset.input_labels['validation'],
|
|
np.argmax(self.val_probs, axis=1),
|
|
validation_labels)
|
|
|
|
|
|
# Saving (optionnal)
|
|
if model.config.saving:
|
|
print("Save confusions")
|
|
conf_list = [C1, C2, C3]
|
|
file_list = ['val_confs.txt', 'training_confs.txt', 'vote_confs.txt']
|
|
for conf, conf_file in zip(conf_list, file_list):
|
|
test_file = join(model.saving_path, conf_file)
|
|
if exists(test_file):
|
|
with open(test_file, "a") as text_file:
|
|
for line in conf:
|
|
for value in line:
|
|
text_file.write('%d ' % value)
|
|
text_file.write('\n')
|
|
else:
|
|
with open(test_file, "w") as text_file:
|
|
for line in conf:
|
|
for value in line:
|
|
text_file.write('%d ' % value)
|
|
text_file.write('\n')
|
|
|
|
train_ACC = 100 * np.sum(np.diag(C2)) / (np.sum(C2) + 1e-6)
|
|
val_ACC = 100 * np.sum(np.diag(C1)) / (np.sum(C1) + 1e-6)
|
|
vote_ACC = 100 * np.sum(np.diag(C3)) / (np.sum(C3) + 1e-6)
|
|
print('Accuracies : train = {:.1f}% / val = {:.1f}% / vote = {:.1f}%'.format(train_ACC, val_ACC, vote_ACC))
|
|
|
|
return C1
|
|
|
|
def segment_validation_error(self, model, dataset):
|
|
"""
|
|
Validation method for single object segmentation models
|
|
"""
|
|
|
|
##########
|
|
# Initialize
|
|
##########
|
|
|
|
# Choose validation smoothing parameter (0 for no smothing, 0.99 for big smoothing)
|
|
val_smooth = 0.95
|
|
|
|
# Initialise iterator with train data
|
|
self.sess.run(dataset.val_init_op)
|
|
|
|
# Number of classes predicted by the model
|
|
nc_model = model.config.num_classes
|
|
|
|
# Initialize global prediction over all models
|
|
if not hasattr(self, 'val_probs'):
|
|
self.val_probs = [np.zeros((len(p_l), nc_model)) for p_l in dataset.input_point_labels['validation']]
|
|
|
|
#####################
|
|
# Network predictions
|
|
#####################
|
|
|
|
probs = []
|
|
targets = []
|
|
obj_inds = []
|
|
mean_dt = np.zeros(2)
|
|
last_display = time.time()
|
|
for i0 in range(model.config.validation_size):
|
|
try:
|
|
# Run one step of the model.
|
|
t = [time.time()]
|
|
ops = (self.prob_logits, model.labels, model.inputs['in_batches'], model.inputs['object_inds'])
|
|
prob, labels, batches, o_inds = self.sess.run(ops, {model.dropout_prob: 1.0})
|
|
t += [time.time()]
|
|
|
|
# Get predictions and labels per instance
|
|
# ***************************************
|
|
|
|
# Stack all validation predictions for each class separately
|
|
max_ind = np.max(batches)
|
|
for b_i, b in enumerate(batches):
|
|
|
|
# Eliminate shadow indices
|
|
b = b[b < max_ind-0.5]
|
|
|
|
# Stack all results
|
|
probs += [prob[b]]
|
|
targets += [labels[b]]
|
|
obj_inds += [o_inds[b_i]]
|
|
|
|
# Average timing
|
|
t += [time.time()]
|
|
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1]))
|
|
|
|
# Display
|
|
if (t[-1] - last_display) > 1.0:
|
|
last_display = t[-1]
|
|
message = 'Validation : {:.1f}% (timings : {:4.2f} {:4.2f})'
|
|
print(message.format(100 * i0 / model.config.validation_size,
|
|
1000 * (mean_dt[0]),
|
|
1000 * (mean_dt[1])))
|
|
|
|
except tf.errors.OutOfRangeError:
|
|
break
|
|
|
|
###################
|
|
# Voting validation
|
|
###################
|
|
|
|
for o_i, o_probs in zip(obj_inds, probs):
|
|
self.val_probs[o_i] = val_smooth * self.val_probs[o_i] + (1 - val_smooth) * o_probs
|
|
|
|
############
|
|
# Confusions
|
|
############
|
|
|
|
# Confusion matrix for each instance
|
|
n_parts = model.config.num_classes
|
|
Confs = np.zeros((len(probs), n_parts, n_parts), dtype=np.int32)
|
|
for i, (pred, truth) in enumerate(zip(probs, targets)):
|
|
parts = [j for j in range(pred.shape[1])]
|
|
Confs[i, :, :] = confusion_matrix(truth, np.argmax(pred, axis=1), parts)
|
|
|
|
# Objects IoU
|
|
IoUs = IoU_from_confusions(Confs)
|
|
|
|
|
|
# Compute votes confusion
|
|
Confs = np.zeros((len(self.val_probs), n_parts, n_parts), dtype=np.int32)
|
|
for i, (pred, truth) in enumerate(zip(self.val_probs, dataset.input_point_labels['validation'])):
|
|
parts = [j for j in range(pred.shape[1])]
|
|
Confs[i, :, :] = confusion_matrix(truth, np.argmax(pred, axis=1), parts)
|
|
|
|
# Objects IoU
|
|
vote_IoUs = IoU_from_confusions(Confs)
|
|
|
|
# Saving (optionnal)
|
|
if model.config.saving:
|
|
|
|
IoU_list = [IoUs, vote_IoUs]
|
|
file_list = ['val_IoUs.txt', 'vote_IoUs.txt']
|
|
for IoUs_to_save, IoU_file in zip(IoU_list, file_list):
|
|
|
|
# Name of saving file
|
|
test_file = join(model.saving_path, IoU_file)
|
|
|
|
# Line to write:
|
|
line = ''
|
|
for instance_IoUs in IoUs_to_save:
|
|
for IoU in instance_IoUs:
|
|
line += '{:.3f} '.format(IoU)
|
|
line = line + '\n'
|
|
|
|
# Write in file
|
|
if exists(test_file):
|
|
with open(test_file, "a") as text_file:
|
|
text_file.write(line)
|
|
else:
|
|
with open(test_file, "w") as text_file:
|
|
text_file.write(line)
|
|
|
|
# Print instance mean
|
|
mIoU = 100 * np.mean(IoUs)
|
|
mIoU2 = 100 * np.mean(vote_IoUs)
|
|
print('{:s} : mIoU = {:.1f}% / vote mIoU = {:.1f}%'.format(model.config.dataset, mIoU, mIoU2))
|
|
|
|
return
|
|
|
|
def cloud_validation_error(self, model, dataset):
|
|
"""
|
|
Validation method for cloud segmentation models
|
|
"""
|
|
|
|
##########
|
|
# Initialize
|
|
##########
|
|
|
|
# Choose validation smoothing parameter (0 for no smothing, 0.99 for big smoothing)
|
|
val_smooth = 0.95
|
|
|
|
# Do not validate if dataset has no validation cloud
|
|
if dataset.validation_split not in dataset.all_splits:
|
|
return
|
|
|
|
# Initialise iterator with train data
|
|
self.sess.run(dataset.val_init_op)
|
|
|
|
# Number of classes including ignored labels
|
|
nc_tot = dataset.num_classes
|
|
|
|
# Number of classes predicted by the model
|
|
nc_model = model.config.num_classes
|
|
|
|
# Initialize global prediction over validation clouds
|
|
if not hasattr(self, 'validation_probs'):
|
|
self.validation_probs = [np.zeros((l.shape[0], nc_model)) for l in dataset.input_labels['validation']]
|
|
self.val_proportions = np.zeros(nc_model, dtype=np.float32)
|
|
i = 0
|
|
for label_value in dataset.label_values:
|
|
if label_value not in dataset.ignored_labels:
|
|
self.val_proportions[i] = np.sum([np.sum(labels == label_value)
|
|
for labels in dataset.validation_labels])
|
|
i += 1
|
|
|
|
#####################
|
|
# Network predictions
|
|
#####################
|
|
|
|
predictions = []
|
|
targets = []
|
|
mean_dt = np.zeros(2)
|
|
last_display = time.time()
|
|
for i0 in range(model.config.validation_size):
|
|
try:
|
|
# Run one step of the model.
|
|
t = [time.time()]
|
|
ops = (self.prob_logits,
|
|
model.labels,
|
|
model.inputs['in_batches'],
|
|
model.inputs['point_inds'],
|
|
model.inputs['cloud_inds'])
|
|
stacked_probs, labels, batches, point_inds, cloud_inds = self.sess.run(ops, {model.dropout_prob: 1.0})
|
|
t += [time.time()]
|
|
|
|
# Get predictions and labels per instance
|
|
# ***************************************
|
|
|
|
# Stack all validation predictions for each class separately
|
|
max_ind = np.max(batches)
|
|
for b_i, b in enumerate(batches):
|
|
|
|
# Eliminate shadow indices
|
|
b = b[b < max_ind-0.5]
|
|
|
|
# Get prediction (only for the concerned parts)
|
|
probs = stacked_probs[b]
|
|
inds = point_inds[b]
|
|
c_i = cloud_inds[b_i]
|
|
|
|
# Update current probs in whole cloud
|
|
self.validation_probs[c_i][inds] = val_smooth * self.validation_probs[c_i][inds] \
|
|
+ (1-val_smooth) * probs
|
|
|
|
# Stack all prediction for this epoch
|
|
predictions += [probs]
|
|
targets += [dataset.input_labels['validation'][c_i][inds]]
|
|
|
|
# Average timing
|
|
t += [time.time()]
|
|
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1]))
|
|
|
|
# Display
|
|
if (t[-1] - last_display) > 1.0:
|
|
last_display = t[-1]
|
|
message = 'Validation : {:.1f}% (timings : {:4.2f} {:4.2f})'
|
|
print(message.format(100 * i0 / model.config.validation_size,
|
|
1000 * (mean_dt[0]),
|
|
1000 * (mean_dt[1])))
|
|
|
|
except tf.errors.OutOfRangeError:
|
|
break
|
|
|
|
# Confusions for our subparts of validation set
|
|
Confs = np.zeros((len(predictions), nc_tot, nc_tot), dtype=np.int32)
|
|
for i, (probs, truth) in enumerate(zip(predictions, targets)):
|
|
|
|
# Insert false columns for ignored labels
|
|
for l_ind, label_value in enumerate(dataset.label_values):
|
|
if label_value in dataset.ignored_labels:
|
|
probs = np.insert(probs, l_ind, 0, axis=1)
|
|
|
|
# Predicted labels
|
|
preds = dataset.label_values[np.argmax(probs, axis=1)]
|
|
|
|
# Confusions
|
|
Confs[i, :, :] = confusion_matrix(truth, preds, dataset.label_values)
|
|
|
|
# Sum all confusions
|
|
C = np.sum(Confs, axis=0).astype(np.float32)
|
|
|
|
# Remove ignored labels from confusions
|
|
for l_ind, label_value in reversed(list(enumerate(dataset.label_values))):
|
|
if label_value in dataset.ignored_labels:
|
|
C = np.delete(C, l_ind, axis=0)
|
|
C = np.delete(C, l_ind, axis=1)
|
|
|
|
# Balance with real validation proportions
|
|
C *= np.expand_dims(self.val_proportions / (np.sum(C, axis=1) + 1e-6), 1)
|
|
|
|
# Objects IoU
|
|
IoUs = IoU_from_confusions(C)
|
|
|
|
# Saving (optionnal)
|
|
if model.config.saving:
|
|
|
|
# Name of saving file
|
|
test_file = join(model.saving_path, 'val_IoUs.txt')
|
|
|
|
# Line to write:
|
|
line = ''
|
|
for IoU in IoUs:
|
|
line += '{:.3f} '.format(IoU)
|
|
line = line + '\n'
|
|
|
|
# Write in file
|
|
if exists(test_file):
|
|
with open(test_file, "a") as text_file:
|
|
text_file.write(line)
|
|
else:
|
|
with open(test_file, "w") as text_file:
|
|
text_file.write(line)
|
|
|
|
# Print instance mean
|
|
mIoU = 100 * np.mean(IoUs)
|
|
print('{:s} mean IoU = {:.1f}%'.format(model.config.dataset, mIoU))
|
|
|
|
# Save predicted cloud occasionally
|
|
if model.config.saving and (self.training_epoch + 1) % model.config.checkpoint_gap == 0:
|
|
val_path = join(model.saving_path, 'val_preds_{:d}'.format(self.training_epoch))
|
|
if not exists(val_path):
|
|
makedirs(val_path)
|
|
files = dataset.train_files
|
|
i_val = 0
|
|
for i, file_path in enumerate(files):
|
|
if dataset.all_splits[i] == dataset.validation_split:
|
|
|
|
# Get points
|
|
points = dataset.load_evaluation_points(file_path)
|
|
|
|
# Get probs on our own ply points
|
|
sub_probs = self.validation_probs[i_val]
|
|
|
|
# Insert false columns for ignored labels
|
|
for l_ind, label_value in enumerate(dataset.label_values):
|
|
if label_value in dataset.ignored_labels:
|
|
sub_probs = np.insert(sub_probs, l_ind, 0, axis=1)
|
|
|
|
# Get the predicted labels
|
|
sub_preds = dataset.label_values[np.argmax(sub_probs, axis=1).astype(np.int32)]
|
|
|
|
# Reproject preds on the evaluations points
|
|
preds = (sub_preds[dataset.validation_proj[i_val]]).astype(np.int32)
|
|
|
|
# Path of saved validation file
|
|
cloud_name = file_path.split('/')[-1]
|
|
val_name = join(val_path, cloud_name)
|
|
|
|
# Save file
|
|
labels = dataset.validation_labels[i_val].astype(np.int32)
|
|
write_ply(val_name,
|
|
[points, preds, labels],
|
|
['x', 'y', 'z', 'preds', 'class'])
|
|
|
|
i_val += 1
|
|
|
|
return
|
|
|
|
def multi_cloud_validation_error(self, model, multi_dataset):
|
|
"""
|
|
Validation method for cloud segmentation models
|
|
"""
|
|
|
|
##########
|
|
# Initialize
|
|
##########
|
|
|
|
# Choose validation smoothing parameter (0 for no smothing, 0.99 for big smoothing)
|
|
val_smooth = 0.95
|
|
|
|
# Initialise iterator with train data
|
|
self.sess.run(multi_dataset.val_init_op)
|
|
|
|
if not hasattr(self, 'validation_probs'):
|
|
|
|
self.validation_probs = []
|
|
self.val_proportions = []
|
|
|
|
for d_i, dataset in enumerate(multi_dataset.datasets):
|
|
|
|
# Do not validate if dataset has no validation cloud
|
|
if dataset.validation_split not in dataset.all_splits:
|
|
continue
|
|
|
|
# Number of classes including ignored labels
|
|
nc_tot = dataset.num_classes
|
|
|
|
# Number of classes predicted by the model
|
|
nc_model = model.config.num_classes[d_i]
|
|
|
|
# Initialize global prediction over validation clouds
|
|
self.validation_probs.append([np.zeros((l.shape[0], nc_model)) for l in dataset.input_labels['validation']])
|
|
self.val_proportions.append(np.zeros(nc_model, dtype=np.float32))
|
|
i = 0
|
|
for label_value in dataset.label_values:
|
|
if label_value not in dataset.ignored_labels:
|
|
self.val_proportions[-1][i] = np.sum([np.sum(labels == label_value)
|
|
for labels in dataset.validation_labels])
|
|
i += 1
|
|
|
|
#####################
|
|
# Network predictions
|
|
#####################
|
|
|
|
pred_d_inds = []
|
|
predictions = []
|
|
targets = []
|
|
mean_dt = np.zeros(2)
|
|
last_display = time.time()
|
|
for i0 in range(model.config.validation_size):
|
|
try:
|
|
# Run one step of the model.
|
|
t = [time.time()]
|
|
ops = (self.val_logits,
|
|
model.labels,
|
|
model.inputs['in_batches'],
|
|
model.inputs['point_inds'],
|
|
model.inputs['cloud_inds'],
|
|
model.inputs['dataset_inds'])
|
|
stacked_probs, labels, batches, p_inds, c_inds, d_inds = self.sess.run(ops, {model.dropout_prob: 1.0})
|
|
t += [time.time()]
|
|
|
|
# Get predictions and labels per instance
|
|
# ***************************************
|
|
|
|
# Stack all validation predictions for each class separately
|
|
max_ind = np.max(batches)
|
|
for b_i, b in enumerate(batches):
|
|
|
|
# Eliminate shadow indices
|
|
b = b[b < max_ind-0.5]
|
|
|
|
# Get prediction (only for the concerned parts)
|
|
d_i = d_inds[b_i]
|
|
probs = stacked_probs[b, :model.config.num_classes[d_i]]
|
|
inds = p_inds[b]
|
|
c_i = c_inds[b_i]
|
|
|
|
# Update current probs in whole cloud
|
|
self.validation_probs[d_i][c_i][inds] = val_smooth * self.validation_probs[d_i][c_i][inds] \
|
|
+ (1-val_smooth) * probs
|
|
|
|
# Stack all prediction for this epoch
|
|
pred_d_inds += [d_i]
|
|
predictions += [probs]
|
|
targets += [multi_dataset.datasets[d_i].input_labels['validation'][c_i][inds]]
|
|
|
|
# Average timing
|
|
t += [time.time()]
|
|
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1]))
|
|
|
|
# Display
|
|
if (t[-1] - last_display) > 1.0:
|
|
last_display = t[-1]
|
|
message = 'Validation : {:.1f}% (timings : {:4.2f} {:4.2f})'
|
|
print(message.format(100 * i0 / model.config.validation_size,
|
|
1000 * (mean_dt[0]),
|
|
1000 * (mean_dt[1])))
|
|
|
|
except tf.errors.OutOfRangeError:
|
|
break
|
|
|
|
# Convert list to np array for indexing
|
|
predictions = np.array(predictions)
|
|
targets = np.array(targets)
|
|
pred_d_inds = np.array(pred_d_inds, np.int32)
|
|
|
|
IoUs = []
|
|
for d_i, dataset in enumerate(multi_dataset.datasets):
|
|
|
|
# Do not validate if dataset has no validation cloud
|
|
if dataset.validation_split not in dataset.all_splits:
|
|
continue
|
|
|
|
# Number of classes including ignored labels
|
|
nc_tot = dataset.num_classes
|
|
|
|
# Number of classes predicted by the model
|
|
nc_model = model.config.num_classes[d_i]
|
|
|
|
# Extract the spheres from this dataset
|
|
tmp_inds = np.where(pred_d_inds == d_i)[0]
|
|
|
|
# Confusions for our subparts of validation set
|
|
Confs = np.zeros((len(tmp_inds), nc_tot, nc_tot), dtype=np.int32)
|
|
for i, (probs, truth) in enumerate(zip(predictions[tmp_inds], targets[tmp_inds])):
|
|
|
|
# Insert false columns for ignored labels
|
|
for l_ind, label_value in enumerate(dataset.label_values):
|
|
if label_value in dataset.ignored_labels:
|
|
probs = np.insert(probs, l_ind, 0, axis=1)
|
|
|
|
# Predicted labels
|
|
preds = dataset.label_values[np.argmax(probs, axis=1)]
|
|
|
|
# Confusions
|
|
Confs[i, :, :] = confusion_matrix(truth, preds, dataset.label_values)
|
|
|
|
# Sum all confusions
|
|
C = np.sum(Confs, axis=0).astype(np.float32)
|
|
|
|
# Remove ignored labels from confusions
|
|
for l_ind, label_value in reversed(list(enumerate(dataset.label_values))):
|
|
if label_value in dataset.ignored_labels:
|
|
C = np.delete(C, l_ind, axis=0)
|
|
C = np.delete(C, l_ind, axis=1)
|
|
|
|
# Balance with real validation proportions
|
|
C *= np.expand_dims(self.val_proportions[d_i] / (np.sum(C, axis=1) + 1e-6), 1)
|
|
|
|
# Objects IoU
|
|
IoUs += [IoU_from_confusions(C)]
|
|
|
|
# Saving (optionnal)
|
|
if model.config.saving:
|
|
|
|
# Name of saving file
|
|
test_file = join(model.saving_path, 'val_IoUs_{:d}_{:s}.txt'.format(d_i, dataset.name))
|
|
|
|
# Line to write:
|
|
line = ''
|
|
for IoU in IoUs[-1]:
|
|
line += '{:.3f} '.format(IoU)
|
|
line = line + '\n'
|
|
|
|
# Write in file
|
|
if exists(test_file):
|
|
with open(test_file, "a") as text_file:
|
|
text_file.write(line)
|
|
else:
|
|
with open(test_file, "w") as text_file:
|
|
text_file.write(line)
|
|
|
|
# Print instance mean
|
|
mIoU = 100 * np.mean(IoUs[-1])
|
|
print('{:s} mean IoU = {:.1f}%'.format(dataset.name, mIoU))
|
|
|
|
# Save predicted cloud occasionally
|
|
if model.config.saving and (self.training_epoch + 1) % model.config.checkpoint_gap == 0:
|
|
val_path = join(model.saving_path, 'val_preds_{:d}'.format(self.training_epoch))
|
|
if not exists(val_path):
|
|
makedirs(val_path)
|
|
|
|
for d_i, dataset in enumerate(multi_dataset.datasets):
|
|
|
|
dataset_val_path = join(val_path, '{:d}_{:s}'.format(d_i, dataset.name))
|
|
if not exists(dataset_val_path):
|
|
makedirs(dataset_val_path)
|
|
|
|
files = dataset.train_files
|
|
i_val = 0
|
|
for i, file_path in enumerate(files):
|
|
if dataset.all_splits[i] == dataset.validation_split:
|
|
|
|
# Get points
|
|
points = dataset.load_evaluation_points(file_path)
|
|
|
|
# Get probs on our own ply points
|
|
sub_probs = self.validation_probs[d_i][i_val]
|
|
|
|
# Insert false columns for ignored labels
|
|
for l_ind, label_value in enumerate(dataset.label_values):
|
|
if label_value in dataset.ignored_labels:
|
|
sub_probs = np.insert(sub_probs, l_ind, 0, axis=1)
|
|
|
|
# Get the predicted labels
|
|
sub_preds = dataset.label_values[np.argmax(sub_probs, axis=1).astype(np.int32)]
|
|
|
|
# Reproject preds on the evaluations points
|
|
preds = (sub_preds[dataset.validation_proj[i_val]]).astype(np.int32)
|
|
|
|
# Path of saved validation file
|
|
cloud_name = file_path.split('/')[-1]
|
|
val_name = join(dataset_val_path, cloud_name)
|
|
|
|
# Save file
|
|
labels = dataset.validation_labels[i_val].astype(np.int32)
|
|
write_ply(val_name,
|
|
[points, preds, labels],
|
|
['x', 'y', 'z', 'preds', 'class'])
|
|
|
|
i_val += 1
|
|
|
|
return
|
|
|
|
def multi_validation_error(self, model, dataset):
|
|
"""
|
|
Validation method for multi object segmentation models
|
|
"""
|
|
|
|
##########
|
|
# Initialize
|
|
##########
|
|
|
|
# Choose validation smoothing parameter (0 for no smothing, 0.99 for big smoothing)
|
|
val_smooth = 0.95
|
|
|
|
# Initialise iterator with train data
|
|
self.sess.run(dataset.val_init_op)
|
|
|
|
# Initialize global prediction over all models
|
|
if not hasattr(self, 'val_probs'):
|
|
self.val_probs = []
|
|
for p_l, o_l in zip(dataset.input_point_labels['validation'], dataset.input_labels['validation']):
|
|
self.val_probs += [np.zeros((len(p_l), dataset.num_parts[o_l]))]
|
|
|
|
#####################
|
|
# Network predictions
|
|
#####################
|
|
|
|
probs = []
|
|
targets = []
|
|
objects = []
|
|
obj_inds = []
|
|
mean_dt = np.zeros(2)
|
|
last_display = time.time()
|
|
for i0 in range(model.config.validation_size):
|
|
try:
|
|
# Run one step of the model.
|
|
t = [time.time()]
|
|
ops = (model.logits,
|
|
model.labels,
|
|
model.inputs['super_labels'],
|
|
model.inputs['object_inds'],
|
|
model.inputs['in_batches'])
|
|
prob, labels, object_labels, o_inds, batches = self.sess.run(ops, {model.dropout_prob: 1.0})
|
|
t += [time.time()]
|
|
|
|
# Get predictions and labels per instance
|
|
# ***************************************
|
|
|
|
# Stack all validation predictions for each class separately
|
|
max_ind = np.max(batches)
|
|
for b_i, b in enumerate(batches):
|
|
|
|
# Eliminate shadow indices
|
|
b = b[b < max_ind-0.5]
|
|
|
|
# Get prediction (only for the concerned parts)
|
|
obj = object_labels[b[0]]
|
|
pred = prob[b][:, :model.config.num_classes[obj]]
|
|
|
|
# Stack all results
|
|
objects += [obj]
|
|
obj_inds += [o_inds[b_i]]
|
|
probs += [prob[b, :model.config.num_classes[obj]]]
|
|
targets += [labels[b]]
|
|
|
|
# Average timing
|
|
t += [time.time()]
|
|
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1]))
|
|
|
|
# Display
|
|
if (t[-1] - last_display) > 1.0:
|
|
last_display = t[-1]
|
|
message = 'Validation : {:.1f}% (timings : {:4.2f} {:4.2f})'
|
|
print(message.format(100 * i0 / model.config.validation_size,
|
|
1000 * (mean_dt[0]),
|
|
1000 * (mean_dt[1])))
|
|
|
|
except tf.errors.OutOfRangeError:
|
|
break
|
|
|
|
###################
|
|
# Voting validation
|
|
###################
|
|
|
|
for o_i, o_probs in zip(obj_inds, probs):
|
|
self.val_probs[o_i] = val_smooth * self.val_probs[o_i] + (1 - val_smooth) * o_probs
|
|
|
|
############
|
|
# Confusions
|
|
############
|
|
|
|
# Confusion matrix for each object
|
|
n_objs = [np.sum(np.array(objects) == l) for l in dataset.label_values]
|
|
Confs = [np.zeros((n_obj, n_parts, n_parts), dtype=np.int32) for n_parts, n_obj in
|
|
zip(dataset.num_parts, n_objs)]
|
|
obj_count = [0 for _ in n_objs]
|
|
for obj, pred, truth in zip(objects, probs, targets):
|
|
parts = [i for i in range(pred.shape[1])]
|
|
Confs[obj][obj_count[obj], :, :] = confusion_matrix(truth, np.argmax(pred, axis=1), parts)
|
|
obj_count[obj] += 1
|
|
|
|
# Objects mIoU
|
|
IoUs = [IoU_from_confusions(C) for C in Confs]
|
|
|
|
|
|
# Compute votes confusion
|
|
n_objs = [np.sum(np.array(dataset.input_labels['validation']) == l) for l in dataset.label_values]
|
|
Confs = [np.zeros((n_obj, n_parts, n_parts), dtype=np.int32) for n_parts, n_obj in
|
|
zip(dataset.num_parts, n_objs)]
|
|
obj_count = [0 for _ in n_objs]
|
|
for obj, pred, truth in zip(dataset.input_labels['validation'],
|
|
self.val_probs,
|
|
dataset.input_point_labels['validation']):
|
|
parts = [i for i in range(pred.shape[1])]
|
|
Confs[obj][obj_count[obj], :, :] = confusion_matrix(truth, np.argmax(pred, axis=1), parts)
|
|
obj_count[obj] += 1
|
|
|
|
# Objects mIoU
|
|
vote_IoUs = [IoU_from_confusions(C) for C in Confs]
|
|
|
|
# Saving (optionnal)
|
|
if model.config.saving:
|
|
|
|
IoU_list = [IoUs, vote_IoUs]
|
|
file_list = ['val_IoUs.txt', 'vote_IoUs.txt']
|
|
|
|
for IoUs_to_save, IoU_file in zip(IoU_list, file_list):
|
|
|
|
# Name of saving file
|
|
test_file = join(model.saving_path, IoU_file)
|
|
|
|
# Line to write:
|
|
line = ''
|
|
for obj_IoUs in IoUs_to_save:
|
|
for part_IoUs in obj_IoUs:
|
|
for IoU in part_IoUs:
|
|
line += '{:.3f} '.format(IoU)
|
|
line += '/ '
|
|
line = line[:-2] + '\n'
|
|
|
|
# Write in file
|
|
if exists(test_file):
|
|
with open(test_file, "a") as text_file:
|
|
text_file.write(line)
|
|
else:
|
|
with open(test_file, "w") as text_file:
|
|
text_file.write(line)
|
|
|
|
# Print instance mean
|
|
mIoU = 100 * np.mean(np.hstack([np.mean(obj_IoUs, axis=1) for obj_IoUs in IoUs]))
|
|
class_mIoUs = [np.mean(obj_IoUs) for obj_IoUs in IoUs]
|
|
mcIoU = 100 * np.mean(class_mIoUs)
|
|
print('Val : mIoU = {:.1f}% / mcIoU = {:.1f}% '.format(mIoU, mcIoU))
|
|
mIoU = 100 * np.mean(np.hstack([np.mean(obj_IoUs, axis=1) for obj_IoUs in vote_IoUs]))
|
|
class_mIoUs = [np.mean(obj_IoUs) for obj_IoUs in vote_IoUs]
|
|
mcIoU = 100 * np.mean(class_mIoUs)
|
|
print('Vote : mIoU = {:.1f}% / mcIoU = {:.1f}% '.format(mIoU, mcIoU))
|
|
|
|
return
|
|
|
|
def slam_validation_error(self, model, dataset):
|
|
"""
|
|
Validation method for slam segmentation models
|
|
"""
|
|
|
|
##########
|
|
# Initialize
|
|
##########
|
|
|
|
# Do not validate if dataset has no validation cloud
|
|
if dataset.validation_split not in dataset.seq_splits:
|
|
return
|
|
|
|
# Create folder for validation predictions
|
|
if not exists (join(model.saving_path, 'val_preds')):
|
|
makedirs(join(model.saving_path, 'val_preds'))
|
|
|
|
# Initialize the dataset validation containers
|
|
dataset.val_points = []
|
|
dataset.val_labels = []
|
|
|
|
# Initialise iterator with train data
|
|
self.sess.run(dataset.val_init_op)
|
|
|
|
# Number of classes including ignored labels
|
|
nc_tot = dataset.num_classes
|
|
|
|
# Number of classes predicted by the model
|
|
nc_model = model.config.num_classes
|
|
|
|
#####################
|
|
# Network predictions
|
|
#####################
|
|
|
|
predictions = []
|
|
targets = []
|
|
inds = []
|
|
mean_dt = np.zeros(2)
|
|
last_display = time.time()
|
|
val_i = 0
|
|
for i0 in range(model.config.validation_size):
|
|
try:
|
|
# Run one step of the model.
|
|
t = [time.time()]
|
|
ops = (self.prob_logits,
|
|
model.labels,
|
|
model.inputs['points'][0],
|
|
model.inputs['in_batches'],
|
|
model.inputs['frame_inds'],
|
|
model.inputs['frame_centers'],
|
|
model.inputs['augment_scales'],
|
|
model.inputs['augment_rotations'])
|
|
s_probs, s_labels, s_points, batches, f_inds, p0s, S, R = self.sess.run(ops, {model.dropout_prob: 1.0})
|
|
t += [time.time()]
|
|
|
|
# Get predictions and labels per instance
|
|
# ***************************************
|
|
|
|
# Stack all validation predictions for each class separately
|
|
max_ind = np.max(batches)
|
|
for b_i, b in enumerate(batches):
|
|
|
|
# Eliminate shadow indices
|
|
b = b[b < max_ind-0.5]
|
|
|
|
# Get prediction (only for the concerned parts)
|
|
probs = s_probs[b]
|
|
labels = s_labels[b]
|
|
points = s_points[b, :]
|
|
S_i = S[b_i]
|
|
R_i = R[b_i]
|
|
p0 = p0s[b_i]
|
|
|
|
# Get input points in their original positions
|
|
points2 = (points * (1/S_i)).dot(R_i.T)
|
|
|
|
# get val_points that are in range
|
|
radiuses = np.sum(np.square(dataset.val_points[val_i] - p0), axis=1)
|
|
mask = radiuses < (0.9 * model.config.in_radius) ** 2
|
|
|
|
# Project predictions on the frame points
|
|
search_tree = KDTree(points2, leaf_size=50)
|
|
proj_inds = search_tree.query(dataset.val_points[val_i][mask, :], return_distance=False)
|
|
proj_inds = np.squeeze(proj_inds).astype(np.int32)
|
|
proj_probs = probs[proj_inds]
|
|
#proj_labels = labels[proj_inds]
|
|
|
|
# Safe check if only one point:
|
|
if proj_probs.ndim < 2:
|
|
proj_probs = np.expand_dims(proj_probs, 0)
|
|
|
|
# Insert false columns for ignored labels
|
|
for l_ind, label_value in enumerate(dataset.label_values):
|
|
if label_value in dataset.ignored_labels:
|
|
proj_probs = np.insert(proj_probs, l_ind, 0, axis=1)
|
|
|
|
# Predicted labels
|
|
preds = dataset.label_values[np.argmax(proj_probs, axis=1)]
|
|
|
|
# Save predictions in a binary file
|
|
filename ='{:02d}_{:07d}.npy'.format(f_inds[b_i, 0], f_inds[b_i, 1])
|
|
filepath = join(model.saving_path, 'val_preds', filename)
|
|
if exists(filepath):
|
|
frame_preds = np.load(filepath)
|
|
else:
|
|
frame_preds = np.zeros(dataset.val_labels[val_i].shape, dtype=np.uint8)
|
|
frame_preds[mask] = preds.astype(np.uint8)
|
|
np.save(filepath, frame_preds)
|
|
|
|
# Save some of the frame pots
|
|
if f_inds[b_i, 1] % 10 == 0:
|
|
pots = dataset.f_potentials['validation'][f_inds[b_i, 0]][f_inds[b_i, 1]]
|
|
write_ply(filepath[:-4]+'_pots.ply',
|
|
[dataset.val_points[val_i], dataset.val_labels[val_i], frame_preds, pots],
|
|
['x', 'y', 'z', 'gt', 'pre', 'pots'])
|
|
|
|
# Update validation confusions
|
|
frame_C = confusion_matrix(dataset.val_labels[val_i], frame_preds, dataset.label_values)
|
|
dataset.val_confs[f_inds[b_i, 0]][f_inds[b_i, 1], :, :] = frame_C
|
|
|
|
# Stack all prediction for this epoch
|
|
predictions += [preds]
|
|
targets += [dataset.val_labels[val_i][mask]]
|
|
inds += [f_inds[b_i, :]]
|
|
val_i += 1
|
|
|
|
# Average timing
|
|
t += [time.time()]
|
|
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1]))
|
|
|
|
# Display
|
|
if (t[-1] - last_display) > 1.0:
|
|
last_display = t[-1]
|
|
message = 'Validation : {:.1f}% (timings : {:4.2f} {:4.2f})'
|
|
print(message.format(100 * i0 / model.config.validation_size,
|
|
1000 * (mean_dt[0]),
|
|
1000 * (mean_dt[1])))
|
|
|
|
except tf.errors.OutOfRangeError:
|
|
break
|
|
|
|
# Confusions for our subparts of validation set
|
|
Confs = np.zeros((len(predictions), nc_tot, nc_tot), dtype=np.int32)
|
|
for i, (preds, truth) in enumerate(zip(predictions, targets)):
|
|
|
|
# Confusions
|
|
Confs[i, :, :] = confusion_matrix(truth, preds, dataset.label_values)
|
|
|
|
#######################################
|
|
# Results on this subpart of validation
|
|
#######################################
|
|
|
|
# Sum all confusions
|
|
C = np.sum(Confs, axis=0).astype(np.float32)
|
|
|
|
# Balance with real validation proportions
|
|
C *= np.expand_dims(dataset.class_proportions['validation'] / (np.sum(C, axis=1) + 1e-6), 1)
|
|
|
|
# Remove ignored labels from confusions
|
|
for l_ind, label_value in reversed(list(enumerate(dataset.label_values))):
|
|
if label_value in dataset.ignored_labels:
|
|
C = np.delete(C, l_ind, axis=0)
|
|
C = np.delete(C, l_ind, axis=1)
|
|
|
|
# Objects IoU
|
|
IoUs = IoU_from_confusions(C)
|
|
|
|
#####################################
|
|
# Results on the whole validation set
|
|
#####################################
|
|
|
|
# Sum all validation confusions
|
|
C_tot = [np.sum(seq_C, axis=0) for seq_C in dataset.val_confs if len(seq_C) > 0]
|
|
C_tot = np.sum(np.stack(C_tot, axis=0), axis=0)
|
|
|
|
s = ''
|
|
for cc in C_tot:
|
|
for c in cc:
|
|
s += '{:8.1f} '.format(c)
|
|
s += '\n'
|
|
print(s)
|
|
|
|
# Remove ignored labels from confusions
|
|
for l_ind, label_value in reversed(list(enumerate(dataset.label_values))):
|
|
if label_value in dataset.ignored_labels:
|
|
C_tot = np.delete(C_tot, l_ind, axis=0)
|
|
C_tot = np.delete(C_tot, l_ind, axis=1)
|
|
|
|
# Objects IoU
|
|
val_IoUs = IoU_from_confusions(C_tot)
|
|
|
|
# Saving (optionnal)
|
|
if model.config.saving:
|
|
|
|
IoU_list = [IoUs, val_IoUs]
|
|
file_list = ['subpart_IoUs.txt', 'val_IoUs.txt']
|
|
for IoUs_to_save, IoU_file in zip(IoU_list, file_list):
|
|
|
|
# Name of saving file
|
|
test_file = join(model.saving_path, IoU_file)
|
|
|
|
# Line to write:
|
|
line = ''
|
|
for IoU in IoUs_to_save:
|
|
line += '{:.3f} '.format(IoU)
|
|
line = line + '\n'
|
|
|
|
# Write in file
|
|
if exists(test_file):
|
|
with open(test_file, "a") as text_file:
|
|
text_file.write(line)
|
|
else:
|
|
with open(test_file, "w") as text_file:
|
|
text_file.write(line)
|
|
|
|
# Print instance mean
|
|
mIoU = 100 * np.mean(IoUs)
|
|
print('{:s} : subpart mIoU = {:.1f} %'.format(model.config.dataset, mIoU))
|
|
mIoU = 100 * np.mean(val_IoUs)
|
|
print('{:s} : val mIoU = {:.1f} %'.format(model.config.dataset, mIoU))
|
|
|
|
return
|
|
|
|
# Saving methods
|
|
# ------------------------------------------------------------------------------------------------------------------
|
|
|
|
def save_kernel_points(self, model, epoch):
|
|
"""
|
|
Method saving kernel point disposition and current model weights for later visualization
|
|
"""
|
|
|
|
if model.config.saving:
|
|
|
|
# Create a directory to save kernels of this epoch
|
|
kernels_dir = join(model.saving_path, 'kernel_points', 'epoch{:d}'.format(epoch))
|
|
if not exists(kernels_dir):
|
|
makedirs(kernels_dir)
|
|
|
|
# Get points
|
|
all_kernel_points_tf = [v for v in tf.global_variables() if 'kernel_points' in v.name
|
|
and v.name.startswith('KernelPoint')]
|
|
all_kernel_points = self.sess.run(all_kernel_points_tf)
|
|
|
|
# Get Extents
|
|
if False and 'gaussian' in model.config.convolution_mode:
|
|
all_kernel_params_tf = [v for v in tf.global_variables() if 'kernel_extents' in v.name
|
|
and v.name.startswith('KernelPoint')]
|
|
all_kernel_params = self.sess.run(all_kernel_params_tf)
|
|
else:
|
|
all_kernel_params = [None for p in all_kernel_points]
|
|
|
|
# Save in ply file
|
|
for kernel_points, kernel_extents, v in zip(all_kernel_points, all_kernel_params, all_kernel_points_tf):
|
|
|
|
# Name of saving file
|
|
ply_name = '_'.join(v.name[:-2].split('/')[1:-1]) + '.ply'
|
|
ply_file = join(kernels_dir, ply_name)
|
|
|
|
# Data to save
|
|
if kernel_points.ndim > 2:
|
|
kernel_points = kernel_points[:, 0, :]
|
|
if False and 'gaussian' in model.config.convolution_mode:
|
|
data = [kernel_points, kernel_extents]
|
|
keys = ['x', 'y', 'z', 'sigma']
|
|
else:
|
|
data = kernel_points
|
|
keys = ['x', 'y', 'z']
|
|
|
|
# Save
|
|
write_ply(ply_file, data, keys)
|
|
|
|
# Get Weights
|
|
all_kernel_weights_tf = [v for v in tf.global_variables() if 'weights' in v.name
|
|
and v.name.startswith('KernelPointNetwork')]
|
|
all_kernel_weights = self.sess.run(all_kernel_weights_tf)
|
|
|
|
# Save in numpy file
|
|
for kernel_weights, v in zip(all_kernel_weights, all_kernel_weights_tf):
|
|
np_name = '_'.join(v.name[:-2].split('/')[1:-1]) + '.npy'
|
|
np_file = join(kernels_dir, np_name)
|
|
np.save(np_file, kernel_weights)
|
|
|
|
# Debug methods
|
|
# ------------------------------------------------------------------------------------------------------------------
|
|
|
|
def show_memory_usage(self, batch_to_feed):
|
|
|
|
for l in range(self.config.num_layers):
|
|
neighb_size = list(batch_to_feed[self.in_neighbors_f32[l]].shape)
|
|
dist_size = neighb_size + [self.config.num_kernel_points, 3]
|
|
dist_memory = np.prod(dist_size) * 4 * 1e-9
|
|
in_feature_size = neighb_size + [self.config.first_features_dim * 2**l]
|
|
in_feature_memory = np.prod(in_feature_size) * 4 * 1e-9
|
|
out_feature_size = [neighb_size[0], self.config.num_kernel_points, self.config.first_features_dim * 2**(l+1)]
|
|
out_feature_memory = np.prod(out_feature_size) * 4 * 1e-9
|
|
|
|
print('Layer {:d} => {:.1f}GB {:.1f}GB {:.1f}GB'.format(l,
|
|
dist_memory,
|
|
in_feature_memory,
|
|
out_feature_memory))
|
|
print('************************************')
|
|
|
|
def debug_nan(self, model, inputs, logits):
|
|
"""
|
|
NaN happened, find where
|
|
"""
|
|
|
|
print('\n\n------------------------ NaN DEBUG ------------------------\n')
|
|
|
|
# First save everything to reproduce error
|
|
file1 = join(model.saving_path, 'all_debug_inputs.pkl')
|
|
with open(file1, 'wb') as f1:
|
|
pickle.dump(inputs, f1)
|
|
|
|
# First save all inputs
|
|
file1 = join(model.saving_path, 'all_debug_logits.pkl')
|
|
with open(file1, 'wb') as f1:
|
|
pickle.dump(logits, f1)
|
|
|
|
# Then print a list of the trainable variables and if they have nan
|
|
print('List of variables :')
|
|
print('*******************\n')
|
|
all_vars = self.sess.run(tf.global_variables())
|
|
for v, value in zip(tf.global_variables(), all_vars):
|
|
nan_percentage = 100 * np.sum(np.isnan(value)) / np.prod(value.shape)
|
|
print(v.name, ' => {:.1f}% of values are NaN'.format(nan_percentage))
|
|
|
|
|
|
print('Inputs :')
|
|
print('********')
|
|
|
|
#Print inputs
|
|
nl = model.config.num_layers
|
|
for layer in range(nl):
|
|
|
|
print('Layer : {:d}'.format(layer))
|
|
|
|
points = inputs[layer]
|
|
neighbors = inputs[nl + layer]
|
|
pools = inputs[2*nl + layer]
|
|
upsamples = inputs[3*nl + layer]
|
|
|
|
nan_percentage = 100 * np.sum(np.isnan(points)) / np.prod(points.shape)
|
|
print('Points =>', points.shape, '{:.1f}% NaN'.format(nan_percentage))
|
|
nan_percentage = 100 * np.sum(np.isnan(neighbors)) / np.prod(neighbors.shape)
|
|
print('neighbors =>', neighbors.shape, '{:.1f}% NaN'.format(nan_percentage))
|
|
nan_percentage = 100 * np.sum(np.isnan(pools)) / np.prod(pools.shape)
|
|
print('pools =>', pools.shape, '{:.1f}% NaN'.format(nan_percentage))
|
|
nan_percentage = 100 * np.sum(np.isnan(upsamples)) / np.prod(upsamples.shape)
|
|
print('upsamples =>', upsamples.shape, '{:.1f}% NaN'.format(nan_percentage))
|
|
|
|
ind = 4 * nl
|
|
features = inputs[ind]
|
|
nan_percentage = 100 * np.sum(np.isnan(features)) / np.prod(features.shape)
|
|
print('features =>', features.shape, '{:.1f}% NaN'.format(nan_percentage))
|
|
ind += 1
|
|
batch_weights = inputs[ind]
|
|
ind += 1
|
|
in_batches = inputs[ind]
|
|
max_b = np.max(in_batches)
|
|
print(in_batches.shape)
|
|
in_b_sizes = np.sum(in_batches < max_b - 0.5, axis=-1)
|
|
print('in_batch_sizes =>', in_b_sizes)
|
|
ind += 1
|
|
out_batches = inputs[ind]
|
|
max_b = np.max(out_batches)
|
|
print(out_batches.shape)
|
|
out_b_sizes = np.sum(out_batches < max_b - 0.5, axis=-1)
|
|
print('out_batch_sizes =>', out_b_sizes)
|
|
ind += 1
|
|
point_labels = inputs[ind]
|
|
print('point labels, ', point_labels.shape, ', values : ', np.unique(point_labels))
|
|
print(np.array([int(100 * np.sum(point_labels == l) / len(point_labels)) for l in np.unique(point_labels)]))
|
|
|
|
ind += 1
|
|
if model.config.dataset.startswith('ShapeNetPart_multi'):
|
|
object_labels = inputs[ind]
|
|
nan_percentage = 100 * np.sum(np.isnan(object_labels)) / np.prod(object_labels.shape)
|
|
print('object_labels =>', object_labels.shape, '{:.1f}% NaN'.format(nan_percentage))
|
|
ind += 1
|
|
augment_scales = inputs[ind]
|
|
ind += 1
|
|
augment_rotations = inputs[ind]
|
|
ind += 1
|
|
|
|
print('\npoolings and upsamples nums :\n')
|
|
|
|
#Print inputs
|
|
for layer in range(nl):
|
|
|
|
print('\nLayer : {:d}'.format(layer))
|
|
|
|
neighbors = inputs[nl + layer]
|
|
pools = inputs[2*nl + layer]
|
|
upsamples = inputs[3*nl + layer]
|
|
|
|
max_n = np.max(neighbors)
|
|
nums = np.sum(neighbors < max_n - 0.5, axis=-1)
|
|
print('min neighbors =>', np.min(nums))
|
|
|
|
if np.prod(pools.shape) > 0:
|
|
max_n = np.max(pools)
|
|
nums = np.sum(pools < max_n - 0.5, axis=-1)
|
|
print('min pools =>', np.min(nums))
|
|
else:
|
|
print('pools empty')
|
|
|
|
|
|
if np.prod(upsamples.shape) > 0:
|
|
max_n = np.max(upsamples)
|
|
nums = np.sum(upsamples < max_n - 0.5, axis=-1)
|
|
print('min upsamples =>', np.min(nums))
|
|
else:
|
|
print('upsamples empty')
|
|
|
|
|
|
print('\nFinished\n\n')
|
|
time.sleep(0.5)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|