From b4e1a9dcc9db8d871a6cf1960239c709fa3c2f63 Mon Sep 17 00:00:00 2001 From: HuguesTHOMAS Date: Mon, 27 Apr 2020 18:01:40 -0400 Subject: [PATCH] Corrections --- datasets/ModelNet40.py | 55 +- datasets/S3DIS.py | 2 - datasets/SemanticKitti.py | 44 +- datasets/common.py | 119 +++- kernels/kernel_points.py | 1 + models/architectures.py | 14 +- models/blocks.py | 40 +- plot_convergence.py | 927 ++------------------------ test_models.py | 16 +- train_ModelNet40.py | 33 +- train_S3DIS.py | 31 +- train_SemanticKitti.py | 17 +- utils/config.py | 40 +- utils/trainer.py | 202 ------ utils/visualizer.py | 1288 ------------------------------------- visualize_deformations.py | 18 +- 16 files changed, 306 insertions(+), 2541 deletions(-) diff --git a/datasets/ModelNet40.py b/datasets/ModelNet40.py index 90c1868..b00630b 100644 --- a/datasets/ModelNet40.py +++ b/datasets/ModelNet40.py @@ -805,38 +805,10 @@ class ModelNet40CustomBatch: return all_p_list - - def ModelNet40Collate(batch_data): return ModelNet40CustomBatch(batch_data) -class ModelNet40WorkerInitDebug(): - """Callable class that Initializes workers.""" - - def __init__(self, dataset): - self.dataset = dataset - return - - def __call__(self, worker_id): - - # Print workers info - worker_info = get_worker_info() - print(worker_info) - - # Get associated dataset - dataset = worker_info.dataset # the dataset copy in this worker process - - # In windows, each worker has its own copy of the dataset. In Linux, this is shared in memory - print(dataset.input_labels.__array_interface__['data']) - print(worker_info.dataset.input_labels.__array_interface__['data']) - print(self.dataset.input_labels.__array_interface__['data']) - - # configure the dataset to only process the split workload - - return - - # ---------------------------------------------------------------------------------------------------------------------- # # Debug functions @@ -994,3 +966,30 @@ def debug_batch_and_neighbors_calib(dataset, sampler, loader): _, counts = np.unique(dataset.input_labels, return_counts=True) print(counts) + + +class ModelNet40WorkerInitDebug: + """Callable class that Initializes workers.""" + + def __init__(self, dataset): + self.dataset = dataset + return + + def __call__(self, worker_id): + + # Print workers info + worker_info = get_worker_info() + print(worker_info) + + # Get associated dataset + dataset = worker_info.dataset # the dataset copy in this worker process + + # In windows, each worker has its own copy of the dataset. In Linux, this is shared in memory + print(dataset.input_labels.__array_interface__['data']) + print(worker_info.dataset.input_labels.__array_interface__['data']) + print(self.dataset.input_labels.__array_interface__['data']) + + # configure the dataset to only process the split workload + + return + diff --git a/datasets/S3DIS.py b/datasets/S3DIS.py index adbe8a7..0816516 100644 --- a/datasets/S3DIS.py +++ b/datasets/S3DIS.py @@ -28,7 +28,6 @@ import numpy as np import pickle import torch import math -#from mayavi import mlab from multiprocessing import Lock @@ -877,7 +876,6 @@ class S3DISDataset(PointCloudDataset): return np.vstack((data['x'], data['y'], data['z'])).T - # ---------------------------------------------------------------------------------------------------------------------- # # Utility classes definition diff --git a/datasets/SemanticKitti.py b/datasets/SemanticKitti.py index 75c493d..cb8d3b7 100644 --- a/datasets/SemanticKitti.py +++ b/datasets/SemanticKitti.py @@ -28,7 +28,6 @@ import numpy as np import pickle import torch import yaml -#from mayavi import mlab from multiprocessing import Lock @@ -245,9 +244,6 @@ class SemanticKittiDataset(PointCloudDataset): merged_labels = np.zeros((0,), dtype=np.int32) merged_coords = np.zeros((0, 4), dtype=np.float32) - # In case of validation also keep original point and reproj indices - - # Get center of the first frame in world coordinates p_origin = np.zeros((1, 4)) p_origin[0, 3] = 1 @@ -346,11 +342,6 @@ class SemanticKittiDataset(PointCloudDataset): # Merge n_frames together ######################### - # Too see yielding speed with debug timings method, collapse points (reduce mapping time to nearly 0) - #merged_points = merged_points[:100, :] - #merged_labels = merged_labels[:100] - #merged_points *= 0.1 - # Subsample merged frames in_pts, in_fts, in_lbls = grid_subsampling(merged_points, features=merged_coords, @@ -455,8 +446,8 @@ class SemanticKittiDataset(PointCloudDataset): else: raise ValueError('Only accepted input dimensions are 1, 4 and 7 (without and with XYZ)') - t += [time.time()] + ####################### # Create network inputs ####################### @@ -546,27 +537,6 @@ class SemanticKittiDataset(PointCloudDataset): ti += 1 print('\n************************\n') - # Timings: (in test configuration) - # Lock ...... 0.1ms - # Init ...... 0.0ms - # Load ...... 40.0ms - # subs ...... 143.6ms - # drop ...... 4.6ms - # reproj .... 297.4ms - # augment ... 7.5ms - # stack ..... 0.0ms - # concat .... 1.4ms - # input ..... 816.0ms - # stack ..... 0.0ms - - # TODO: Where can we gain time for the robot real time test? - # > Load: no disk read necessary + pose useless if we only use one frame for testing - # > Drop: We can drop even more points. Random choice could be faster without replace=False - # > reproj: No reprojection needed - # > Augment: See which data agment we want at test time - # > input: MAIN BOTTLENECK. We need to see if we can do faster, maybe with some parallelisation. neighbors - # and subsampling accelerated with lidar frame order - return [self.config.num_layers] + input_list def load_calib_poses(self): @@ -744,6 +714,12 @@ class SemanticKittiDataset(PointCloudDataset): return poses +# ---------------------------------------------------------------------------------------------------------------------- +# +# Utility classes definition +# \********************************/ + + class SemanticKittiSampler(Sampler): """Sampler for SemanticKitti""" @@ -1390,6 +1366,12 @@ def SemanticKittiCollate(batch_data): return SemanticKittiCustomBatch(batch_data) +# ---------------------------------------------------------------------------------------------------------------------- +# +# Debug functions +# \*********************/ + + def debug_timing(dataset, loader): """Timing of generator function""" diff --git a/datasets/common.py b/datasets/common.py index f5f8a5e..4e86813 100644 --- a/datasets/common.py +++ b/datasets/common.py @@ -74,7 +74,8 @@ def grid_subsampling(points, features=None, labels=None, sampleDl=0.1, verbose=0 verbose=verbose) -def batch_grid_subsampling(points, batches_len, features=None, labels=None, sampleDl=0.1, max_p=0, verbose=0): +def batch_grid_subsampling(points, batches_len, features=None, labels=None, + sampleDl=0.1, max_p=0, verbose=0, random_grid_orient=True): """ CPP wrapper for a grid subsampling (method = barycenter for points and features) :param points: (N, 3) matrix of input points @@ -85,34 +86,100 @@ def batch_grid_subsampling(points, batches_len, features=None, labels=None, samp :return: subsampled points, with features and/or labels depending of the input """ + R = None + B = len(batches_len) + if random_grid_orient: + + ######################################################## + # Create a random rotation matrix for each batch element + ######################################################## + + # Choose two random angles for the first vector in polar coordinates + theta = np.random.rand(B) * 2 * np.pi + phi = (np.random.rand(B) - 0.5) * np.pi + + # Create the first vector in carthesian coordinates + u = np.vstack([np.cos(theta) * np.cos(phi), np.sin(theta) * np.cos(phi), np.sin(phi)]) + + # Choose a random rotation angle + alpha = np.random.rand(B) * 2 * np.pi + + # Create the rotation matrix with this vector and angle + R = create_3D_rotations(u.T, alpha).astype(np.float32) + + ################# + # Apply rotations + ################# + + i0 = 0 + points = points.copy() + for bi, length in enumerate(batches_len): + # Apply the rotation + points[i0:i0 + length, :] = np.sum(np.expand_dims(points[i0:i0 + length, :], 2) * R[bi], axis=1) + i0 += length + + ####################### + # Sunsample and realign + ####################### + if (features is None) and (labels is None): - return cpp_subsampling.subsample_batch(points, - batches_len, - sampleDl=sampleDl, - max_p=max_p, - verbose=verbose) + s_points, s_len = cpp_subsampling.subsample_batch(points, + batches_len, + sampleDl=sampleDl, + max_p=max_p, + verbose=verbose) + if random_grid_orient: + i0 = 0 + for bi, length in enumerate(s_len): + s_points[i0:i0 + length, :] = np.sum(np.expand_dims(s_points[i0:i0 + length, :], 2) * R[bi].T, axis=1) + i0 += length + return s_points, s_len + elif (labels is None): - return cpp_subsampling.subsample_batch(points, - batches_len, - features=features, - sampleDl=sampleDl, - max_p=max_p, - verbose=verbose) + s_points, s_len, s_features = cpp_subsampling.subsample_batch(points, + batches_len, + features=features, + sampleDl=sampleDl, + max_p=max_p, + verbose=verbose) + if random_grid_orient: + i0 = 0 + for bi, length in enumerate(s_len): + # Apply the rotation + s_points[i0:i0 + length, :] = np.sum(np.expand_dims(s_points[i0:i0 + length, :], 2) * R[bi].T, axis=1) + i0 += length + return s_points, s_len, s_features + elif (features is None): - return cpp_subsampling.subsample_batch(points, - batches_len, - classes=labels, - sampleDl=sampleDl, - max_p=max_p, - verbose=verbose) + s_points, s_len, s_labels = cpp_subsampling.subsample_batch(points, + batches_len, + classes=labels, + sampleDl=sampleDl, + max_p=max_p, + verbose=verbose) + if random_grid_orient: + i0 = 0 + for bi, length in enumerate(s_len): + # Apply the rotation + s_points[i0:i0 + length, :] = np.sum(np.expand_dims(s_points[i0:i0 + length, :], 2) * R[bi].T, axis=1) + i0 += length + return s_points, s_len, s_labels + else: - return cpp_subsampling.subsample_batch(points, - batches_len, - features=features, - classes=labels, - sampleDl=sampleDl, - max_p=max_p, - verbose=verbose) + s_points, s_len, s_features, s_labels = cpp_subsampling.subsample_batch(points, + batches_len, + features=features, + classes=labels, + sampleDl=sampleDl, + max_p=max_p, + verbose=verbose) + if random_grid_orient: + i0 = 0 + for bi, length in enumerate(s_len): + # Apply the rotation + s_points[i0:i0 + length, :] = np.sum(np.expand_dims(s_points[i0:i0 + length, :], 2) * R[bi].T, axis=1) + i0 += length + return s_points, s_len, s_features, s_labels def batch_neighbors(queries, supports, q_batches, s_batches, radius): @@ -499,8 +566,6 @@ class PointCloudDataset(Dataset): # Return inputs ############### - # Save deform layers - # list of network inputs li = input_points + input_neighbors + input_pools + input_upsamples + input_stack_lengths li += [stacked_features, labels] diff --git a/kernels/kernel_points.py b/kernels/kernel_points.py index 9eb6d16..b8d7461 100644 --- a/kernels/kernel_points.py +++ b/kernels/kernel_points.py @@ -74,6 +74,7 @@ def create_3D_rotations(axis, angle): return np.reshape(R, (-1, 3, 3)) + def spherical_Lloyd(radius, num_cells, dimension=3, fixed='center', approximation='monte-carlo', approx_n=5000, max_iter=500, momentum=0.9, verbose=0): """ diff --git a/models/architectures.py b/models/architectures.py index b01c8bc..684ab02 100644 --- a/models/architectures.py +++ b/models/architectures.py @@ -44,12 +44,8 @@ def p2p_fitting_regularizer(net): # Fitting loss ############## - # Get the distance to closest input point - - KP_min_d2, _ = torch.min(m.deformed_d2, dim=1) - - # Normalize KP locations to be independant from layers - KP_min_d2 = KP_min_d2 / (m.KP_extent ** 2) + # Get the distance to closest input point and normalize to be independant from layers + KP_min_d2 = m.min_d2 / (m.KP_extent ** 2) # Loss will be the square distance to closest input point. We use L1 because dist is already squared fitting_loss += net.l1(KP_min_d2, torch.zeros_like(KP_min_d2)) @@ -65,11 +61,11 @@ def p2p_fitting_regularizer(net): for i in range(net.K): other_KP = torch.cat([KP_locs[:, :i, :], KP_locs[:, i + 1:, :]], dim=1).detach() distances = torch.sqrt(torch.sum((other_KP - KP_locs[:, i:i + 1, :]) ** 2, dim=2)) - rep_loss = torch.sum(torch.clamp_max(distances - 1.0, max=0.0) ** 2, dim=1) + rep_loss = torch.sum(torch.clamp_max(distances - net.repulse_extent, max=0.0) ** 2, dim=1) repulsive_loss += net.l1(rep_loss, torch.zeros_like(rep_loss)) / net.K # The hook effectively affect both regularizer and output loss. So here we have to divide by deform_loss_power - return (net.deform_fitting_power / net.deform_loss_power) * (fitting_loss + repulsive_loss) + return (net.deform_fitting_power / net.deform_loss_power) * (2 * fitting_loss + repulsive_loss) class KPCNN(nn.Module): @@ -144,6 +140,7 @@ class KPCNN(nn.Module): self.deform_fitting_mode = config.deform_fitting_mode self.deform_fitting_power = config.deform_fitting_power self.deform_loss_power = config.deform_loss_power + self.repulse_extent = config.repulse_extent self.output_loss = 0 self.reg_loss = 0 self.l1 = nn.L1Loss() @@ -329,6 +326,7 @@ class KPFCNN(nn.Module): self.deform_fitting_mode = config.deform_fitting_mode self.deform_fitting_power = config.deform_fitting_power self.deform_loss_power = config.deform_loss_power + self.repulse_extent = config.repulse_extent self.output_loss = 0 self.reg_loss = 0 self.l1 = nn.L1Loss() diff --git a/models/blocks.py b/models/blocks.py index 4a9a241..918cd14 100644 --- a/models/blocks.py +++ b/models/blocks.py @@ -174,8 +174,10 @@ class KPConv(nn.Module): self.deformable = deformable self.modulated = modulated + self.in_offset_channels = in_channels + # Running variable containing deformed KP distance to input points. (used in regularization loss) - self.deformed_d2 = None + self.min_d2 = None self.deformed_KP = None self.offset_features = None @@ -191,7 +193,7 @@ class KPConv(nn.Module): self.offset_dim = self.p_dim * self.K self.offset_conv = KPConv(self.K, self.p_dim, - in_channels, + self.in_offset_channels, self.offset_dim, KP_extent, radius, @@ -241,7 +243,10 @@ class KPConv(nn.Module): ################### if self.deformable: - self.offset_features = self.offset_conv(q_pts, s_pts, neighb_inds, x) + self.offset_bias + + # Get offsets with a KPConv that only takes part of the features + x_offsets = x[:, :self.in_offset_channels] + self.offset_features = self.offset_conv(q_pts, s_pts, neighb_inds, x_offsets) + self.offset_bias if self.modulated: @@ -295,18 +300,33 @@ class KPConv(nn.Module): sq_distances = torch.sum(differences ** 2, dim=3) # Optimization by ignoring points outside a deformed KP range - if False and self.deformable: + if self.deformable: + + # Save distances for loss + self.min_d2, _ = torch.min(sq_distances, dim=1) + # Boolean of the neighbors in range of a kernel point [n_points, n_neighbors] - in_range = torch.any(sq_distances < self.KP_extent ** 2, dim=2) + in_range = torch.any(sq_distances < self.KP_extent ** 2, dim=2).type(torch.int32) # New value of max neighbors new_max_neighb = torch.max(torch.sum(in_range, dim=1)) - print(sq_distances.shape[1], '=>', new_max_neighb.item()) + # For each row of neighbors, indices of the ones that are in range [n_points, new_max_neighb] + neighb_row_bool, neighb_row_inds = torch.topk(in_range, new_max_neighb.item(), dim=1) - # Save distances for loss - if self.deformable: - self.deformed_d2 = sq_distances + # Gather new neighbor indices [n_points, new_max_neighb] + new_neighb_inds = neighb_inds.gather(1, neighb_row_inds, sparse_grad=False) + + # Gather new distances to KP [n_points, new_max_neighb, n_kpoints] + neighb_row_inds.unsqueeze_(2) + neighb_row_inds = neighb_row_inds.expand(-1, -1, self.K) + sq_distances = sq_distances.gather(1, neighb_row_inds, sparse_grad=False) + + # New shadow neighbors have to point to the last shadow point + new_neighb_inds *= neighb_row_bool + new_neighb_inds -= (neighb_row_bool.type(torch.int64) - 1) * int(s_pts.shape[0] - 1) + else: + new_neighb_inds = neighb_inds # Get Kernel point influences [n_points, n_kpoints, n_neighbors] if self.KP_influence == 'constant': @@ -339,7 +359,7 @@ class KPConv(nn.Module): x = torch.cat((x, torch.zeros_like(x[:1, :])), 0) # Get the features of each neighborhood [n_points, n_neighbors, in_fdim] - neighb_x = gather(x, neighb_inds) + neighb_x = gather(x, new_neighb_inds) # Apply distance weights [n_points, n_kpoints, in_fdim] weighted_features = torch.matmul(all_weights, neighb_x) diff --git a/plot_convergence.py b/plot_convergence.py index bc81afe..fa03105 100644 --- a/plot_convergence.py +++ b/plot_convergence.py @@ -80,32 +80,6 @@ def running_mean(signal, n, axis=0, stride=1): return None -def IoU_multi_metrics(all_IoUs, smooth_n): - - # Get mean IoU for consecutive epochs to directly get a mean - all_mIoUs = [np.hstack([np.mean(obj_IoUs, axis=1) for obj_IoUs in epoch_IoUs]) for epoch_IoUs in all_IoUs] - smoothed_mIoUs = [] - for epoch in range(len(all_mIoUs)): - i0 = max(epoch - smooth_n, 0) - i1 = min(epoch + smooth_n + 1, len(all_mIoUs)) - smoothed_mIoUs += [np.mean(np.hstack(all_mIoUs[i0:i1]))] - - # Get mean for each class - all_objs_mIoUs = [[np.mean(obj_IoUs, axis=1) for obj_IoUs in epoch_IoUs] for epoch_IoUs in all_IoUs] - smoothed_obj_mIoUs = [] - for epoch in range(len(all_objs_mIoUs)): - i0 = max(epoch - smooth_n, 0) - i1 = min(epoch + smooth_n + 1, len(all_objs_mIoUs)) - - epoch_obj_mIoUs = [] - for obj in range(len(all_objs_mIoUs[0])): - epoch_obj_mIoUs += [np.mean(np.hstack([objs_mIoUs[obj] for objs_mIoUs in all_objs_mIoUs[i0:i1]]))] - - smoothed_obj_mIoUs += [epoch_obj_mIoUs] - - return np.array(smoothed_mIoUs), np.array(smoothed_obj_mIoUs) - - def IoU_class_metrics(all_IoUs, smooth_n): # Get mean IoU per class for consecutive epochs to directly get a mean without further smoothing @@ -215,66 +189,11 @@ def load_snap_clouds(path, dataset, only_last=False): return cloud_epochs, IoU_from_confusions(Confs) -def load_multi_snap_clouds(path, dataset, file_i, only_last=False): - - cloud_folders = np.array([join(path, f) for f in listdir(path) if f.startswith('val_preds')]) - cloud_epochs = np.array([int(f.split('_')[-1]) for f in cloud_folders]) - epoch_order = np.argsort(cloud_epochs) - cloud_epochs = cloud_epochs[epoch_order] - cloud_folders = cloud_folders[epoch_order] - - if len(cloud_folders) > 0: - dataset_folders = [f for f in listdir(cloud_folders[0]) if dataset.name in f] - cloud_folders = [join(f, dataset_folders[file_i]) for f in cloud_folders] - - Confs = np.zeros((len(cloud_epochs), dataset.num_classes, dataset.num_classes), dtype=np.int32) - for c_i, cloud_folder in enumerate(cloud_folders): - if only_last and c_i < len(cloud_epochs) - 1: - continue - - # Load confusion if previously saved - conf_file = join(cloud_folder, 'conf_{:s}.txt'.format(dataset.name)) - if isfile(conf_file): - Confs[c_i] += np.loadtxt(conf_file, dtype=np.int32) - - else: - for f in listdir(cloud_folder): - if f.endswith('.ply') and not f.endswith('sub.ply'): - if np.any([cloud_path.endswith(f) for cloud_path in dataset.files]): - data = read_ply(join(cloud_folder, f)) - labels = data['class'] - preds = data['preds'] - Confs[c_i] += confusion_matrix(labels, preds, dataset.label_values).astype(np.int32) - - np.savetxt(conf_file, Confs[c_i], '%12d') - - # Erase ply to save disk memory - if c_i < len(cloud_folders) - 1: - for f in listdir(cloud_folder): - if f.endswith('.ply'): - remove(join(cloud_folder, f)) - - # Remove ignored labels from confusions - for l_ind, label_value in reversed(list(enumerate(dataset.label_values))): - if label_value in dataset.ignored_labels: - Confs = np.delete(Confs, l_ind, axis=1) - Confs = np.delete(Confs, l_ind, axis=2) - - return cloud_epochs, IoU_from_confusions(Confs) - - -def load_multi_IoU(filename, n_parts): - - with open(filename, 'r') as f: - lines = f.readlines() - - # Load all IoUs - all_IoUs = [] - for i, line in enumerate(lines): - obj_IoUs = [[float(IoU) for IoU in s.split()] for s in line.split('/')] - obj_IoUs = [np.reshape(IoUs, [-1, n_parts[obj]]) for obj, IoUs in enumerate(obj_IoUs)] - all_IoUs += [obj_IoUs] - return all_IoUs +# ---------------------------------------------------------------------------------------------------------------------- +# +# Plot functions +# \********************/ +# def compare_trainings(list_of_paths, list_of_labels=None): @@ -410,149 +329,6 @@ def compare_trainings(list_of_paths, list_of_labels=None): plt.show() -def compare_convergences_multisegment(list_of_paths, list_of_labels=None): - - # Parameters - # ********** - - steps_per_epoch = 0 - smooth_n = 5 - - if list_of_labels is None: - list_of_labels = [str(i) for i in range(len(list_of_paths))] - - # Read Logs - # ********* - - all_pred_epochs = [] - all_instances_mIoUs = [] - all_objs_mIoUs = [] - all_objs_IoUs = [] - all_parts = [] - - obj_list = ['Air', 'Bag', 'Cap', 'Car', 'Cha', 'Ear', 'Gui', 'Kni', - 'Lam', 'Lap', 'Mot', 'Mug', 'Pis', 'Roc', 'Ska', 'Tab'] - print('Objs | Inst | Air Bag Cap Car Cha Ear Gui Kni Lam Lap Mot Mug Pis Roc Ska Tab') - print('-----|------|--------------------------------------------------------------------------------') - for path in list_of_paths: - - # Load parameters - config = Config() - config.load(path) - - # Get the number of classes - n_parts = [4, 2, 2, 4, 4, 3, 3, 2, 4, 2, 6, 2, 3, 3, 3, 3] - part = config.dataset.split('_')[-1] - - # Get validation confusions - file = join(path, 'val_IoUs.txt') - val_IoUs = load_multi_IoU(file, n_parts) - - file = join(path, 'vote_IoUs.txt') - vote_IoUs = load_multi_IoU(file, n_parts) - - #print(len(val_IoUs[0])) - #print(val_IoUs[0][0].shape) - - # Get mean IoU - #instances_mIoUs, objs_mIoUs = IoU_multi_metrics(val_IoUs, smooth_n) - - # Get mean IoU - instances_mIoUs, objs_mIoUs = IoU_multi_metrics(vote_IoUs, smooth_n) - - # Aggregate results - all_pred_epochs += [np.array([i for i in range(len(val_IoUs))])] - all_instances_mIoUs += [instances_mIoUs] - all_objs_IoUs += [objs_mIoUs] - all_objs_mIoUs += [np.mean(objs_mIoUs, axis=1)] - - if part == 'multi': - s = '{:4.1f} | {:4.1f} | '.format(100 * np.mean(objs_mIoUs[-1]), 100 * instances_mIoUs[-1]) - for obj_mIoU in objs_mIoUs[-1]: - s += '{:4.1f} '.format(100 * obj_mIoU) - print(s) - else: - s = ' -- | -- | ' - for obj_name in obj_list: - if part.startswith(obj_name): - s += '{:4.1f} '.format(100 * instances_mIoUs[-1]) - else: - s += ' -- '.format(100 * instances_mIoUs[-1]) - print(s) - all_parts += [part] - - # Plots - # ***** - - if 'multi' in all_parts: - - # Figure - fig = plt.figure('Instances mIoU') - for i, label in enumerate(list_of_labels): - if all_parts[i] == 'multi': - plt.plot(all_pred_epochs[i], all_instances_mIoUs[i], linewidth=1, label=label) - plt.xlabel('epochs') - plt.ylabel('IoU') - - # Set limits for y axis - #plt.ylim(0.55, 0.95) - - # Display legends and title - plt.legend(loc=4) - - # Customize the graph - ax = fig.gca() - ax.grid(linestyle='-.', which='both') - #ax.set_yticks(np.arange(0.8, 1.02, 0.02)) - - # Figure - fig = plt.figure('mean of categories mIoU') - for i, label in enumerate(list_of_labels): - if all_parts[i] == 'multi': - plt.plot(all_pred_epochs[i], all_objs_mIoUs[i], linewidth=1, label=label) - plt.xlabel('epochs') - plt.ylabel('IoU') - - # Set limits for y axis - #plt.ylim(0.8, 1) - - # Display legends and title - plt.legend(loc=4) - - # Customize the graph - ax = fig.gca() - ax.grid(linestyle='-.', which='both') - #ax.set_yticks(np.arange(0.8, 1.02, 0.02)) - - for obj_i, obj_name in enumerate(obj_list): - if np.any([part.startswith(obj_name) for part in all_parts]): - # Figure - fig = plt.figure(obj_name + ' mIoU') - for i, label in enumerate(list_of_labels): - if all_parts[i] == 'multi': - plt.plot(all_pred_epochs[i], all_objs_IoUs[i][:, obj_i], linewidth=1, label=label) - elif all_parts[i].startswith(obj_name): - plt.plot(all_pred_epochs[i], all_objs_mIoUs[i], linewidth=1, label=label) - plt.xlabel('epochs') - plt.ylabel('IoU') - - # Set limits for y axis - #plt.ylim(0.8, 1) - - # Display legends and title - plt.legend(loc=4) - - # Customize the graph - ax = fig.gca() - ax.grid(linestyle='-.', which='both') - #ax.set_yticks(np.arange(0.8, 1.02, 0.02)) - - - - # Show all - plt.show() - - def compare_convergences_segment(dataset, list_of_paths, list_of_names=None): # Parameters @@ -784,168 +560,6 @@ def compare_convergences_classif(list_of_paths, list_of_labels=None): plt.show() -def compare_convergences_multicloud(list_of_paths, multi, multi_datasets, list_of_names=None): - - # Parameters - # ********** - - smooth_n = 10 - - if list_of_names is None: - list_of_names = [str(i) for i in range(len(list_of_paths))] - - - # Loop on all datasets: - for plot_dataset in multi_datasets: - print('\n') - print(plot_dataset) - print('*'*len(plot_dataset)) - print() - - # Load dataset parameters - if plot_dataset.startswith('S3DIS'): - dataset = S3DISDataset() - elif plot_dataset.startswith('Scann'): - dataset = ScannetDataset() - elif plot_dataset.startswith('Seman'): - dataset = Semantic3DDataset() - elif plot_dataset.startswith('NPM3D'): - dataset = NPM3DDataset() - else: - raise ValueError('Unsupported dataset : ' + plot_dataset) - - # Read Logs - # ********* - - all_pred_epochs = [] - all_mIoUs = [] - all_class_IoUs = [] - all_snap_epochs = [] - all_snap_IoUs = [] - all_names = [] - - class_list = [dataset.label_to_names[label] for label in dataset.label_values - if label not in dataset.ignored_labels] - - s = '{:^10}|'.format('mean') - for c in class_list: - s += '{:^10}'.format(c) - print(s) - print(10*'-' + '|' + 10*dataset.num_classes*'-') - for log_i, (path, is_multi) in enumerate(zip(list_of_paths, multi)): - - n_c = None - if is_multi: - config = MultiConfig() - config.load(path) - if plot_dataset in config.datasets: - val_IoU_files = [] - for d_i in np.where(np.array(config.datasets) == plot_dataset)[0]: - n_c = config.num_classes[d_i] - val_IoU_files.append(join(path, 'val_IoUs_{:d}_{:s}.txt'.format(d_i, plot_dataset))) - else: - continue - else: - config = Config() - config.load(path) - if plot_dataset == config.dataset: - n_c = config.num_classes - val_IoU_files = [join(path, 'val_IoUs.txt')] - else: - continue - - for file_i, file in enumerate(val_IoU_files): - - # Load validation IoUs - val_IoUs = load_single_IoU(file, n_c) - - # Get mean IoU - class_IoUs, mIoUs = IoU_class_metrics(val_IoUs, smooth_n) - - # Aggregate results - all_pred_epochs += [np.array([i for i in range(len(val_IoUs))])] - all_mIoUs += [mIoUs] - all_class_IoUs += [class_IoUs] - all_names += [list_of_names[log_i]+'_{:d}'.format(file_i+1)] - - s = '{:^10.1f}|'.format(100*mIoUs[-1]) - for IoU in class_IoUs[-1]: - s += '{:^10.1f}'.format(100*IoU) - print(s) - - # Get optional full validation on clouds - if is_multi: - snap_epochs, snap_IoUs = load_multi_snap_clouds(path, dataset, file_i) - else: - snap_epochs, snap_IoUs = load_snap_clouds(path, dataset) - all_snap_epochs += [snap_epochs] - all_snap_IoUs += [snap_IoUs] - - print(10*'-' + '|' + 10*dataset.num_classes*'-') - for snap_IoUs in all_snap_IoUs: - if len(snap_IoUs) > 0: - s = '{:^10.1f}|'.format(100*np.mean(snap_IoUs[-1])) - for IoU in snap_IoUs[-1]: - s += '{:^10.1f}'.format(100*IoU) - else: - s = '{:^10s}'.format('-') - for _ in range(dataset.num_classes): - s += '{:^10s}'.format('-') - print(s) - - # Plots - # ***** - - # Figure - fig = plt.figure('mIoUs') - for i, name in enumerate(all_names): - p = plt.plot(all_pred_epochs[i], all_mIoUs[i], '--', linewidth=1, label=name) - plt.plot(all_snap_epochs[i], np.mean(all_snap_IoUs[i], axis=1), linewidth=1, color=p[-1].get_color()) - - plt.title(plot_dataset) - plt.xlabel('epochs') - plt.ylabel('IoU') - - # Set limits for y axis - #plt.ylim(0.55, 0.95) - - # Display legends and title - plt.legend(loc=4) - - # Customize the graph - ax = fig.gca() - ax.grid(linestyle='-.', which='both') - #ax.set_yticks(np.arange(0.8, 1.02, 0.02)) - - displayed_classes = [0, 1, 2, 3, 4, 5, 6, 7] - displayed_classes = [] - for c_i, c_name in enumerate(class_list): - if c_i in displayed_classes: - - # Figure - fig = plt.figure(c_name + ' IoU') - for i, name in enumerate(list_of_names): - plt.plot(all_pred_epochs[i], all_class_IoUs[i][:, c_i], linewidth=1, label=name) - plt.xlabel('epochs') - plt.ylabel('IoU') - - # Set limits for y axis - #plt.ylim(0.8, 1) - - # Display legends and title - plt.legend(loc=4) - - # Customize the graph - ax = fig.gca() - ax.grid(linestyle='-.', which='both') - #ax.set_yticks(np.arange(0.8, 1.02, 0.02)) - - - - # Show all - plt.show() - - def compare_convergences_SLAM(dataset, list_of_paths, list_of_names=None): # Parameters @@ -1069,491 +683,79 @@ def compare_convergences_SLAM(dataset, list_of_paths, list_of_names=None): # ---------------------------------------------------------------------------------------------------------------------- # -# Main Call -# \***************/ +# Experiments +# \*****************/ # -def ModelNet40_first_test(): +def experiment_name_1(): """ - First tries with ModelNet40 - First we compare convergence of a very very deep network on ModelNet40, with our without bn - Then, We try the resuming of previous trainings. Which works quite well. - However in the mean time, we change how validation worked by calling net.eval()/net.train() before/after - validation. It seems that the network perform strange when calling net.eval()/net.train() although it should be the - right way to do it. - Then we try to change BatchNorm1D with InstanceNorm1D and compare with and without calling eval/train at validation. - (Also with a faster lr decay). - --- MISTAKE FOUND --- the batch norm momentum was inverted 0.98 instead of 0.02. - See next experiment for correct convergences. Instance norm seems not as good - """ - - # Using the dates of the logs, you can easily gather consecutive ones. All logs should be of the same dataset. - start = 'Log_2020-03-18_16-04-20' - end = 'Log_2020-03-20_16-59-40' - - if end < 'Log_2020-03-22_19-30-19': - res_path = 'old_results' - else: - res_path = 'results' - - logs = np.sort([join(res_path, l) for l in listdir(res_path) if start <= l <= end]) - - # Give names to the logs (for legends) - logs_names = ['with_bn', - 'without_bn', - 'with_bn2', - 'without_bn2', - 'lrd_80_Inorm_eval_train', - 'lrd_80_Inorm_always_train', - 'test'] - - logs_names = np.array(logs_names[:len(logs)]) - - return logs, logs_names - - -def ModelNet40_batch_norm(): - """ - Compare different type of batch norm now that it has been fixed. Batch norm seems the best easily. Instance norm - crewated a NAN loss so avoid this one. - Now try fast experiments. First reduce network size. Reducing the number of convolution per layer does not affect - results (maybe because dataset is too simple???). 5 small layers is way better that 4 big layers. - Now reduce number of step per epoch and maybe try balanced sampler. Balanced sampler with fewer steps per epoch is - way faster for convergence and gets nearly the same scores. so good for experimenting. However we cant really - conclude between parameters which will get the same score (like the more layers) because the dataset my be - limitating. We can only conclude if something is not good and reduce score. - """ - - # Using the dates of the logs, you can easily gather consecutive ones. All logs should be of the same dataset. - start = 'Log_2020-03-20_16-59-41' - end = 'Log_2020-04-13_18-14-44' - - if end < 'Log_2020-03-22_19-30-19': - res_path = 'old_results' - else: - res_path = 'results' - - logs = np.sort([join(res_path, l) for l in listdir(res_path) if start <= l <= end]) - - # Give names to the logs (for legends) - logs_names = ['no_norm', - 'IN', - 'BN', - '5_small_layer-d0=0.02', - '3_big_layer-d0=0.02', - '3_big_layer-d0=0.04', - 'small-e_n=300', - 'small-e_n=300-balanced_train', - 'small-e_n=300-balanced_traintest', - 'test'] - - logs_names = np.array(logs_names[:len(logs)]) - - return logs, logs_names - - -def ModelNet40_fast_vs_results(): - """ - Try lr decay with fast convergence (epoch_n=300 and balanced traintest). 80 is a good value. - """ - - # Using the dates of the logs, you can easily gather consecutive ones. All logs should be of the same dataset. - start = 'Log_2020-03-21_16-09-17' - end = 'Log_2020-03-21_16-09-36' - - if end < 'Log_2020-03-22_19-30-19': - res_path = 'old_results' - else: - res_path = 'results' - - logs = np.sort([join(res_path, l) for l in listdir(res_path) if start <= l <= end]) - logs = np.insert(logs, 1, join(res_path, 'Log_2020-03-21_11-57-45')) - - # Give names to the logs (for legends) - logs_names = ['lrd=120', - 'lrd=80', - 'lrd=40', - 'test'] - - logs_names = np.array(logs_names[:len(logs)]) - - return logs, logs_names - - -def ModelNet40_grad_clipping(): - """ - Test different grad clipping. No difference so we can move on - """ - - # Using the dates of the logs, you can easily gather consecutive ones. All logs should be of the same dataset. - start = 'Log_2020-03-21_18-21-37' - end = 'Log_2020-03-21_18-30-01' - - if end < 'Log_2020-03-22_19-30-19': - res_path = 'old_results' - else: - res_path = 'results' - - logs = np.sort([join(res_path, l) for l in listdir(res_path) if start <= l <= end]) - logs = np.insert(logs, 0, join(res_path, 'Log_2020-03-21_11-57-45')) - - # Give names to the logs (for legends) - logs_names = ['value_clip_100', - 'norm_clip_100', - 'value_clip_10', - 'norm_clip_10', - 'no_clip', - 'test'] - - logs_names = np.array(logs_names[:len(logs)]) - - return logs, logs_names - - -def ModelNet40_KP_extent(): - """ - Test differents mode et kp extent. sum et extent=2.0 definitivement moins bon (trop de recouvrement des kp - influences, noyau moins versatile). les closest semble plutot bon et le sum extent=1.5 pas mal du tout () peut - etre le meilleur. A confirmer sur gros dataset - """ - - # Using the dates of the logs, you can easily gather consecutive ones. All logs should be of the same dataset. - start = 'Log_2020-03-21_18-30-02' - end = 'Log_2020-03-21_23-36-18' - - if end < 'Log_2020-03-22_19-30-19': - res_path = 'old_results' - else: - res_path = 'results' - - logs = np.sort([join(res_path, l) for l in listdir(res_path) if start <= l <= end]) - logs = np.insert(logs, 0, join(res_path, 'Log_2020-03-21_11-57-45')) - - # Give names to the logs (for legends) - logs_names = ['KPe=1.0_sum_linear', - 'KPe=1.5_sum_linear', - 'KPe=2.0_sum_linear', - 'KPe=1.5_closest_linear', - 'KPe=2.0_closest_linear', - 'test'] - - logs_names = np.array(logs_names[:len(logs)]) - - return logs, logs_names - - -def ModelNet40_gaussian(): - """ - Test different extent in gaussian mode. extent=1.5 seems the best. 2.0 is not bad. But in any case, it does not - perform better than 1.5-linear-sum at least on this dataset. - """ - - # Using the dates of the logs, you can easily gather consecutive ones. All logs should be of the same dataset. - start = 'Log_2020-03-21_23-36-19' - end = 'Log_2020-04-13_18-14-44' - - if end < 'Log_2020-03-22_19-30-19': - res_path = 'old_results' - else: - res_path = 'results' - - logs = np.sort([join(res_path, l) for l in listdir(res_path) if start <= l <= end]) - logs = np.insert(logs, 4, join(res_path, 'Log_2020-03-21_19-35-11')) - - # Give names to the logs (for legends) - logs_names = ['KPe=1.0_sum_gaussian', - 'KPe=1.5_sum_gaussian', - 'KPe=2.0_sum_gaussian', - 'KPe=2.5_sum_gaussian', - 'KPe=1.5_sum_linear', - 'test'] - - logs_names = np.array(logs_names[:len(logs)]) - - return logs, logs_names - - -def ModelNet40_normals(): - """ - Test different way to add normals. Seems pretty much the same and we dont care about normals. - """ - - # Using the dates of the logs, you can easily gather consecutive ones. All logs should be of the same dataset. - start = 'Log_2020-03-22_10-18-56' - end = 'Log_2020-03-22_13-32-51' - - if end < 'Log_2020-03-22_19-30-19': - res_path = 'old_results' - else: - res_path = 'results' - - logs = np.sort([join(res_path, l) for l in listdir(res_path) if start <= l <= end]) - logs = np.insert(logs, 0, join(res_path, 'Log_2020-03-21_19-35-11')) - - # Give names to the logs (for legends) - logs_names = ['no_normals', - 'anisotropic_scale_normals', - 'wrong_scale_normals', - 'only_symmetries_normals(cheating)', - 'test'] - - logs_names = np.array(logs_names[:len(logs)]) - - return logs, logs_names - - -def ModelNet40_radius(): - """ - Test different convolution radius. It was expected that larger radius would means slower networks but better - performances. In fact we do not see much difference (again because of the dataset maybe?) - """ - - # Using the dates of the logs, you can easily gather consecutive ones. All logs should be of the same dataset. - start = 'Log_2020-03-22_13-32-52' - end = 'Log_2020-03-22_19-30-17' - - if end < 'Log_2020-03-22_19-30-19': - res_path = 'old_results' - else: - res_path = 'results' - - logs = np.sort([join(res_path, l) for l in listdir(res_path) if start <= l <= end]) - logs = np.insert(logs, 2, join(res_path, 'Log_2020-03-21_19-35-11')) - - # Give names to the logs (for legends) - logs_names = ['KPe=0.9_r=1.5', - 'KPe=1.2_r=2.0', - 'KPe=1.5_r=2.5', - 'KPe=1.8_r=3.0', - 'KPe=2.1_r=3.5', - 'test'] - - logs_names = np.array(logs_names[:len(logs)]) - - return logs, logs_names - - -def ModelNet40_deform(old_result_limit): - """ - Test deformable convolution with different offset decay. Without modulations 0.01 seems the best. With - modulations 0.1 seems the best. In all cases 1.0 is to much. We need to show deformations for verification. - - It seems that deformations are not really fittig the point cloud. They just reach further away. W need to try on - other datasets and with deformations earlier to see if fitting loss works - """ - - # Using the dates of the logs, you can easily gather consecutive ones. All logs should be of the same dataset. - start = 'Log_2020-03-22_19-30-21' - end = 'Log_2020-03-25_19-30-17' - - if end < old_result_limit: - res_path = 'old_results' - else: - res_path = 'results' - - logs = np.sort([join(res_path, l) for l in listdir(res_path) if start <= l <= end]) - logs = logs.astype(' ModelNet40 - # You can also choose the index of the snapshot to load (last by default) - chkp_idx = -1 + # Choose the index of the checkpoint to load OR None if you want to load the current checkpoint + chkp_idx = None # Choose to test on validation or test split on_val = True @@ -111,7 +111,7 @@ if __name__ == '__main__': ############################ # Set which gpu is going to be used - GPU_ID = '1' + GPU_ID = '0' # Set GPU visible device os.environ['CUDA_VISIBLE_DEVICES'] = GPU_ID @@ -215,12 +215,4 @@ if __name__ == '__main__': elif config.dataset_task == 'slam_segmentation': tester.slam_segmentation_test(net, test_loader, config) else: - raise ValueError('Unsupported dataset_task for testing: ' + config.dataset_task) - - - # TODO: For test and also for training. When changing epoch do not restart the worker initiation. Keep workers - # active with a while loop instead of using for loops. - # For training and validation, keep two sets of worker active in parallel? is it possible? - - # TODO: We have to verify if training on smaller spheres and testing on whole frame changes the score because - # batchnorm may not have the same result as distribution of points will be different. \ No newline at end of file + raise ValueError('Unsupported dataset_task for testing: ' + config.dataset_task) \ No newline at end of file diff --git a/train_ModelNet40.py b/train_ModelNet40.py index 40af46c..c8cb1b2 100644 --- a/train_ModelNet40.py +++ b/train_ModelNet40.py @@ -70,14 +70,19 @@ class Modelnet40Config(Config): # Define layers architecture = ['simple', - 'resnetb_strided', 'resnetb', 'resnetb_strided', 'resnetb', + 'resnetb', 'resnetb_strided', - 'resnetb_deformable', - 'resnetb_deformable_strided', - 'resnetb_deformable', + 'resnetb', + 'resnetb', + 'resnetb_strided', + 'resnetb', + 'resnetb', + 'resnetb_strided', + 'resnetb', + 'resnetb', 'global_average'] ################### @@ -97,7 +102,7 @@ class Modelnet40Config(Config): deform_radius = 6.0 # Radius of the area of influence of each kernel point in "number grid cell". (1.0 is the standard value) - KP_extent = 1.5 + KP_extent = 1.2 # Behavior of convolutions in ('constant', 'linear', 'gaussian') KP_influence = 'linear' @@ -115,11 +120,13 @@ class Modelnet40Config(Config): use_batch_norm = True batch_norm_momentum = 0.05 - # Offset loss - # 'permissive' only constrains offsets inside the deform radius - # 'fitting' helps deformed kernels to adapt to the geometry by penalizing distance to input points - offsets_loss = 'fitting' - offsets_decay = 0.1 + # Deformable offset loss + # 'point2point' fitting geometry by penalizing distance from deform point to input points + # 'point2plane' fitting geometry by penalizing distance from deform point to input point triplet (not implemented) + deform_fitting_mode = 'point2point' + deform_fitting_power = 0.1 # Multiplier for the fitting/repulsive loss + deform_loss_power = 0.1 # Multiplier for output loss applied to the deformations + repulse_extent = 0.8 # Distance of repulsion for deformed kernel points ##################### # Training parameters @@ -131,11 +138,11 @@ class Modelnet40Config(Config): # Learning rate management learning_rate = 1e-2 momentum = 0.98 - lr_decays = {i: 0.1**(1/80) for i in range(1, max_epoch)} + lr_decays = {i: 0.1**(1/100) for i in range(1, max_epoch)} grad_clip_norm = 100.0 # Number of batch - batch_num = 16 + batch_num = 10 # Number of steps per epochs epoch_steps = 300 @@ -179,7 +186,7 @@ if __name__ == '__main__': ############################ # Set which gpu is going to be used - GPU_ID = '3' + GPU_ID = '0' # Set GPU visible device os.environ['CUDA_VISIBLE_DEVICES'] = GPU_ID diff --git a/train_S3DIS.py b/train_S3DIS.py index cf7a018..3da3146 100644 --- a/train_S3DIS.py +++ b/train_S3DIS.py @@ -59,7 +59,7 @@ class S3DISConfig(Config): dataset_task = '' # Number of CPU threads for the input pipeline - input_threads = 20 + input_threads = 10 ######################### # Architecture definition @@ -72,14 +72,14 @@ class S3DISConfig(Config): 'resnetb', 'resnetb', 'resnetb_strided', - 'resnetb', - 'resnetb', - 'resnetb_strided', - 'resnetb', - 'resnetb', - 'resnetb_strided', - 'resnetb', - 'resnetb', + 'resnetb_deformable', + 'resnetb_deformable', + 'resnetb_deformable_strided', + 'resnetb_deformable', + 'resnetb_deformable', + 'resnetb_deformable_strided', + 'resnetb_deformable', + 'resnetb_deformable', 'nearest_upsample', 'unary', 'nearest_upsample', @@ -109,7 +109,7 @@ class S3DISConfig(Config): deform_radius = 6.0 # Radius of the area of influence of each kernel point in "number grid cell". (1.0 is the standard value) - KP_extent = 1.5 + KP_extent = 1.2 # Behavior of convolutions in ('constant', 'linear', 'gaussian') KP_influence = 'linear' @@ -128,12 +128,13 @@ class S3DISConfig(Config): use_batch_norm = True batch_norm_momentum = 0.02 - # Offset loss + # Deformable offset loss # 'point2point' fitting geometry by penalizing distance from deform point to input points - # 'point2plane' fitting geometry by penalizing distance from deform point to input point triplet + # 'point2plane' fitting geometry by penalizing distance from deform point to input point triplet (not implemented) deform_fitting_mode = 'point2point' - deform_fitting_power = 0.05 - deform_loss_power = 0.5 + deform_fitting_power = 0.1 # Multiplier for the fitting/repulsive loss + deform_loss_power = 0.1 # Multiplier for output loss applied to the deformations + repulse_extent = 0.8 # Distance of repulsion for deformed kernel points ##################### # Training parameters @@ -193,7 +194,7 @@ if __name__ == '__main__': ############################ # Set which gpu is going to be used - GPU_ID = '2' + GPU_ID = '0' # Set GPU visible device os.environ['CUDA_VISIBLE_DEVICES'] = GPU_ID diff --git a/train_SemanticKitti.py b/train_SemanticKitti.py index 7d4857f..d4579c5 100644 --- a/train_SemanticKitti.py +++ b/train_SemanticKitti.py @@ -123,7 +123,7 @@ class SemanticKittiConfig(Config): deform_radius = 6.0 # Radius of the area of influence of each kernel point in "number grid cell". (1.0 is the standard value) - KP_extent = 1.5 + KP_extent = 1.2 # Behavior of convolutions in ('constant', 'linear', 'gaussian') KP_influence = 'linear' @@ -142,11 +142,13 @@ class SemanticKittiConfig(Config): use_batch_norm = True batch_norm_momentum = 0.02 - # Offset loss - # 'permissive' only constrains offsets inside the deform radius (NOT implemented yet) - # 'fitting' helps deformed kernels to adapt to the geometry by penalizing distance to input points - offsets_loss = 'fitting' - offsets_decay = 0.01 + # Deformable offset loss + # 'point2point' fitting geometry by penalizing distance from deform point to input points + # 'point2plane' fitting geometry by penalizing distance from deform point to input point triplet (not implemented) + deform_fitting_mode = 'point2point' + deform_fitting_power = 0.1 # Multiplier for the fitting/repulsive loss + deform_loss_power = 0.1 # Multiplier for output loss applied to the deformations + repulse_extent = 0.8 # Distance of repulsion for deformed kernel points ##################### # Training parameters @@ -193,7 +195,6 @@ class SemanticKittiConfig(Config): # class_w = [1.430, 5.000, 5.000, 4.226, 5.000, 5.000, 5.000, 5.000, 0.719, 2.377, # 0.886, 3.863, 0.869, 1.209, 0.594, 3.780, 1.129, 5.000, 5.000] - # Do we nee to save convergence saving = True saving_path = None @@ -212,7 +213,7 @@ if __name__ == '__main__': ############################ # Set which gpu is going to be used - GPU_ID = '2' + GPU_ID = '0' # Set GPU visible device os.environ['CUDA_VISIBLE_DEVICES'] = GPU_ID diff --git a/utils/config.py b/utils/config.py index c77083e..688f735 100644 --- a/utils/config.py +++ b/utils/config.py @@ -159,10 +159,13 @@ class Config: # Choose weights for class (used in segmentation loss). Empty list for no weights class_w = [] - # New offset regularization parameters + # Deformable offset loss + # 'point2point' fitting geometry by penalizing distance from deform point to input points + # 'point2plane' fitting geometry by penalizing distance from deform point to input point triplet (not implemented) deform_fitting_mode = 'point2point' - deform_fitting_power = 0.05 - deform_loss_power = 0.5 + deform_fitting_power = 0.1 # Multiplier for the fitting/repulsive loss + deform_loss_power = 0.1 # Multiplier for output loss applied to the deformations + repulse_extent = 1.0 # Distance of repulsion for deformed kernel points # Number of batch batch_num = 10 @@ -293,7 +296,7 @@ class Config: text_file.write('num_classes = {:d}\n'.format(self.num_classes)) text_file.write('in_points_dim = {:d}\n'.format(self.in_points_dim)) text_file.write('in_features_dim = {:d}\n'.format(self.in_features_dim)) - text_file.write('in_radius = {:.3f}\n'.format(self.in_radius)) + text_file.write('in_radius = {:.6f}\n'.format(self.in_radius)) text_file.write('input_threads = {:d}\n\n'.format(self.input_threads)) # Model parameters @@ -309,26 +312,26 @@ class Config: text_file.write('num_layers = {:d}\n'.format(self.num_layers)) text_file.write('first_features_dim = {:d}\n'.format(self.first_features_dim)) text_file.write('use_batch_norm = {:d}\n'.format(int(self.use_batch_norm))) - text_file.write('batch_norm_momentum = {:.3f}\n\n'.format(self.batch_norm_momentum)) - text_file.write('segmentation_ratio = {:.3f}\n\n'.format(self.segmentation_ratio)) + text_file.write('batch_norm_momentum = {:.6f}\n\n'.format(self.batch_norm_momentum)) + text_file.write('segmentation_ratio = {:.6f}\n\n'.format(self.segmentation_ratio)) # KPConv parameters text_file.write('# KPConv parameters\n') text_file.write('# *****************\n\n') - text_file.write('first_subsampling_dl = {:.3f}\n'.format(self.first_subsampling_dl)) + text_file.write('first_subsampling_dl = {:.6f}\n'.format(self.first_subsampling_dl)) text_file.write('num_kernel_points = {:d}\n'.format(self.num_kernel_points)) - text_file.write('conv_radius = {:.3f}\n'.format(self.conv_radius)) - text_file.write('deform_radius = {:.3f}\n'.format(self.deform_radius)) + text_file.write('conv_radius = {:.6f}\n'.format(self.conv_radius)) + text_file.write('deform_radius = {:.6f}\n'.format(self.deform_radius)) text_file.write('fixed_kernel_points = {:s}\n'.format(self.fixed_kernel_points)) - text_file.write('KP_extent = {:.3f}\n'.format(self.KP_extent)) + text_file.write('KP_extent = {:.6f}\n'.format(self.KP_extent)) text_file.write('KP_influence = {:s}\n'.format(self.KP_influence)) text_file.write('aggregation_mode = {:s}\n'.format(self.aggregation_mode)) text_file.write('modulated = {:d}\n'.format(int(self.modulated))) text_file.write('n_frames = {:d}\n'.format(self.n_frames)) text_file.write('max_in_points = {:d}\n\n'.format(self.max_in_points)) text_file.write('max_val_points = {:d}\n\n'.format(self.max_val_points)) - text_file.write('val_radius = {:.3f}\n\n'.format(self.val_radius)) + text_file.write('val_radius = {:.6f}\n\n'.format(self.val_radius)) # Training parameters text_file.write('# Training parameters\n') @@ -350,22 +353,23 @@ class Config: text_file.write('augment_rotation = {:s}\n'.format(self.augment_rotation)) text_file.write('augment_noise = {:f}\n'.format(self.augment_noise)) text_file.write('augment_occlusion = {:s}\n'.format(self.augment_occlusion)) - text_file.write('augment_occlusion_ratio = {:.3f}\n'.format(self.augment_occlusion_ratio)) + text_file.write('augment_occlusion_ratio = {:.6f}\n'.format(self.augment_occlusion_ratio)) text_file.write('augment_occlusion_num = {:d}\n'.format(self.augment_occlusion_num)) text_file.write('augment_scale_anisotropic = {:d}\n'.format(int(self.augment_scale_anisotropic))) - text_file.write('augment_scale_min = {:.3f}\n'.format(self.augment_scale_min)) - text_file.write('augment_scale_max = {:.3f}\n'.format(self.augment_scale_max)) - text_file.write('augment_color = {:.3f}\n\n'.format(self.augment_color)) + text_file.write('augment_scale_min = {:.6f}\n'.format(self.augment_scale_min)) + text_file.write('augment_scale_max = {:.6f}\n'.format(self.augment_scale_max)) + text_file.write('augment_color = {:.6f}\n\n'.format(self.augment_color)) text_file.write('weight_decay = {:f}\n'.format(self.weight_decay)) text_file.write('segloss_balance = {:s}\n'.format(self.segloss_balance)) text_file.write('class_w =') for a in self.class_w: - text_file.write(' {:.3f}'.format(a)) + text_file.write(' {:.6f}'.format(a)) text_file.write('\n') text_file.write('deform_fitting_mode = {:s}\n'.format(self.deform_fitting_mode)) - text_file.write('deform_fitting_power = {:f}\n'.format(self.deform_fitting_power)) - text_file.write('deform_loss_power = {:f}\n'.format(self.deform_loss_power)) + text_file.write('deform_fitting_power = {:.6f}\n'.format(self.deform_fitting_power)) + text_file.write('deform_loss_power = {:.6f}\n'.format(self.deform_loss_power)) + text_file.write('repulse_extent = {:.6f}\n'.format(self.repulse_extent)) text_file.write('batch_num = {:d}\n'.format(self.batch_num)) text_file.write('val_batch_num = {:d}\n'.format(self.val_batch_num)) text_file.write('max_epoch = {:d}\n'.format(self.max_epoch)) diff --git a/utils/trainer.py b/utils/trainer.py index 1223d11..c4a255e 100644 --- a/utils/trainer.py +++ b/utils/trainer.py @@ -894,208 +894,6 @@ class ModelTrainer: - # Saving methods - # ------------------------------------------------------------------------------------------------------------------ - - def save_kernel_points(self, model, epoch): - """ - Method saving kernel point disposition and current model weights for later visualization - """ - - if model.config.saving: - - # Create a directory to save kernels of this epoch - kernels_dir = join(model.saving_path, 'kernel_points', 'epoch{:d}'.format(epoch)) - if not exists(kernels_dir): - makedirs(kernels_dir) - - # Get points - all_kernel_points_tf = [v for v in tf.global_variables() if 'kernel_points' in v.name - and v.name.startswith('KernelPoint')] - all_kernel_points = self.sess.run(all_kernel_points_tf) - - # Get Extents - if False and 'gaussian' in model.config.convolution_mode: - all_kernel_params_tf = [v for v in tf.global_variables() if 'kernel_extents' in v.name - and v.name.startswith('KernelPoint')] - all_kernel_params = self.sess.run(all_kernel_params_tf) - else: - all_kernel_params = [None for p in all_kernel_points] - - # Save in ply file - for kernel_points, kernel_extents, v in zip(all_kernel_points, all_kernel_params, all_kernel_points_tf): - - # Name of saving file - ply_name = '_'.join(v.name[:-2].split('/')[1:-1]) + '.ply' - ply_file = join(kernels_dir, ply_name) - - # Data to save - if kernel_points.ndim > 2: - kernel_points = kernel_points[:, 0, :] - if False and 'gaussian' in model.config.convolution_mode: - data = [kernel_points, kernel_extents] - keys = ['x', 'y', 'z', 'sigma'] - else: - data = kernel_points - keys = ['x', 'y', 'z'] - - # Save - write_ply(ply_file, data, keys) - - # Get Weights - all_kernel_weights_tf = [v for v in tf.global_variables() if 'weights' in v.name - and v.name.startswith('KernelPointNetwork')] - all_kernel_weights = self.sess.run(all_kernel_weights_tf) - - # Save in numpy file - for kernel_weights, v in zip(all_kernel_weights, all_kernel_weights_tf): - np_name = '_'.join(v.name[:-2].split('/')[1:-1]) + '.npy' - np_file = join(kernels_dir, np_name) - np.save(np_file, kernel_weights) - - # Debug methods - # ------------------------------------------------------------------------------------------------------------------ - - def show_memory_usage(self, batch_to_feed): - - for l in range(self.config.num_layers): - neighb_size = list(batch_to_feed[self.in_neighbors_f32[l]].shape) - dist_size = neighb_size + [self.config.num_kernel_points, 3] - dist_memory = np.prod(dist_size) * 4 * 1e-9 - in_feature_size = neighb_size + [self.config.first_features_dim * 2**l] - in_feature_memory = np.prod(in_feature_size) * 4 * 1e-9 - out_feature_size = [neighb_size[0], self.config.num_kernel_points, self.config.first_features_dim * 2**(l+1)] - out_feature_memory = np.prod(out_feature_size) * 4 * 1e-9 - - print('Layer {:d} => {:.1f}GB {:.1f}GB {:.1f}GB'.format(l, - dist_memory, - in_feature_memory, - out_feature_memory)) - print('************************************') - - def debug_nan(self, model, inputs, logits): - """ - NaN happened, find where - """ - - print('\n\n------------------------ NaN DEBUG ------------------------\n') - - # First save everything to reproduce error - file1 = join(model.saving_path, 'all_debug_inputs.pkl') - with open(file1, 'wb') as f1: - pickle.dump(inputs, f1) - - # First save all inputs - file1 = join(model.saving_path, 'all_debug_logits.pkl') - with open(file1, 'wb') as f1: - pickle.dump(logits, f1) - - # Then print a list of the trainable variables and if they have nan - print('List of variables :') - print('*******************\n') - all_vars = self.sess.run(tf.global_variables()) - for v, value in zip(tf.global_variables(), all_vars): - nan_percentage = 100 * np.sum(np.isnan(value)) / np.prod(value.shape) - print(v.name, ' => {:.1f}% of values are NaN'.format(nan_percentage)) - - - print('Inputs :') - print('********') - - #Print inputs - nl = model.config.num_layers - for layer in range(nl): - - print('Layer : {:d}'.format(layer)) - - points = inputs[layer] - neighbors = inputs[nl + layer] - pools = inputs[2*nl + layer] - upsamples = inputs[3*nl + layer] - - nan_percentage = 100 * np.sum(np.isnan(points)) / np.prod(points.shape) - print('Points =>', points.shape, '{:.1f}% NaN'.format(nan_percentage)) - nan_percentage = 100 * np.sum(np.isnan(neighbors)) / np.prod(neighbors.shape) - print('neighbors =>', neighbors.shape, '{:.1f}% NaN'.format(nan_percentage)) - nan_percentage = 100 * np.sum(np.isnan(pools)) / np.prod(pools.shape) - print('pools =>', pools.shape, '{:.1f}% NaN'.format(nan_percentage)) - nan_percentage = 100 * np.sum(np.isnan(upsamples)) / np.prod(upsamples.shape) - print('upsamples =>', upsamples.shape, '{:.1f}% NaN'.format(nan_percentage)) - - ind = 4 * nl - features = inputs[ind] - nan_percentage = 100 * np.sum(np.isnan(features)) / np.prod(features.shape) - print('features =>', features.shape, '{:.1f}% NaN'.format(nan_percentage)) - ind += 1 - batch_weights = inputs[ind] - ind += 1 - in_batches = inputs[ind] - max_b = np.max(in_batches) - print(in_batches.shape) - in_b_sizes = np.sum(in_batches < max_b - 0.5, axis=-1) - print('in_batch_sizes =>', in_b_sizes) - ind += 1 - out_batches = inputs[ind] - max_b = np.max(out_batches) - print(out_batches.shape) - out_b_sizes = np.sum(out_batches < max_b - 0.5, axis=-1) - print('out_batch_sizes =>', out_b_sizes) - ind += 1 - point_labels = inputs[ind] - print('point labels, ', point_labels.shape, ', values : ', np.unique(point_labels)) - print(np.array([int(100 * np.sum(point_labels == l) / len(point_labels)) for l in np.unique(point_labels)])) - - ind += 1 - if model.config.dataset.startswith('ShapeNetPart_multi'): - object_labels = inputs[ind] - nan_percentage = 100 * np.sum(np.isnan(object_labels)) / np.prod(object_labels.shape) - print('object_labels =>', object_labels.shape, '{:.1f}% NaN'.format(nan_percentage)) - ind += 1 - augment_scales = inputs[ind] - ind += 1 - augment_rotations = inputs[ind] - ind += 1 - - print('\npoolings and upsamples nums :\n') - - #Print inputs - for layer in range(nl): - - print('\nLayer : {:d}'.format(layer)) - - neighbors = inputs[nl + layer] - pools = inputs[2*nl + layer] - upsamples = inputs[3*nl + layer] - - max_n = np.max(neighbors) - nums = np.sum(neighbors < max_n - 0.5, axis=-1) - print('min neighbors =>', np.min(nums)) - - if np.prod(pools.shape) > 0: - max_n = np.max(pools) - nums = np.sum(pools < max_n - 0.5, axis=-1) - print('min pools =>', np.min(nums)) - else: - print('pools empty') - - - if np.prod(upsamples.shape) > 0: - max_n = np.max(upsamples) - nums = np.sum(upsamples < max_n - 0.5, axis=-1) - print('min upsamples =>', np.min(nums)) - else: - print('upsamples empty') - - - print('\nFinished\n\n') - time.sleep(0.5) - - - - - - - diff --git a/utils/visualizer.py b/utils/visualizer.py index f01f1b4..cda24b6 100644 --- a/utils/visualizer.py +++ b/utils/visualizer.py @@ -96,967 +96,6 @@ class ModelVisualizer: # Main visualization methods # ------------------------------------------------------------------------------------------------------------------ - def top_relu_activations(self, model, dataset, relu_idx=0, top_num=5): - """ - Test the model on test dataset to see which points activate the most each neurons in a relu layer - :param model: model used at training - :param dataset: dataset used at training - :param relu_idx: which features are to be visualized - :param top_num: how many top candidates are kept per features - """ - - ##################################### - # First choose the visualized feature - ##################################### - - # List all relu ops - all_ops = [op for op in tf.get_default_graph().get_operations() if op.name.startswith('KernelPointNetwork') - and op.name.endswith('LeakyRelu')] - - # List all possible Relu indices - print('\nPossible Relu indices:') - for i, t in enumerate(all_ops): - print(i, ': ', t.name) - - # Print the chosen one - if relu_idx is not None: - features_tensor = all_ops[relu_idx].outputs[0] - else: - relu_idx = int(input('Choose a Relu index: ')) - features_tensor = all_ops[relu_idx].outputs[0] - - # Get parameters - layer_idx = int(features_tensor.name.split('/')[1][6:]) - if 'strided' in all_ops[relu_idx].name and not ('strided' in all_ops[relu_idx+1].name): - layer_idx += 1 - features_dim = int(features_tensor.shape[1]) - radius = model.config.first_subsampling_dl * model.config.density_parameter * (2 ** layer_idx) - - print('You chose to compute the output of operation named:\n' + all_ops[relu_idx].name) - print('\nIt contains {:d} features.'.format(int(features_tensor.shape[1]))) - - print('\n****************************************************************************') - - ####################### - # Initialize containers - ####################### - - # Initialize containers - self.top_features = -np.ones((top_num, features_dim)) - self.top_classes = -np.ones((top_num, features_dim), dtype=np.int32) - self.saving = model.config.saving - - # Testing parameters - num_votes = 3 - - # Create visu folder - self.visu_path = None - self.fmt_str = None - if model.config.saving: - self.visu_path = join('visu', - 'visu_' + model.saving_path.split('/')[-1], - 'top_activations', - 'Relu{:02d}'.format(relu_idx)) - self.fmt_str = 'f{:04d}_top{:02d}.ply' - if not exists(self.visu_path): - makedirs(self.visu_path) - - # ******************* - # Network predictions - # ******************* - - mean_dt = np.zeros(2) - last_display = time.time() - for v in range(num_votes): - - # Run model on all test examples - # ****************************** - - # Initialise iterator with test data - if model.config.dataset.startswith('S3DIS'): - self.sess.run(dataset.val_init_op) - else: - self.sess.run(dataset.test_init_op) - count = 0 - - while True: - try: - - if model.config.dataset.startswith('ShapeNetPart'): - if model.config.dataset.split('_')[1] == 'multi': - label_op = model.inputs['super_labels'] - else: - label_op = model.inputs['point_labels'] - elif model.config.dataset.startswith('S3DIS'): - label_op = model.inputs['point_labels'] - elif model.config.dataset.startswith('Scannet'): - label_op = model.inputs['point_labels'] - elif model.config.dataset.startswith('ModelNet40'): - label_op = model.inputs['labels'] - else: - raise ValueError('Unsupported dataset') - - # Run one step of the model - t = [time.time()] - ops = (all_ops[-1].outputs[0], - features_tensor, - label_op, - model.inputs['points'], - model.inputs['pools'], - model.inputs['in_batches']) - _, stacked_features, labels, all_points, all_pools, in_batches = self.sess.run(ops, {model.dropout_prob: 1.0}) - t += [time.time()] - count += in_batches.shape[0] - - # Stack all batches - max_ind = np.max(in_batches) - stacked_batches = [] - for b_i, b in enumerate(in_batches): - stacked_batches += [b[b < max_ind - 0.5]*0+b_i] - stacked_batches = np.hstack(stacked_batches) - - # Find batches at wanted layer - for l in range(model.config.num_layers - 1): - if l >= layer_idx: - break - stacked_batches = stacked_batches[all_pools[l][:, 0]] - - # Get each example and update top_activations - for b_i, b in enumerate(in_batches): - b = b[b < max_ind - 0.5] - in_points = all_points[0][b] - features = stacked_features[stacked_batches == b_i] - points = all_points[layer_idx][stacked_batches == b_i] - if model.config.dataset in ['ShapeNetPart_multi', 'ModelNet40_classif']: - l = labels[b_i] - else: - l = np.argmax(np.bincount(labels[b])) - - self.update_top_activations(features, labels[b_i], points, in_points, radius) - - # Average timing - t += [time.time()] - mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1])) - - # Display - if (t[-1] - last_display) > 1.0: - last_display = t[-1] - if model.config.dataset.startswith('S3DIS'): - completed = count / (model.config.validation_size * model.config.batch_num) - else: - completed = count / dataset.num_test - message = 'Vote {:d} : {:.1f}% (timings : {:4.2f} {:4.2f})' - print(message.format(v, - 100 * completed, - 1000 * (mean_dt[0]), - 1000 * (mean_dt[1]))) - #class_names = np.array([dataset.label_to_names[i] for i in range(dataset.num_classes)]) - #print(class_names[self.top_classes[:, :20]].T) - - except tf.errors.OutOfRangeError: - break - - return relu_idx - - def update_top_activations(self, features, label, l_points, input_points, radius, max_computed=60): - - top_num = self.top_features.shape[0] - - # Compute top indice for each feature - max_indices = np.argmax(features, axis=0) - - # get top_point neighborhoods - for features_i, idx in enumerate(max_indices[:max_computed]): - if features[idx, features_i] <= self.top_features[-1, features_i]: - continue - if label in self.top_classes[:, features_i]: - ind0 = np.where(self.top_classes[:, features_i] == label)[0][0] - if features[idx, features_i] <= self.top_features[ind0, features_i]: - continue - elif ind0 < top_num - 1: - self.top_features[ind0:-1, features_i] = self.top_features[ind0+1:, features_i] - self.top_classes[ind0:-1, features_i] = self.top_classes[ind0+1:, features_i] - for next_i in range(ind0 + 1, top_num): - old_f = join(self.visu_path, self.fmt_str.format(features_i, next_i + 1)) - new_f = join(self.visu_path, self.fmt_str.format(features_i, next_i)) - if exists(old_f): - if exists(new_f): - remove(new_f) - rename(old_f, new_f) - - # Find indice where new top should be placed - top_i = np.where(features[idx, features_i] > self.top_features[:, features_i])[0][0] - - # Update top features - if top_i < top_num - 1: - self.top_features[top_i + 1:, features_i] = self.top_features[top_i:-1, features_i] - self.top_features[top_i, features_i] = features[idx, features_i] - self.top_classes[top_i + 1:, features_i] = self.top_classes[top_i:-1, features_i] - self.top_classes[top_i, features_i] = label - - # Find in which batch lays the point - if self.saving: - - # Get inputs - l_features = features[:, features_i] - point = l_points[idx, :] - dist = np.linalg.norm(input_points - point, axis=1) - influence = (radius - dist) / radius - - # Project response on input cloud - if l_points.shape[0] == input_points.shape[0]: - responses = l_features - else: - tree = KDTree(l_points, leaf_size=50) - nn_k = min(l_points.shape[0], 10) - interp_dists, interp_inds = tree.query(input_points, nn_k, return_distance=True) - tukeys = np.square(1 - np.square(interp_dists / radius)) - tukeys[interp_dists > radius] = 0 - responses = np.sum(l_features[interp_inds] * tukeys, axis=1) - - # Handle last examples - for next_i in range(top_num - 1, top_i, -1): - old_f = join(self.visu_path, self.fmt_str.format(features_i, next_i)) - new_f = join(self.visu_path, self.fmt_str.format(features_i, next_i + 1)) - if exists(old_f): - if exists(new_f): - remove(new_f) - rename(old_f, new_f) - - # Save - filename = join(self.visu_path, self.fmt_str.format(features_i, top_i + 1)) - write_ply(filename, - [input_points, influence, responses], - ['x', 'y', 'z', 'influence', 'responses']) - - def show_deformable_kernels_old(self, model, dataset, deform_idx=0): - - ########################################## - # First choose the visualized deformations - ########################################## - - # List all deformation ops - all_ops = [op for op in tf.get_default_graph().get_operations() if op.name.startswith('KernelPointNetwork') - and op.name.endswith('deformed_KP')] - - print('\nPossible deformed indices:') - for i, t in enumerate(all_ops): - print(i, ': ', t.name) - - # Chosen deformations - deformed_KP_tensor = all_ops[deform_idx].outputs[0] - - # Layer index - layer_idx = int(all_ops[deform_idx].name.split('/')[1].split('_')[-1]) - - # Original kernel point positions - KP_vars = [v for v in tf.global_variables() if 'kernel_points' in v.name] - tmp = np.array(all_ops[deform_idx].name.split('/')) - test = [] - for v in KP_vars: - cmp = np.array(v.name.split('/')) - l = min(len(cmp), len(tmp)) - cmp = cmp[:l] - tmp = tmp[:l] - test += [np.sum(cmp == tmp)] - chosen_KP = np.argmax(test) - - print('You chose to visualize the output of operation named: ' + all_ops[deform_idx].name) - - print('\n****************************************************************************') - - # Run model on all test examples - # ****************************** - - # Initialise iterator with test data - if model.config.dataset.startswith('S3DIS'): - self.sess.run(dataset.val_init_op) - else: - self.sess.run(dataset.test_init_op) - count = 0 - - while True: - try: - - # Run one step of the model - t = [time.time()] - ops = (deformed_KP_tensor, - model.inputs['points'], - model.inputs['features'], - model.inputs['pools'], - model.inputs['in_batches'], - KP_vars) - stacked_deformed_KP, \ - all_points, \ - all_colors, \ - all_pools, \ - in_batches, \ - original_KPs = self.sess.run(ops, {model.dropout_prob: 1.0}) - t += [time.time()] - count += in_batches.shape[0] - - # Stack all batches - max_ind = np.max(in_batches) - stacked_batches = [] - for b_i, b in enumerate(in_batches): - stacked_batches += [b[b < max_ind - 0.5] * 0 + b_i] - stacked_batches = np.hstack(stacked_batches) - - # Find batches at wanted layer - for l in range(model.config.num_layers - 1): - if l >= layer_idx: - break - stacked_batches = stacked_batches[all_pools[l][:, 0]] - - # Get each example and update top_activations - in_points = [] - in_colors = [] - deformed_KP = [] - points = [] - lookuptrees = [] - for b_i, b in enumerate(in_batches): - b = b[b < max_ind - 0.5] - in_points += [all_points[0][b]] - deformed_KP += [stacked_deformed_KP[stacked_batches == b_i]] - points += [all_points[layer_idx][stacked_batches == b_i]] - lookuptrees += [KDTree(points[-1])] - if all_colors.shape[1] == 4: - in_colors += [all_colors[b, 1:]] - else: - in_colors += [None] - - print('New batch size : ', len(in_batches)) - - ########################### - # Interactive visualization - ########################### - - # Create figure for features - fig1 = mlab.figure('Features', bgcolor=(1.0, 1.0, 1.0), size=(1280, 920)) - fig1.scene.parallel_projection = False - - # Indices - global obj_i, point_i, plots, offsets, p_scale, show_in_p, aim_point - p_scale = 0.03 - obj_i = 0 - point_i = 0 - plots = {} - offsets = False - show_in_p = 2 - aim_point = np.zeros((1, 3)) - - def picker_callback(picker): - """ Picker callback: this get called when on pick events. - """ - global plots, aim_point - - if 'in_points' in plots: - if plots['in_points'].actor.actor._vtk_obj in [o._vtk_obj for o in picker.actors]: - point_rez = plots['in_points'].glyph.glyph_source.glyph_source.output.points.to_array().shape[0] - new_point_i = int(np.floor(picker.point_id / point_rez)) - if new_point_i < len(plots['in_points'].mlab_source.points): - # Get closest point in the layer we are interested in - aim_point = plots['in_points'].mlab_source.points[new_point_i:new_point_i + 1] - update_scene() - - if 'points' in plots: - if plots['points'].actor.actor._vtk_obj in [o._vtk_obj for o in picker.actors]: - point_rez = plots['points'].glyph.glyph_source.glyph_source.output.points.to_array().shape[0] - new_point_i = int(np.floor(picker.point_id / point_rez)) - if new_point_i < len(plots['points'].mlab_source.points): - # Get closest point in the layer we are interested in - aim_point = plots['points'].mlab_source.points[new_point_i:new_point_i + 1] - update_scene() - - def update_scene(): - global plots, offsets, p_scale, show_in_p, aim_point, point_i - - # Get the current view - v = mlab.view() - roll = mlab.roll() - - # clear figure - for key in plots.keys(): - plots[key].remove() - - plots = {} - - # Plot new data feature - p = points[obj_i] - - # Rescale points for visu - p = (p * 1.5 / model.config.in_radius) - - - # Show point cloud - if show_in_p <= 1: - plots['points'] = mlab.points3d(p[:, 0], - p[:, 1], - p[:, 2], - resolution=8, - scale_factor=p_scale, - scale_mode='none', - color=(0, 1, 1), - figure=fig1) - - if show_in_p >= 1: - - # Get points and colors - in_p = in_points[obj_i] - in_p = (in_p * 1.5 / model.config.in_radius) - - # Color point cloud if possible - in_c = in_colors[obj_i] - if in_c is not None: - - # Primitives - scalars = np.arange(len(in_p)) # Key point: set an integer for each point - - # Define color table (including alpha), which must be uint8 and [0,255] - colors = np.hstack((in_c, np.ones_like(in_c[:, :1]))) - colors = (colors * 255).astype(np.uint8) - - plots['in_points'] = mlab.points3d(in_p[:, 0], - in_p[:, 1], - in_p[:, 2], - scalars, - resolution=8, - scale_factor=p_scale*0.8, - scale_mode='none', - figure=fig1) - plots['in_points'].module_manager.scalar_lut_manager.lut.table = colors - - else: - - plots['in_points'] = mlab.points3d(in_p[:, 0], - in_p[:, 1], - in_p[:, 2], - resolution=8, - scale_factor=p_scale*0.8, - scale_mode='none', - figure=fig1) - - - # Get KP locations - rescaled_aim_point = aim_point * model.config.in_radius / 1.5 - point_i = lookuptrees[obj_i].query(rescaled_aim_point, return_distance=False)[0][0] - if offsets: - KP = points[obj_i][point_i] + deformed_KP[obj_i][point_i] - scals = np.ones_like(KP[:, 0]) - else: - KP = points[obj_i][point_i] + original_KPs[chosen_KP] - scals = np.zeros_like(KP[:, 0]) - - KP = (KP * 1.5 / model.config.in_radius) - - plots['KP'] = mlab.points3d(KP[:, 0], - KP[:, 1], - KP[:, 2], - scals, - colormap='autumn', - resolution=8, - scale_factor=1.2*p_scale, - scale_mode='none', - vmin=0, - vmax=1, - figure=fig1) - - - if True: - plots['center'] = mlab.points3d(p[point_i, 0], - p[point_i, 1], - p[point_i, 2], - scale_factor=1.1*p_scale, - scale_mode='none', - color=(0, 1, 0), - figure=fig1) - - # New title - plots['title'] = mlab.title(str(obj_i), color=(0, 0, 0), size=0.3, height=0.01) - text = '<--- (press g for previous)' + 50 * ' ' + '(press h for next) --->' - plots['text'] = mlab.text(0.01, 0.01, text, color=(0, 0, 0), width=0.98) - plots['orient'] = mlab.orientation_axes() - - # Set the saved view - mlab.view(*v) - mlab.roll(roll) - - return - - def animate_kernel(): - global plots, offsets, p_scale, show_in_p - - # Get KP locations - - KP_def = points[obj_i][point_i] + deformed_KP[obj_i][point_i] - KP_def = (KP_def * 1.5 / model.config.in_radius) - KP_def_color = (1, 0, 0) - - KP_rigid = points[obj_i][point_i] + original_KPs[chosen_KP] - KP_rigid = (KP_rigid * 1.5 / model.config.in_radius) - KP_rigid_color = (1, 0.7, 0) - - if offsets: - t_list = np.linspace(0, 1, 150, dtype=np.float32) - else: - t_list = np.linspace(1, 0, 150, dtype=np.float32) - - @mlab.animate(delay=10) - def anim(): - for t in t_list: - plots['KP'].mlab_source.set(x=t * KP_def[:, 0] + (1 - t) * KP_rigid[:, 0], - y=t * KP_def[:, 1] + (1 - t) * KP_rigid[:, 1], - z=t * KP_def[:, 2] + (1 - t) * KP_rigid[:, 2], - scalars=t * np.ones_like(KP_def[:, 0])) - - yield - - anim() - - return - - def keyboard_callback(vtk_obj, event): - global obj_i, point_i, offsets, p_scale, show_in_p - - if vtk_obj.GetKeyCode() in ['b', 'B']: - p_scale /= 1.5 - update_scene() - - elif vtk_obj.GetKeyCode() in ['n', 'N']: - p_scale *= 1.5 - update_scene() - - if vtk_obj.GetKeyCode() in ['g', 'G']: - obj_i = (obj_i - 1) % len(deformed_KP) - point_i = 0 - update_scene() - - elif vtk_obj.GetKeyCode() in ['h', 'H']: - obj_i = (obj_i + 1) % len(deformed_KP) - point_i = 0 - update_scene() - - elif vtk_obj.GetKeyCode() in ['k', 'K']: - offsets = not offsets - animate_kernel() - - elif vtk_obj.GetKeyCode() in ['z', 'Z']: - show_in_p = (show_in_p + 1) % 3 - update_scene() - - elif vtk_obj.GetKeyCode() in ['0']: - - print('Saving') - - # Find a new name - file_i = 0 - file_name = 'KP_{:03d}.ply'.format(file_i) - files = [f for f in listdir('KP_clouds') if f.endswith('.ply')] - while file_name in files: - file_i += 1 - file_name = 'KP_{:03d}.ply'.format(file_i) - - KP_deform = points[obj_i][point_i] + deformed_KP[obj_i][point_i] - KP_normal = points[obj_i][point_i] + original_KPs[chosen_KP] - - # Save - write_ply(join('KP_clouds', file_name), - [in_points[obj_i], in_colors[obj_i]], - ['x', 'y', 'z', 'red', 'green', 'blue']) - write_ply(join('KP_clouds', 'KP_{:03d}_deform.ply'.format(file_i)), - [KP_deform], - ['x', 'y', 'z']) - write_ply(join('KP_clouds', 'KP_{:03d}_normal.ply'.format(file_i)), - [KP_normal], - ['x', 'y', 'z']) - print('OK') - - return - - # Draw a first plot - pick_func = fig1.on_mouse_pick(picker_callback) - pick_func.tolerance = 0.01 - update_scene() - fig1.scene.interactor.add_observer('KeyPressEvent', keyboard_callback) - mlab.show() - - - - - except tf.errors.OutOfRangeError: - break - - def show_effective_recep_field(self, net, loader, config, f_idx=0): - - ########################################## - # First choose the visualized deformations - ########################################## - - blocks = {} - - named_blocks = [(m_name, m) for m_name, m in net.named_modules() - if len(m_name.split('.')) == 2 and m_name.split('.')[0].endswith('_blocks')] - chosen_block = named_blocks[-1][0] - - for mi, (m_name, m) in enumerate(named_blocks): - - - c1 = bcolors.OKBLUE - c2 = bcolors.BOLD - ce = bcolors.ENDC - print('{:}{:}{:s}{:}{:} {:s}'.format(c1, c2, m_name, ce, ce, m.__repr__())) - blocks[m_name] = m - - if mi == f_idx: - chosen_block = m_name - - print('\nChoose which block output you want to visualize by entering the block name in blue') - override_block = input('Block name: ') - - if len(override_block) > 0: - chosen_block = override_block - print('{:}{:}{:s}{:}{:} {:s}'.format(c1, c2, chosen_block, ce, ce, blocks[chosen_block].__repr__())) - features_dim = blocks[chosen_block].out_dim - - # Fix all the trainable variables in the network (is it needed in eval mode?) - print('\n*************************************\n') - for p_name, param in net.named_parameters(): - if param.requires_grad: - param.requires_grad = False - print('\n*************************************\n') - - # Create modulation variable that requires grad - input_modulations = torch.nn.Parameter(torch.zeros((200000, 1), - dtype=torch.float32), - requires_grad=True) - - print('\n*************************************\n') - for p_name, param in net.named_parameters(): - if param.requires_grad: - print(p_name, param.shape) - print('\n*************************************\n') - - # Create ERF loss - - # Create ERF optimizer - - - - - - global plots, p_scale, show_in_p, remove_h, aim_point - aim_point = np.zeros((1, 3), dtype=np.float32) - remove_h = 1.05 - p_scale = 0.1 - plots = {} - show_in_p = False - - global points, in_points, grad_values, chosen_point, in_colors - points = None - in_points = np.zeros((0, 3)) - grad_values = None - chosen_point = None - in_colors = None - - ########################### - # Interactive visualization - ########################### - - # Create figure for features - fig1 = mlab.figure('Features', bgcolor=(0.5, 0.5, 0.5), size=(640, 480)) - fig1.scene.parallel_projection = False - - # Indices - - def update_ERF(only_points=False): - global points, in_points, grad_values, chosen_point, aim_point, in_colors - - # Generate clouds until we effectively changed - batch = None - if only_points: - # get a new batch (index does not matter given our input pipeline) - for batch in loader: - if batch.points[0].shape[0] != in_points.shape[0]: - break - - sum_grads = 0 - if only_points: - num_tries = 1 - else: - num_tries = 10 - - ################################################# - # Apply ERF optim to the same batch several times - ################################################# - - if 'cuda' in self.device.type: - batch.to(self.device) - - - - for test_i in range(num_tries): - - print('Updating ERF {:.0f}%'.format((test_i + 1) * 100 / num_tries)) - rand_f_i = np.random.randint(features_dim) - - # Reset input modulation variable - torch.nn.init.zeros_(input_modulations) - - reset_op = input_modulations_var.assign(tf.zeros_like(input_modulations_var)) - self.sess.run(reset_op) - - # zero the parameter gradients - ERF_optimizer.zero_grad() - - # Forward pass - outputs = net(batch, config) - - loss = net.ERF_loss(outputs) - - # Backward - loss.backward() - - # Get result from hook here? - - ERF_optimizer.step() - torch.cuda.synchronize(self.device) - - - - - - - # Forward pass - outputs = net(batch, config) - original_KP = deform_convs[deform_idx].kernel_points.cpu().detach().numpy() - stacked_deformed_KP = deform_convs[deform_idx].deformed_KP.cpu().detach().numpy() - count += batch.lengths[0].shape[0] - - if 'cuda' in self.device.type: - torch.cuda.synchronize(self.device) - - - - - - - - - - - # Reset input modulation variable - reset_op = input_modulations_var.assign(tf.zeros_like(input_modulations_var)) - self.sess.run(reset_op) - - # Apply gradient to input modulations - t = [time.time()] - ops = (ERF_train_op, - chosen_i_tf, - input_modulations_var, - model.inputs['points'], - model.inputs['features'], - model.inputs['pools'], - model.inputs['in_batches']) - feed_dict = {aimed_coordinates: aim_point, - chosen_f_tf: rand_f_i, - model.dropout_prob: 1.0} - _, chosen_i, new_mods, all_points, all_colors, all_pools, in_batches = self.sess.run(ops, feed_dict) - t += [time.time()] - - # Get the new value of the modulations - sum_grads += np.abs(self.sess.run(input_modulations_var)) - - grad = sum_grads / num_tries - - # Stack all batches - max_ind = np.max(in_batches) - stacked_batches = [] - for b_i, b in enumerate(in_batches): - stacked_batches += [b[b < max_ind - 0.5] * 0 + b_i] - stacked_batches = np.hstack(stacked_batches) - - # Find batches at wanted layer - for l in range(model.config.num_layers - 1): - if l >= layer_idx: - break - stacked_batches = stacked_batches[all_pools[l][:, 0]] - - # Get each example and update top_activations - for b_i, b in enumerate(in_batches): - b = b[b < max_ind - 0.5] - in_points = all_points[0][b] - in_colors = all_colors[b, 1:] - points = all_points[layer_idx][stacked_batches == b_i] - grad_values = grad[b] - - chosen_point = all_points[layer_idx][chosen_i] - - def update_scene(): - global plots, p_scale, show_in_p, remove_h - global points, in_points, grad_values, chosen_point - - # Get the current view - v = mlab.view() - roll = mlab.roll() - - # clear figure - for key in plots.keys(): - plots[key].remove() - - plots = {} - - # Plot new data feature - in_p = in_points - p = points - p0 = chosen_point - responses = 100 * np.abs(np.ravel(grad_values)) - #xresponses = responses ** (1/2) - - # Remove roof - if 0.0 < remove_h < 1.0: - floor_h = np.min(in_p[:, 2]) - ceil_h = np.max(in_p[:, 2]) - threshold = floor_h + (ceil_h - floor_h) * remove_h - responses = responses[in_p[:, 2] < threshold] - in_p = in_p[in_p[:, 2] < threshold] - p = p[p[:, 2] < threshold] - - # Rescale responses - min_response, max_response = np.min(responses), np.max(responses) - - # Show point cloud - if show_in_p: - plots['points'] = mlab.points3d(p[:, 0], - p[:, 1], - p[:, 2], - resolution=8, - scale_factor=p_scale, - scale_mode='none', - color=(0, 1, 1), - figure=fig1) - - plots['in_points'] = mlab.points3d(in_p[:, 0], - in_p[:, 1], - in_p[:, 2], - responses, - resolution=8, - scale_factor=p_scale * 0.8, - scale_mode='none', - vmin=0.1, - vmax=1.5, - figure=fig1) - - plots['center'] = mlab.points3d(p0[0], - p0[1], - p0[2], - scale_factor=1.5 * p_scale, - scale_mode='none', - color=(0, 0, 0), - figure=fig1) - - # New title - plots['title'] = mlab.title(str(int(100*remove_h)) + '%', color=(0, 0, 0), size=0.3, height=0.01) - text = '<--- (press g to remove ceiling)' + 50 * ' ' + '(press h to add ceiling) --->' - plots['text'] = mlab.text(0.01, 0.01, text, color=(0, 0, 0), width=0.98) - plots['orient'] = mlab.orientation_axes() - - # Set the saved view - mlab.view(*v) - mlab.roll(roll) - - return - - def picker_callback(picker): - """ Picker callback: this get called when on pick events. - """ - global plots, aim_point, in_points - - if plots['in_points'].actor.actor._vtk_obj in [o._vtk_obj for o in picker.actors]: - point_rez = plots['in_points'].glyph.glyph_source.glyph_source.output.points.to_array().shape[0] - new_point_i = int(np.floor(picker.point_id / point_rez)) - if new_point_i < len(plots['in_points'].mlab_source.points): - - # Get closest point in the layer we are interested in - aim_point = plots['in_points'].mlab_source.points[new_point_i:new_point_i + 1] - update_ERF() - update_scene() - - def keyboard_callback(vtk_obj, event): - global remove_h, p_scale, show_in_p - global in_points, grad_values, chosen_point, in_colors - - print(vtk_obj.GetKeyCode()) - - - if vtk_obj.GetKeyCode() in ['b', 'B']: - p_scale /= 1.5 - update_scene() - - elif vtk_obj.GetKeyCode() in ['n', 'N']: - p_scale *= 1.5 - update_scene() - - if vtk_obj.GetKeyCode() in ['g', 'G']: - if remove_h > 0.0: - remove_h -= 0.1 - update_scene() - - elif vtk_obj.GetKeyCode() in ['h', 'H']: - if remove_h < 1.0: - remove_h += 0.1 - update_ERF() - update_scene() - - elif vtk_obj.GetKeyCode() in ['z', 'Z']: - show_in_p = not show_in_p - update_scene() - - elif vtk_obj.GetKeyCode() in ['x', 'X']: - # Reset potentials - dataset.potentials['ERF'] = [] - dataset.min_potentials['ERF'] = [] - for i, tree in enumerate(dataset.input_trees['test']): - dataset.potentials['ERF'] += [np.random.rand(tree.data.shape[0]) * 1e-3] - dataset.min_potentials['ERF'] += [float(np.min(dataset.potentials['ERF'][-1]))] - - # Update figure - update_ERF(only_points=True) - update_scene() - - elif vtk_obj.GetKeyCode() in ['0']: - - print('Saving') - - # Find a new name - file_i = 0 - file_name = 'ERF_{:03d}.ply'.format(file_i) - files = [f for f in listdir('ERF_clouds') if f.endswith('.ply')] - while file_name in files: - file_i += 1 - file_name = 'ERF_{:03d}.ply'.format(file_i) - - # Save - responses = 100 * np.abs(np.ravel(grad_values)) - write_ply(join('ERF_clouds', file_name), - [in_points, in_colors, responses], - ['x', 'y', 'z', 'red', 'green', 'blue', 'erf']) - write_ply(join('ERF_clouds', 'ERF_{:03d}_center.ply'.format(file_i)), - [chosen_point.reshape([1, -1])], - ['x', 'y', 'z']) - print('OK') - - return - - # Draw a first plot - pick_func = fig1.on_mouse_pick(picker_callback) - pick_func.tolerance = 0.01 - update_ERF(only_points=True) - update_scene() - fig1.scene.interactor.add_observer('KeyPressEvent', keyboard_callback) - mlab.show() - - return - def show_deformable_kernels(self, net, loader, config, deform_idx=0): """ Show some inference with deformable kernels @@ -1401,336 +440,9 @@ class ModelVisualizer: return - @staticmethod - def show_activation(path, relu_idx=0, save_video=False): - """ - This function show the saved input point clouds maximizing the activations. You can also directly load the files - in a visualization software like CloudCompare. - In the case of relu_idx = 0 and if gaussian mode, the associated filter is also shown. This function can only - show the filters for the last saved epoch. - """ - - ################ - # Find the files - ################ - - # Check visu folder - visu_path = join('visu', - 'visu_' + path.split('/')[-1], - 'top_activations', - 'Relu{:02d}'.format(relu_idx)) - if not exists(visu_path): - message = 'Relu {:d} activations of the model {:s} not found.' - raise ValueError(message.format(relu_idx, path.split('/')[-1])) - - # Get the list of files - feature_files = np.sort([f for f in listdir(visu_path) if f.endswith('.ply')]) - if len(feature_files) == 0: - message = 'Relu {:d} activations of the model {:s} not found.' - raise ValueError(message.format(relu_idx, path.split('/')[-1])) - - # Load mode - config = Config() - config.load(path) - mode = config.convolution_mode - - ################# - # Get activations - ################# - - all_points = [] - all_responses = [] - - for file in feature_files: - - # Load points - data = read_ply(join(visu_path, file)) - all_points += [np.vstack((data['x'], data['y'], data['z'])).T] - all_responses += [data['responses']] - - ########################### - # Interactive visualization - ########################### - - # Create figure for features - fig1 = mlab.figure('Features', bgcolor=(0.5, 0.5, 0.5), size=(640, 480)) - fig1.scene.parallel_projection = False - - # Indices - global file_i - file_i = 0 - - def update_scene(): - - # clear figure - mlab.clf(fig1) - - # Plot new data feature - points = all_points[file_i] - responses = all_responses[file_i] - min_response, max_response = np.min(responses), np.max(responses) - responses = (responses - min_response) / (max_response - min_response) - - # Rescale points for visu - points = (points * 1.5 / config.in_radius + np.array([1.0, 1.0, 1.0])) * 50.0 - - # Show point clouds colorized with activations - activations = mlab.points3d(points[:, 0], - points[:, 1], - points[:, 2], - responses, - scale_factor=3.0, - scale_mode='none', - vmin=0.1, - vmax=0.9, - figure=fig1) - - # New title - mlab.title(feature_files[file_i], color=(0, 0, 0), size=0.3, height=0.01) - text = '<--- (press g for previous)' + 50*' ' + '(press h for next) --->' - mlab.text(0.01, 0.01, text, color=(0, 0, 0), width=0.98) - mlab.orientation_axes() - - return - - def keyboard_callback(vtk_obj, event): - global file_i - - if vtk_obj.GetKeyCode() in ['g', 'G']: - - file_i = (file_i - 1) % len(all_responses) - update_scene() - - elif vtk_obj.GetKeyCode() in ['h', 'H']: - - file_i = (file_i + 1) % len(all_responses) - update_scene() - - return - - # Draw a first plot - update_scene() - fig1.scene.interactor.add_observer('KeyPressEvent', keyboard_callback) - mlab.show() - - return - # Utilities # ------------------------------------------------------------------------------------------------------------------ - @staticmethod - def load_last_kernels(path): - - # Directories of validation error - kernel_dirs = np.array([f for f in listdir(join(path, 'kernel_points')) if f.startswith('epoch')]) - - # Find last epoch folder - epochs = np.array([int(f[5:]) for f in kernel_dirs]) - last_dir = kernel_dirs[np.argmax(epochs)] - - # Find saved files for the first layer - kernel_file = join(path, 'kernel_points', last_dir, 'layer_0_simple_0.ply') - weights_file = join(path, 'kernel_points', last_dir, 'layer_0_simple_0.npy') - - # Read kernel file - data = read_ply(kernel_file) - points = np.vstack((data['x'], data['y'], data['z'])).T - extents = data['sigma'].astype(np.float32) - - # Read weight file - w = np.load(weights_file) - - return points, extents, w - - @staticmethod - def apply_weights(points, kernel, weights, extents): - - # Get all difference matrices [n_points, n_kpoints, dim] - points = np.expand_dims(points, 1) - points = np.tile(points, [1, kernel.shape[0], 1]) - differences = points - kernel - - # Compute distance matrices [n_points, n_kpoints] - sq_distances = np.sum(np.square(differences), axis=-1) - - # Compute gaussians [n_points, n_kpoints] - gaussian_values = np.exp(-sq_distances / (2 * np.square(extents))) - - # Apply weights - return np.matmul(gaussian_values, np.squeeze(weights)) - - - def top_relu_activations_old(self, model, dataset, relu_idx=0, top_num=5): - """ - Test the model on test dataset to see which points activate the most each neurons in a relu layer - :param model: model used at training - :param dataset: dataset used at training - :param relu_idx: which features are to be visualized - :param top_num: how many top candidates are kept per features - """ - - ##################################### - # First choose the visualized feature - ##################################### - - # List all relu ops - all_ops = [op for op in tf.get_default_graph().get_operations() if op.name.startswith('KernelPointNetwork') - and op.name.endswith('LeakyRelu')] - - # Non relu ops in case we want the first KPConv features - KPConv_0 = [op for op in tf.get_default_graph().get_operations() if op.name.endswith('layer_0/simple_0/Sum_1')] - - # Print the chosen one - if relu_idx == 0: - features_tensor = KPConv_0[relu_idx].outputs[0] - else: - features_tensor = all_ops[relu_idx].outputs[0] - - # Get parameters - layer_idx = int(features_tensor.name.split('/')[1][6:]) - if 'strided' in all_ops[relu_idx].name and not ('strided' in all_ops[relu_idx+1].name): - layer_idx += 1 - features_dim = int(features_tensor.shape[1]) - radius = model.config.first_subsampling_dl * model.config.density_parameter * (2 ** layer_idx) - - if relu_idx == 0 : - print('SPECIAL CASE : relu_idx = 0 => visualization of the fist KPConv before relu') - print('You chose to visualize the output of operation named: ' + KPConv_0[0].name) - print('It contains {:d} features.'.format(int(features_tensor.shape[1]))) - else : - print('You chose to visualize the output of operation named: ' + all_ops[relu_idx].name) - print('It contains {:d} features.'.format(int(features_tensor.shape[1]))) - - print('\nPossible Relu indices:') - for i, t in enumerate(all_ops): - print(i, ': ', t.name) - - print('\n****************************************************************************') - - ##################### - # Initialize containers - ##################### - - # Initialize containers - self.top_features = -np.ones((top_num, features_dim)) - self.top_classes = -np.ones((top_num, features_dim), dtype=np.int32) - self.saving = model.config.saving - - # Testing parameters - num_votes = 3 - - # Create visu folder - self.visu_path = None - self.fmt_str = None - if model.config.saving: - self.visu_path = join('visu', - 'visu_' + model.saving_path.split('/')[-1], - 'top_activations', - 'Relu{:02d}'.format(relu_idx)) - self.fmt_str = 'f{:04d}_top{:02d}.ply' - if not exists(self.visu_path): - makedirs(self.visu_path) - - # ******************* - # Network predictions - # ******************* - - mean_dt = np.zeros(2) - last_display = time.time() - for v in range(num_votes): - - # Run model on all test examples - # ****************************** - - # Initialise iterator with test data - if model.config.dataset.startswith('S3DIS'): - self.sess.run(dataset.val_init_op) - else: - self.sess.run(dataset.test_init_op) - count = 0 - - while True: - try: - - if model.config.dataset.startswith('ShapeNetPart'): - if model.config.dataset.split('_')[1] == 'multi': - label_op = model.inputs['super_labels'] - else: - label_op = model.inputs['point_labels'] - elif model.config.dataset.startswith('S3DIS'): - label_op = model.inputs['point_labels'] - elif model.config.dataset.startswith('Scannet'): - label_op = model.inputs['point_labels'] - elif model.config.dataset.startswith('ModelNet40'): - label_op = model.inputs['labels'] - else: - raise ValueError('Unsupported dataset') - - # Run one step of the model - t = [time.time()] - ops = (all_ops[-1].outputs[0], - features_tensor, - label_op, - model.inputs['points'], - model.inputs['pools'], - model.inputs['in_batches']) - _, stacked_features, labels, all_points, all_pools, in_batches = self.sess.run(ops, {model.dropout_prob: 1.0}) - t += [time.time()] - count += in_batches.shape[0] - - - # Stack all batches - max_ind = np.max(in_batches) - stacked_batches = [] - for b_i, b in enumerate(in_batches): - stacked_batches += [b[b < max_ind - 0.5]*0+b_i] - stacked_batches = np.hstack(stacked_batches) - - # Find batches at wanted layer - for l in range(model.config.num_layers - 1): - if l >= layer_idx: - break - stacked_batches = stacked_batches[all_pools[l][:, 0]] - - # Get each example and update top_activations - for b_i, b in enumerate(in_batches): - b = b[b < max_ind - 0.5] - in_points = all_points[0][b] - features = stacked_features[stacked_batches == b_i] - points = all_points[layer_idx][stacked_batches == b_i] - if model.config.dataset in ['ShapeNetPart_multi', 'ModelNet40_classif']: - l = labels[b_i] - else: - l = np.argmax(np.bincount(labels[b])) - - self.update_top_activations(features, labels[b_i], points, in_points, radius) - - # Average timing - t += [time.time()] - mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1])) - - # Display - if (t[-1] - last_display) > 1.0: - last_display = t[-1] - if model.config.dataset.startswith('S3DIS'): - completed = count / (model.config.validation_size * model.config.batch_num) - else: - completed = count / dataset.num_test - message = 'Vote {:d} : {:.1f}% (timings : {:4.2f} {:4.2f})' - print(message.format(v, - 100 * completed, - 1000 * (mean_dt[0]), - 1000 * (mean_dt[1]))) - #class_names = np.array([dataset.label_to_names[i] for i in range(dataset.num_classes)]) - #print(class_names[self.top_classes[:, :20]].T) - - except tf.errors.OutOfRangeError: - break - - return - - - def show_ModelNet_models(all_points): diff --git a/visualize_deformations.py b/visualize_deformations.py index c4519b0..a1c73b9 100644 --- a/visualize_deformations.py +++ b/visualize_deformations.py @@ -92,21 +92,14 @@ if __name__ == '__main__': # Here you can choose which model you want to test with the variable test_model. Here are the possible values : # # > 'last_XXX': Automatically retrieve the last trained model on dataset XXX - # > '(old_)results/Log_YYYY-MM-DD_HH-MM-SS': Directly provide the path of a trained model + # > 'results/Log_YYYY-MM-DD_HH-MM-SS': Directly provide the path of a trained model - # chosen_log = 'results/Log_2020-04-04_10-04-42' # => ModelNet40 - # chosen_log = 'results/Log_2020-04-22_11-53-45' # => S3DIS - # chosen_log = 'results/Log_2020-04-22_12-28-37' # => S3DIS corrected - # chosen_log = 'results/Log_2020-04-23_09-48-15' # => S3DIS no repulsive - # chosen_log = 'results/Log_2020-04-23_09-49-49' # => S3DIS repulsive 0.5 - # chosen_log = 'results/Log_2020-04-23_19-41-12' # => S3DIS 10*fitting - chosen_log = 'results/Log_2020-04-23_19-42-18' # => S3DIS no hook + chosen_log = 'results/Log_2020-04-23_19-42-18' + # Choose the index of the checkpoint to load OR None if you want to load the current checkpoint + chkp_idx = None - # You can also choose the index of the snapshot to load (last by default) - chkp_idx = -1 - - # Eventually you can choose which feature is visualized (index of the deform operation in the network) + # Eventually you can choose which feature is visualized (index of the deform convolution in the network) deform_idx = 0 # Deal with 'last_XXX' choices @@ -148,7 +141,6 @@ if __name__ == '__main__': # Change parameters for the test here. For example, you can stop augmenting the input data. config.augment_noise = 0.0001 - #config.augment_symmetries = False config.batch_num = 1 config.in_radius = 2.0 config.input_threads = 0