Safran/KPConv-PyTorch
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity
KPConv-PyTorch/utils/visualizer.py

1832 lines
74 KiB
Python
Raw Normal View History

Initial commit
2020-03-31 19:42:35 +00:00
#
#
# 0=================================0
# | Kernel Point Convolutions |
# 0=================================0
#
#
# ----------------------------------------------------------------------------------------------------------------------
#
# Class handling the visualization
#
# ----------------------------------------------------------------------------------------------------------------------
#
# Hugues THOMAS - 11/06/2018
#
# ----------------------------------------------------------------------------------------------------------------------
#
# Imports and global variables
# \**********************************/
#
# Basic libs
import torch
import numpy as np
from sklearn.neighbors import KDTree
from os import makedirs, remove, rename, listdir
from os.path import exists, join
import time
from mayavi import mlab
import sys
from models.blocks import KPConv
# PLY reader
from utils.ply import write_ply, read_ply
# Configuration class
from utils.config import Config, bcolors
# ----------------------------------------------------------------------------------------------------------------------
#
# Trainer Class
# \*******************/
#
class ModelVisualizer:
# Initialization methods
# ------------------------------------------------------------------------------------------------------------------
def __init__(self, net, config, chkp_path, on_gpu=True):
"""
Initialize training parameters and reload previous model for restore/finetune
:param net: network object
:param config: configuration object
:param chkp_path: path to the checkpoint that needs to be loaded (None for new training)
:param finetune: finetune from checkpoint (True) or restore training from checkpoint (False)
:param on_gpu: Train on GPU or CPU
"""
############
# Parameters
############
# Choose to train on CPU or GPU
if on_gpu and torch.cuda.is_available():
self.device = torch.device("cuda:0")
else:
self.device = torch.device("cpu")
net.to(self.device)
##########################
# Load previous checkpoint
##########################
checkpoint = torch.load(chkp_path)
net.load_state_dict(checkpoint['model_state_dict'])
self.epoch = checkpoint['epoch']
net.eval()
print("Model and training state restored.")
return
# Main visualization methods
# ------------------------------------------------------------------------------------------------------------------
def top_relu_activations(self, model, dataset, relu_idx=0, top_num=5):
"""
Test the model on test dataset to see which points activate the most each neurons in a relu layer
:param model: model used at training
:param dataset: dataset used at training
:param relu_idx: which features are to be visualized
:param top_num: how many top candidates are kept per features
"""
#####################################
# First choose the visualized feature
#####################################
# List all relu ops
all_ops = [op for op in tf.get_default_graph().get_operations() if op.name.startswith('KernelPointNetwork')
and op.name.endswith('LeakyRelu')]
# List all possible Relu indices
print('\nPossible Relu indices:')
for i, t in enumerate(all_ops):
print(i, ': ', t.name)
# Print the chosen one
if relu_idx is not None:
features_tensor = all_ops[relu_idx].outputs[0]
else:
relu_idx = int(input('Choose a Relu index: '))
features_tensor = all_ops[relu_idx].outputs[0]
# Get parameters
layer_idx = int(features_tensor.name.split('/')[1][6:])
if 'strided' in all_ops[relu_idx].name and not ('strided' in all_ops[relu_idx+1].name):
layer_idx += 1
features_dim = int(features_tensor.shape[1])
radius = model.config.first_subsampling_dl * model.config.density_parameter * (2 ** layer_idx)
print('You chose to compute the output of operation named:\n' + all_ops[relu_idx].name)
print('\nIt contains {:d} features.'.format(int(features_tensor.shape[1])))
print('\n****************************************************************************')
#######################
# Initialize containers
#######################
# Initialize containers
self.top_features = -np.ones((top_num, features_dim))
self.top_classes = -np.ones((top_num, features_dim), dtype=np.int32)
self.saving = model.config.saving
# Testing parameters
num_votes = 3
# Create visu folder
self.visu_path = None
self.fmt_str = None
if model.config.saving:
self.visu_path = join('visu',
'visu_' + model.saving_path.split('/')[-1],
'top_activations',
'Relu{:02d}'.format(relu_idx))
self.fmt_str = 'f{:04d}_top{:02d}.ply'
if not exists(self.visu_path):
makedirs(self.visu_path)
# *******************
# Network predictions
# *******************
mean_dt = np.zeros(2)
last_display = time.time()
for v in range(num_votes):
# Run model on all test examples
# ******************************
# Initialise iterator with test data
if model.config.dataset.startswith('S3DIS'):
self.sess.run(dataset.val_init_op)
else:
self.sess.run(dataset.test_init_op)
count = 0
while True:
try:
if model.config.dataset.startswith('ShapeNetPart'):
if model.config.dataset.split('_')[1] == 'multi':
label_op = model.inputs['super_labels']
else:
label_op = model.inputs['point_labels']
elif model.config.dataset.startswith('S3DIS'):
label_op = model.inputs['point_labels']
elif model.config.dataset.startswith('Scannet'):
label_op = model.inputs['point_labels']
elif model.config.dataset.startswith('ModelNet40'):
label_op = model.inputs['labels']
else:
raise ValueError('Unsupported dataset')
# Run one step of the model
t = [time.time()]
ops = (all_ops[-1].outputs[0],
features_tensor,
label_op,
model.inputs['points'],
model.inputs['pools'],
model.inputs['in_batches'])
_, stacked_features, labels, all_points, all_pools, in_batches = self.sess.run(ops, {model.dropout_prob: 1.0})
t += [time.time()]
count += in_batches.shape[0]
# Stack all batches
max_ind = np.max(in_batches)
stacked_batches = []
for b_i, b in enumerate(in_batches):
stacked_batches += [b[b < max_ind - 0.5]*0+b_i]
stacked_batches = np.hstack(stacked_batches)
# Find batches at wanted layer
for l in range(model.config.num_layers - 1):
if l >= layer_idx:
break
stacked_batches = stacked_batches[all_pools[l][:, 0]]
# Get each example and update top_activations
for b_i, b in enumerate(in_batches):
b = b[b < max_ind - 0.5]
in_points = all_points[0][b]
features = stacked_features[stacked_batches == b_i]
points = all_points[layer_idx][stacked_batches == b_i]
if model.config.dataset in ['ShapeNetPart_multi', 'ModelNet40_classif']:
l = labels[b_i]
else:
l = np.argmax(np.bincount(labels[b]))
self.update_top_activations(features, labels[b_i], points, in_points, radius)
# Average timing
t += [time.time()]
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1]))
# Display
if (t[-1] - last_display) > 1.0:
last_display = t[-1]
if model.config.dataset.startswith('S3DIS'):
completed = count / (model.config.validation_size * model.config.batch_num)
else:
completed = count / dataset.num_test
message = 'Vote {:d} : {:.1f}% (timings : {:4.2f} {:4.2f})'
print(message.format(v,
100 * completed,
1000 * (mean_dt[0]),
1000 * (mean_dt[1])))
#class_names = np.array([dataset.label_to_names[i] for i in range(dataset.num_classes)])
#print(class_names[self.top_classes[:, :20]].T)
except tf.errors.OutOfRangeError:
break
return relu_idx
def update_top_activations(self, features, label, l_points, input_points, radius, max_computed=60):
top_num = self.top_features.shape[0]
# Compute top indice for each feature
max_indices = np.argmax(features, axis=0)
# get top_point neighborhoods
for features_i, idx in enumerate(max_indices[:max_computed]):
if features[idx, features_i] <= self.top_features[-1, features_i]:
continue
if label in self.top_classes[:, features_i]:
ind0 = np.where(self.top_classes[:, features_i] == label)[0][0]
if features[idx, features_i] <= self.top_features[ind0, features_i]:
continue
elif ind0 < top_num - 1:
self.top_features[ind0:-1, features_i] = self.top_features[ind0+1:, features_i]
self.top_classes[ind0:-1, features_i] = self.top_classes[ind0+1:, features_i]
for next_i in range(ind0 + 1, top_num):
old_f = join(self.visu_path, self.fmt_str.format(features_i, next_i + 1))
new_f = join(self.visu_path, self.fmt_str.format(features_i, next_i))
if exists(old_f):
if exists(new_f):
remove(new_f)
rename(old_f, new_f)
# Find indice where new top should be placed
top_i = np.where(features[idx, features_i] > self.top_features[:, features_i])[0][0]
# Update top features
if top_i < top_num - 1:
self.top_features[top_i + 1:, features_i] = self.top_features[top_i:-1, features_i]
self.top_features[top_i, features_i] = features[idx, features_i]
self.top_classes[top_i + 1:, features_i] = self.top_classes[top_i:-1, features_i]
self.top_classes[top_i, features_i] = label
# Find in which batch lays the point
if self.saving:
# Get inputs
l_features = features[:, features_i]
point = l_points[idx, :]
dist = np.linalg.norm(input_points - point, axis=1)
influence = (radius - dist) / radius
# Project response on input cloud
if l_points.shape[0] == input_points.shape[0]:
responses = l_features
else:
tree = KDTree(l_points, leaf_size=50)
nn_k = min(l_points.shape[0], 10)
interp_dists, interp_inds = tree.query(input_points, nn_k, return_distance=True)
tukeys = np.square(1 - np.square(interp_dists / radius))
tukeys[interp_dists > radius] = 0
responses = np.sum(l_features[interp_inds] * tukeys, axis=1)
# Handle last examples
for next_i in range(top_num - 1, top_i, -1):
old_f = join(self.visu_path, self.fmt_str.format(features_i, next_i))
new_f = join(self.visu_path, self.fmt_str.format(features_i, next_i + 1))
if exists(old_f):
if exists(new_f):
remove(new_f)
rename(old_f, new_f)
# Save
filename = join(self.visu_path, self.fmt_str.format(features_i, top_i + 1))
write_ply(filename,
[input_points, influence, responses],
['x', 'y', 'z', 'influence', 'responses'])
def show_deformable_kernels_old(self, model, dataset, deform_idx=0):
##########################################
# First choose the visualized deformations
##########################################
# List all deformation ops
all_ops = [op for op in tf.get_default_graph().get_operations() if op.name.startswith('KernelPointNetwork')
and op.name.endswith('deformed_KP')]
print('\nPossible deformed indices:')
for i, t in enumerate(all_ops):
print(i, ': ', t.name)
# Chosen deformations
deformed_KP_tensor = all_ops[deform_idx].outputs[0]
# Layer index
layer_idx = int(all_ops[deform_idx].name.split('/')[1].split('_')[-1])
# Original kernel point positions
KP_vars = [v for v in tf.global_variables() if 'kernel_points' in v.name]
tmp = np.array(all_ops[deform_idx].name.split('/'))
test = []
for v in KP_vars:
cmp = np.array(v.name.split('/'))
l = min(len(cmp), len(tmp))
cmp = cmp[:l]
tmp = tmp[:l]
test += [np.sum(cmp == tmp)]
chosen_KP = np.argmax(test)
print('You chose to visualize the output of operation named: ' + all_ops[deform_idx].name)
print('\n****************************************************************************')
# Run model on all test examples
# ******************************
# Initialise iterator with test data
if model.config.dataset.startswith('S3DIS'):
self.sess.run(dataset.val_init_op)
else:
self.sess.run(dataset.test_init_op)
count = 0
while True:
try:
# Run one step of the model
t = [time.time()]
ops = (deformed_KP_tensor,
model.inputs['points'],
model.inputs['features'],
model.inputs['pools'],
model.inputs['in_batches'],
KP_vars)
stacked_deformed_KP, \
all_points, \
all_colors, \
all_pools, \
in_batches, \
original_KPs = self.sess.run(ops, {model.dropout_prob: 1.0})
t += [time.time()]
count += in_batches.shape[0]
# Stack all batches
max_ind = np.max(in_batches)
stacked_batches = []
for b_i, b in enumerate(in_batches):
stacked_batches += [b[b < max_ind - 0.5] * 0 + b_i]
stacked_batches = np.hstack(stacked_batches)
# Find batches at wanted layer
for l in range(model.config.num_layers - 1):
if l >= layer_idx:
break
stacked_batches = stacked_batches[all_pools[l][:, 0]]
# Get each example and update top_activations
in_points = []
in_colors = []
deformed_KP = []
points = []
lookuptrees = []
for b_i, b in enumerate(in_batches):
b = b[b < max_ind - 0.5]
in_points += [all_points[0][b]]
deformed_KP += [stacked_deformed_KP[stacked_batches == b_i]]
points += [all_points[layer_idx][stacked_batches == b_i]]
lookuptrees += [KDTree(points[-1])]
if all_colors.shape[1] == 4:
in_colors += [all_colors[b, 1:]]
else:
in_colors += [None]
print('New batch size : ', len(in_batches))
###########################
# Interactive visualization
###########################
# Create figure for features
fig1 = mlab.figure('Features', bgcolor=(1.0, 1.0, 1.0), size=(1280, 920))
fig1.scene.parallel_projection = False
# Indices
global obj_i, point_i, plots, offsets, p_scale, show_in_p, aim_point
p_scale = 0.03
obj_i = 0
point_i = 0
plots = {}
offsets = False
show_in_p = 2
aim_point = np.zeros((1, 3))
def picker_callback(picker):
""" Picker callback: this get called when on pick events.
"""
global plots, aim_point
if 'in_points' in plots:
if plots['in_points'].actor.actor._vtk_obj in [o._vtk_obj for o in picker.actors]:
point_rez = plots['in_points'].glyph.glyph_source.glyph_source.output.points.to_array().shape[0]
new_point_i = int(np.floor(picker.point_id / point_rez))
if new_point_i < len(plots['in_points'].mlab_source.points):
# Get closest point in the layer we are interested in
aim_point = plots['in_points'].mlab_source.points[new_point_i:new_point_i + 1]
update_scene()
if 'points' in plots:
if plots['points'].actor.actor._vtk_obj in [o._vtk_obj for o in picker.actors]:
point_rez = plots['points'].glyph.glyph_source.glyph_source.output.points.to_array().shape[0]
new_point_i = int(np.floor(picker.point_id / point_rez))
if new_point_i < len(plots['points'].mlab_source.points):
# Get closest point in the layer we are interested in
aim_point = plots['points'].mlab_source.points[new_point_i:new_point_i + 1]
update_scene()
def update_scene():
global plots, offsets, p_scale, show_in_p, aim_point, point_i
# Get the current view
v = mlab.view()
roll = mlab.roll()
# clear figure
for key in plots.keys():
plots[key].remove()
plots = {}
# Plot new data feature
p = points[obj_i]
# Rescale points for visu
p = (p * 1.5 / model.config.in_radius)
# Show point cloud
if show_in_p <= 1:
plots['points'] = mlab.points3d(p[:, 0],
p[:, 1],
p[:, 2],
resolution=8,
scale_factor=p_scale,
scale_mode='none',
color=(0, 1, 1),
figure=fig1)
if show_in_p >= 1:
# Get points and colors
in_p = in_points[obj_i]
in_p = (in_p * 1.5 / model.config.in_radius)
# Color point cloud if possible
in_c = in_colors[obj_i]
if in_c is not None:
# Primitives
scalars = np.arange(len(in_p)) # Key point: set an integer for each point
# Define color table (including alpha), which must be uint8 and [0,255]
colors = np.hstack((in_c, np.ones_like(in_c[:, :1])))
colors = (colors * 255).astype(np.uint8)
plots['in_points'] = mlab.points3d(in_p[:, 0],
in_p[:, 1],
in_p[:, 2],
scalars,
resolution=8,
scale_factor=p_scale*0.8,
scale_mode='none',
figure=fig1)
plots['in_points'].module_manager.scalar_lut_manager.lut.table = colors
else:
plots['in_points'] = mlab.points3d(in_p[:, 0],
in_p[:, 1],
in_p[:, 2],
resolution=8,
scale_factor=p_scale*0.8,
scale_mode='none',
figure=fig1)
# Get KP locations
rescaled_aim_point = aim_point * model.config.in_radius / 1.5
point_i = lookuptrees[obj_i].query(rescaled_aim_point, return_distance=False)[0][0]
if offsets:
KP = points[obj_i][point_i] + deformed_KP[obj_i][point_i]
scals = np.ones_like(KP[:, 0])
else:
KP = points[obj_i][point_i] + original_KPs[chosen_KP]
scals = np.zeros_like(KP[:, 0])
KP = (KP * 1.5 / model.config.in_radius)
plots['KP'] = mlab.points3d(KP[:, 0],
KP[:, 1],
KP[:, 2],
scals,
colormap='autumn',
resolution=8,
scale_factor=1.2*p_scale,
scale_mode='none',
vmin=0,
vmax=1,
figure=fig1)
if True:
plots['center'] = mlab.points3d(p[point_i, 0],
p[point_i, 1],
p[point_i, 2],
scale_factor=1.1*p_scale,
scale_mode='none',
color=(0, 1, 0),
figure=fig1)
# New title
plots['title'] = mlab.title(str(obj_i), color=(0, 0, 0), size=0.3, height=0.01)
text = '<--- (press g for previous)' + 50 * ' ' + '(press h for next) --->'
plots['text'] = mlab.text(0.01, 0.01, text, color=(0, 0, 0), width=0.98)
plots['orient'] = mlab.orientation_axes()
# Set the saved view
mlab.view(*v)
mlab.roll(roll)
return
def animate_kernel():
global plots, offsets, p_scale, show_in_p
# Get KP locations
KP_def = points[obj_i][point_i] + deformed_KP[obj_i][point_i]
KP_def = (KP_def * 1.5 / model.config.in_radius)
KP_def_color = (1, 0, 0)
KP_rigid = points[obj_i][point_i] + original_KPs[chosen_KP]
KP_rigid = (KP_rigid * 1.5 / model.config.in_radius)
KP_rigid_color = (1, 0.7, 0)
if offsets:
t_list = np.linspace(0, 1, 150, dtype=np.float32)
else:
t_list = np.linspace(1, 0, 150, dtype=np.float32)
@mlab.animate(delay=10)
def anim():
for t in t_list:
plots['KP'].mlab_source.set(x=t * KP_def[:, 0] + (1 - t) * KP_rigid[:, 0],
y=t * KP_def[:, 1] + (1 - t) * KP_rigid[:, 1],
z=t * KP_def[:, 2] + (1 - t) * KP_rigid[:, 2],
scalars=t * np.ones_like(KP_def[:, 0]))
yield
anim()
return
def keyboard_callback(vtk_obj, event):
global obj_i, point_i, offsets, p_scale, show_in_p
if vtk_obj.GetKeyCode() in ['b', 'B']:
p_scale /= 1.5
update_scene()
elif vtk_obj.GetKeyCode() in ['n', 'N']:
p_scale *= 1.5
update_scene()
if vtk_obj.GetKeyCode() in ['g', 'G']:
obj_i = (obj_i - 1) % len(deformed_KP)
point_i = 0
update_scene()
elif vtk_obj.GetKeyCode() in ['h', 'H']:
obj_i = (obj_i + 1) % len(deformed_KP)
point_i = 0
update_scene()
elif vtk_obj.GetKeyCode() in ['k', 'K']:
offsets = not offsets
animate_kernel()
elif vtk_obj.GetKeyCode() in ['z', 'Z']:
show_in_p = (show_in_p + 1) % 3
update_scene()
elif vtk_obj.GetKeyCode() in ['0']:
print('Saving')
# Find a new name
file_i = 0
file_name = 'KP_{:03d}.ply'.format(file_i)
files = [f for f in listdir('KP_clouds') if f.endswith('.ply')]
while file_name in files:
file_i += 1
file_name = 'KP_{:03d}.ply'.format(file_i)
KP_deform = points[obj_i][point_i] + deformed_KP[obj_i][point_i]
KP_normal = points[obj_i][point_i] + original_KPs[chosen_KP]
# Save
write_ply(join('KP_clouds', file_name),
[in_points[obj_i], in_colors[obj_i]],
['x', 'y', 'z', 'red', 'green', 'blue'])
write_ply(join('KP_clouds', 'KP_{:03d}_deform.ply'.format(file_i)),
[KP_deform],
['x', 'y', 'z'])
write_ply(join('KP_clouds', 'KP_{:03d}_normal.ply'.format(file_i)),
[KP_normal],
['x', 'y', 'z'])
print('OK')
return
# Draw a first plot
pick_func = fig1.on_mouse_pick(picker_callback)
pick_func.tolerance = 0.01
update_scene()
fig1.scene.interactor.add_observer('KeyPressEvent', keyboard_callback)
mlab.show()
except tf.errors.OutOfRangeError:
break
def show_effective_recep_field(self, model, dataset, relu_idx=0):
###################################################
# First add a modulation variable on input features
###################################################
# Tensorflow random seed
random_seed = 42
# Create a modulated input feature op
with tf.variable_scope('input_modulations'):
initial = tf.constant(0., shape=[200000, 1])
input_modulations_var = tf.Variable(initial, name='alphas')
input_modulations = 2 * tf.sigmoid(input_modulations_var)
assert_op = tf.assert_less(tf.shape(model.inputs['features'])[0], tf.shape(input_modulations)[0])
with tf.control_dependencies([assert_op]):
modulated_input = model.inputs['features'] * input_modulations[:tf.shape(model.inputs['features'])[0]]
modulated_input = tf.identity(modulated_input, name='modulated_features')
print('*******************************************')
# Swap the op with the normal input features
for op in tf.get_default_graph().get_operations():
if 'input_modulations' in op.name:
continue
if model.inputs['features'].name in [in_t.name for in_t in op.inputs]:
input_list = []
for in_t in op.inputs:
if in_t.name == model.inputs['features'].name:
input_list += [modulated_input]
else:
input_list += [in_t]
print('swapping op ', op.name)
print('old inputs ', [in_t.name for in_t in op.inputs])
print('new inputs ', [in_t.name for in_t in input_list])
ge.swap_inputs(op, input_list)
print('*******************************************')
##########################
# Create the ERF optimizer
##########################
# This optimizer only computes gradients for the feature modulation variables. We set the ERF loss, which
# consists of modifying the features in one location a the wanted layer
with tf.variable_scope('ERF_loss'):
# List all relu ops
all_ops = [op for op in tf.get_default_graph().get_operations() if op.name.startswith('KernelPointNetwork')
and op.name.endswith('LeakyRelu')]
# Print the chosen one
features_tensor = all_ops[relu_idx].outputs[0]
# Get parameters
layer_idx = int(features_tensor.name.split('/')[1][6:])
if 'strided' in all_ops[relu_idx].name and not ('strided' in all_ops[relu_idx + 1].name):
layer_idx += 1
features_dim = int(features_tensor.shape[1])
radius = model.config.first_subsampling_dl * model.config.density_parameter * (2 ** layer_idx)
print('You chose to visualize the output of operation named: ' + all_ops[relu_idx].name)
print('It contains {:d} features.'.format(int(features_tensor.shape[1])))
print('\nPossible Relu indices:')
for i, t in enumerate(all_ops):
print(i, ': ', t.name)
print('\n****************************************************************************')
# Get the receptive field of a random point
N = tf.shape(features_tensor)[0]
#random_ind = tf.random_uniform([1], minval=0, maxval=N, dtype=np.int32, seed=random_seed)[0]
#chosen_i_holder = tf.placeholder(tf.int32, name='chosen_ind')
aimed_coordinates = tf.placeholder(tf.float32, shape=(1, 3), name='aimed_coordinates')
d2 = tf.reduce_sum(tf.square(model.inputs['points'][layer_idx] - aimed_coordinates), axis=1)
chosen_i_tf = tf.argmin(d2, output_type=tf.int32)
#test1 = tf.multiply(features_tensor, 2.0, name='test1')
#test2 = tf.multiply(features_tensor, 2.0, name='test2')
# Gradient scaling operation
@tf.custom_gradient
def scale_grad_layer(x):
def scaled_grad(dy):
p_op = tf.print(x.name,
tf.reduce_mean(tf.abs(x)),
tf.reduce_mean(tf.abs(dy)),
output_stream=sys.stdout)
with tf.control_dependencies([p_op]):
new_dy = 1.0 * dy
return new_dy
return tf.identity(x), scaled_grad
#test2 = scale_grad_layer(test2)
# Get the tensor of error for these features (one for the chosen point, zero for the rest)
chosen_f_tf = tf.placeholder(tf.int32, name='feature_ind')
ERF_error = tf.expand_dims(tf.cast(tf.equal(tf.range(N), chosen_i_tf), tf.float32), 1)
ERF_error *= tf.expand_dims(tf.cast(tf.equal(tf.range(features_dim), chosen_f_tf), tf.float32), 0)
# Get objective for the features (with a stop gradient so that we can get a gradient on the loss)
objective_features = features_tensor + ERF_error
objective_features = tf.stop_gradient(objective_features)
# Loss is the error but with the features that can be learned to correct it
ERF_loss = tf.reduce_sum(tf.square(objective_features - features_tensor))
with tf.variable_scope('ERF_optimizer'):
# Create the gradient descent optimizer with a dummy learning rate
optimizer = tf.train.GradientDescentOptimizer(1.0)
# Get the gradients with respect to the modulation variable
ERF_var_grads = optimizer.compute_gradients(ERF_loss, var_list=[input_modulations_var])
# Gradient of the modulations
ERF_train_op = optimizer.apply_gradients(ERF_var_grads)
################################
# Run model on all test examples
################################
# Init our modulation variable
self.sess.run(tf.variables_initializer([input_modulations_var]))
# Initialise iterator with test data
self.sess.run(dataset.test_init_op)
count = 0
global plots, p_scale, show_in_p, remove_h, aim_point
aim_point = np.zeros((1, 3), dtype=np.float32)
remove_h = 1.05
p_scale = 0.1
plots = {}
show_in_p = False
global points, in_points, grad_values, chosen_point, in_colors
points = None
in_points = np.zeros((0, 3))
grad_values = None
chosen_point = None
in_colors = None
###########################
# Interactive visualization
###########################
# Create figure for features
fig1 = mlab.figure('Features', bgcolor=(0.5, 0.5, 0.5), size=(640, 480))
fig1.scene.parallel_projection = False
# Indices
def update_ERF(only_points=False):
global points, in_points, grad_values, chosen_point, aim_point, in_colors
# Generate clouds until we effectively changed
if only_points:
for i in range(50):
all_points = self.sess.run(model.inputs['points'])
if all_points[0].shape[0] != in_points.shape[0]:
break
sum_grads = 0
if only_points:
num_tries = 1
else:
num_tries = 10
for test_i in range(num_tries):
print('Updating ERF {:.0f}%'.format((test_i + 1) * 100 / num_tries))
rand_f_i = np.random.randint(features_dim)
# Reset input modulation variable
reset_op = input_modulations_var.assign(tf.zeros_like(input_modulations_var))
self.sess.run(reset_op)
# Apply gradient to input modulations
t = [time.time()]
ops = (ERF_train_op,
chosen_i_tf,
input_modulations_var,
model.inputs['points'],
model.inputs['features'],
model.inputs['pools'],
model.inputs['in_batches'])
feed_dict = {aimed_coordinates: aim_point,
chosen_f_tf: rand_f_i,
model.dropout_prob: 1.0}
_, chosen_i, new_mods, all_points, all_colors, all_pools, in_batches = self.sess.run(ops, feed_dict)
t += [time.time()]
# Get the new value of the modulations
sum_grads += np.abs(self.sess.run(input_modulations_var))
grad = sum_grads / num_tries
# Stack all batches
max_ind = np.max(in_batches)
stacked_batches = []
for b_i, b in enumerate(in_batches):
stacked_batches += [b[b < max_ind - 0.5] * 0 + b_i]
stacked_batches = np.hstack(stacked_batches)
# Find batches at wanted layer
for l in range(model.config.num_layers - 1):
if l >= layer_idx:
break
stacked_batches = stacked_batches[all_pools[l][:, 0]]
# Get each example and update top_activations
for b_i, b in enumerate(in_batches):
b = b[b < max_ind - 0.5]
in_points = all_points[0][b]
in_colors = all_colors[b, 1:]
points = all_points[layer_idx][stacked_batches == b_i]
grad_values = grad[b]
chosen_point = all_points[layer_idx][chosen_i]
def update_scene():
global plots, p_scale, show_in_p, remove_h
global points, in_points, grad_values, chosen_point
# Get the current view
v = mlab.view()
roll = mlab.roll()
# clear figure
for key in plots.keys():
plots[key].remove()
plots = {}
# Plot new data feature
in_p = in_points
p = points
p0 = chosen_point
responses = 100 * np.abs(np.ravel(grad_values))
#xresponses = responses ** (1/2)
# Remove roof
if 0.0 < remove_h < 1.0:
floor_h = np.min(in_p[:, 2])
ceil_h = np.max(in_p[:, 2])
threshold = floor_h + (ceil_h - floor_h) * remove_h
responses = responses[in_p[:, 2] < threshold]
in_p = in_p[in_p[:, 2] < threshold]
p = p[p[:, 2] < threshold]
# Rescale responses
min_response, max_response = np.min(responses), np.max(responses)
# Show point cloud
if show_in_p:
plots['points'] = mlab.points3d(p[:, 0],
p[:, 1],
p[:, 2],
resolution=8,
scale_factor=p_scale,
scale_mode='none',
color=(0, 1, 1),
figure=fig1)
plots['in_points'] = mlab.points3d(in_p[:, 0],
in_p[:, 1],
in_p[:, 2],
responses,
resolution=8,
scale_factor=p_scale * 0.8,
scale_mode='none',
vmin=0.1,
vmax=1.5,
figure=fig1)
plots['center'] = mlab.points3d(p0[0],
p0[1],
p0[2],
scale_factor=1.5 * p_scale,
scale_mode='none',
color=(0, 0, 0),
figure=fig1)
# New title
plots['title'] = mlab.title(str(int(100*remove_h)) + '%', color=(0, 0, 0), size=0.3, height=0.01)
text = '<--- (press g to remove ceiling)' + 50 * ' ' + '(press h to add ceiling) --->'
plots['text'] = mlab.text(0.01, 0.01, text, color=(0, 0, 0), width=0.98)
plots['orient'] = mlab.orientation_axes()
# Set the saved view
mlab.view(*v)
mlab.roll(roll)
return
def picker_callback(picker):
""" Picker callback: this get called when on pick events.
"""
global plots, aim_point, in_points
if plots['in_points'].actor.actor._vtk_obj in [o._vtk_obj for o in picker.actors]:
point_rez = plots['in_points'].glyph.glyph_source.glyph_source.output.points.to_array().shape[0]
new_point_i = int(np.floor(picker.point_id / point_rez))
if new_point_i < len(plots['in_points'].mlab_source.points):
# Get closest point in the layer we are interested in
aim_point = plots['in_points'].mlab_source.points[new_point_i:new_point_i + 1]
update_ERF()
update_scene()
def keyboard_callback(vtk_obj, event):
global remove_h, p_scale, show_in_p
global in_points, grad_values, chosen_point, in_colors
print(vtk_obj.GetKeyCode())
if vtk_obj.GetKeyCode() in ['b', 'B']:
p_scale /= 1.5
update_scene()
elif vtk_obj.GetKeyCode() in ['n', 'N']:
p_scale *= 1.5
update_scene()
if vtk_obj.GetKeyCode() in ['g', 'G']:
if remove_h > 0.0:
remove_h -= 0.1
update_scene()
elif vtk_obj.GetKeyCode() in ['h', 'H']:
if remove_h < 1.0:
remove_h += 0.1
update_ERF()
update_scene()
elif vtk_obj.GetKeyCode() in ['z', 'Z']:
show_in_p = not show_in_p
update_scene()
elif vtk_obj.GetKeyCode() in ['x', 'X']:
# Reset potentials
dataset.potentials['ERF'] = []
dataset.min_potentials['ERF'] = []
for i, tree in enumerate(dataset.input_trees['test']):
dataset.potentials['ERF'] += [np.random.rand(tree.data.shape[0]) * 1e-3]
dataset.min_potentials['ERF'] += [float(np.min(dataset.potentials['ERF'][-1]))]
# Update figure
update_ERF(only_points=True)
update_scene()
elif vtk_obj.GetKeyCode() in ['0']:
print('Saving')
# Find a new name
file_i = 0
file_name = 'ERF_{:03d}.ply'.format(file_i)
files = [f for f in listdir('ERF_clouds') if f.endswith('.ply')]
while file_name in files:
file_i += 1
file_name = 'ERF_{:03d}.ply'.format(file_i)
# Save
responses = 100 * np.abs(np.ravel(grad_values))
write_ply(join('ERF_clouds', file_name),
[in_points, in_colors, responses],
['x', 'y', 'z', 'red', 'green', 'blue', 'erf'])
write_ply(join('ERF_clouds', 'ERF_{:03d}_center.ply'.format(file_i)),
[chosen_point.reshape([1, -1])],
['x', 'y', 'z'])
print('OK')
return
# Draw a first plot
pick_func = fig1.on_mouse_pick(picker_callback)
pick_func.tolerance = 0.01
update_ERF(only_points=True)
update_scene()
fig1.scene.interactor.add_observer('KeyPressEvent', keyboard_callback)
mlab.show()
def show_deformable_kernels(self, net, loader, config, deform_idx=0):
"""
Show some inference with deformable kernels
"""
##########################################
# First choose the visualized deformations
##########################################
print('\nList of the deformable convolution available (chosen one highlighted in green)')
fmt_str = ' {:}{:2d} > KPConv(r={:.3f}, Din={:d}, Dout={:d}){:}'
deform_convs = []
for m in net.modules():
if isinstance(m, KPConv) and m.deformable:
if len(deform_convs) == deform_idx:
color = bcolors.OKGREEN
else:
color = bcolors.FAIL
print(fmt_str.format(color, len(deform_convs), m.radius, m.in_channels, m.out_channels, bcolors.ENDC))
deform_convs.append(m)
################
# Initialization
################
print('\n****************************************************\n')
# Loop variables
t0 = time.time()
t = [time.time()]
last_display = time.time()
mean_dt = np.zeros(1)
count = 0
# Start training loop
for epoch in range(config.max_epoch):
for batch in loader:
##################
# Processing batch
##################
# New time
t = t[-1:]
t += [time.time()]
if 'cuda' in self.device.type:
batch.to(self.device)
# Forward pass
outputs = net(batch, config)
original_KP = deform_convs[deform_idx].kernel_points.cpu().detach().numpy()
stacked_deformed_KP = deform_convs[deform_idx].deformed_KP.cpu().detach().numpy()
count += batch.lengths[0].shape[0]
if 'cuda' in self.device.type:
torch.cuda.synchronize(self.device)
# Find layer
l = None
for i, p in enumerate(batch.points):
if p.shape[0] == stacked_deformed_KP.shape[0]:
l = i
t += [time.time()]
# Get data
in_points = []
in_colors = []
deformed_KP = []
points = []
lookuptrees = []
i0 = 0
for b_i, length in enumerate(batch.lengths[0]):
in_points.append(batch.points[0][i0:i0 + length].cpu().detach().numpy())
if batch.features.shape[1] == 4:
in_colors.append(batch.features[i0:i0 + length, 1:].cpu().detach().numpy())
else:
in_colors.append(None)
i0 += length
i0 = 0
for b_i, length in enumerate(batch.lengths[l]):
points.append(batch.points[l][i0:i0 + length].cpu().detach().numpy())
deformed_KP.append(stacked_deformed_KP[i0:i0 + length])
lookuptrees.append(KDTree(points[-1]))
i0 += length
###########################
# Interactive visualization
###########################
# Create figure for features
fig1 = mlab.figure('Deformations', bgcolor=(1.0, 1.0, 1.0), size=(1280, 920))
fig1.scene.parallel_projection = False
# Indices
global obj_i, point_i, plots, offsets, p_scale, show_in_p, aim_point
p_scale = 0.03
obj_i = 0
point_i = 0
plots = {}
offsets = False
show_in_p = 2
aim_point = np.zeros((1, 3))
def picker_callback(picker):
""" Picker callback: this get called when on pick events.
"""
global plots, aim_point
if 'in_points' in plots:
if plots['in_points'].actor.actor._vtk_obj in [o._vtk_obj for o in picker.actors]:
point_rez = plots['in_points'].glyph.glyph_source.glyph_source.output.points.to_array().shape[0]
new_point_i = int(np.floor(picker.point_id / point_rez))
if new_point_i < len(plots['in_points'].mlab_source.points):
# Get closest point in the layer we are interested in
aim_point = plots['in_points'].mlab_source.points[new_point_i:new_point_i + 1]
update_scene()
if 'points' in plots:
if plots['points'].actor.actor._vtk_obj in [o._vtk_obj for o in picker.actors]:
point_rez = plots['points'].glyph.glyph_source.glyph_source.output.points.to_array().shape[0]
new_point_i = int(np.floor(picker.point_id / point_rez))
if new_point_i < len(plots['points'].mlab_source.points):
# Get closest point in the layer we are interested in
aim_point = plots['points'].mlab_source.points[new_point_i:new_point_i + 1]
update_scene()
def update_scene():
global plots, offsets, p_scale, show_in_p, aim_point, point_i
# Get the current view
v = mlab.view()
roll = mlab.roll()
# clear figure
for key in plots.keys():
plots[key].remove()
plots = {}
# Plot new data feature
p = points[obj_i]
# Rescale points for visu
p = (p * 1.5 / config.in_radius)
# Show point cloud
if show_in_p <= 1:
plots['points'] = mlab.points3d(p[:, 0],
p[:, 1],
p[:, 2],
resolution=8,
scale_factor=p_scale,
scale_mode='none',
color=(0, 1, 1),
figure=fig1)
if show_in_p >= 1:
# Get points and colors
in_p = in_points[obj_i]
in_p = (in_p * 1.5 / config.in_radius)
# Color point cloud if possible
in_c = in_colors[obj_i]
if in_c is not None:
# Primitives
scalars = np.arange(len(in_p)) # Key point: set an integer for each point
# Define color table (including alpha), which must be uint8 and [0,255]
colors = np.hstack((in_c, np.ones_like(in_c[:, :1])))
colors = (colors * 255).astype(np.uint8)
plots['in_points'] = mlab.points3d(in_p[:, 0],
in_p[:, 1],
in_p[:, 2],
scalars,
resolution=8,
scale_factor=p_scale*0.8,
scale_mode='none',
figure=fig1)
plots['in_points'].module_manager.scalar_lut_manager.lut.table = colors
else:
plots['in_points'] = mlab.points3d(in_p[:, 0],
in_p[:, 1],
in_p[:, 2],
resolution=8,
scale_factor=p_scale*0.8,
scale_mode='none',
figure=fig1)
# Get KP locations
rescaled_aim_point = aim_point * config.in_radius / 1.5
point_i = lookuptrees[obj_i].query(rescaled_aim_point, return_distance=False)[0][0]
if offsets:
KP = points[obj_i][point_i] + deformed_KP[obj_i][point_i]
scals = np.ones_like(KP[:, 0])
else:
KP = points[obj_i][point_i] + original_KP
scals = np.zeros_like(KP[:, 0])
KP = (KP * 1.5 / config.in_radius)
plots['KP'] = mlab.points3d(KP[:, 0],
KP[:, 1],
KP[:, 2],
scals,
colormap='autumn',
resolution=8,
scale_factor=1.2*p_scale,
scale_mode='none',
vmin=0,
vmax=1,
figure=fig1)
if True:
plots['center'] = mlab.points3d(p[point_i, 0],
p[point_i, 1],
p[point_i, 2],
scale_factor=1.1*p_scale,
scale_mode='none',
color=(0, 1, 0),
figure=fig1)
# New title
plots['title'] = mlab.title(str(obj_i), color=(0, 0, 0), size=0.3, height=0.01)
text = '<--- (press g for previous)' + 50 * ' ' + '(press h for next) --->'
plots['text'] = mlab.text(0.01, 0.01, text, color=(0, 0, 0), width=0.98)
plots['orient'] = mlab.orientation_axes()
# Set the saved view
mlab.view(*v)
mlab.roll(roll)
return
def animate_kernel():
global plots, offsets, p_scale, show_in_p
# Get KP locations
KP_def = points[obj_i][point_i] + deformed_KP[obj_i][point_i]
KP_def = (KP_def * 1.5 / config.in_radius)
KP_def_color = (1, 0, 0)
KP_rigid = points[obj_i][point_i] + original_KP
KP_rigid = (KP_rigid * 1.5 / config.in_radius)
KP_rigid_color = (1, 0.7, 0)
if offsets:
t_list = np.linspace(0, 1, 150, dtype=np.float32)
else:
t_list = np.linspace(1, 0, 150, dtype=np.float32)
@mlab.animate(delay=10)
def anim():
for t in t_list:
plots['KP'].mlab_source.set(x=t * KP_def[:, 0] + (1 - t) * KP_rigid[:, 0],
y=t * KP_def[:, 1] + (1 - t) * KP_rigid[:, 1],
z=t * KP_def[:, 2] + (1 - t) * KP_rigid[:, 2],
scalars=t * np.ones_like(KP_def[:, 0]))
yield
anim()
return
def keyboard_callback(vtk_obj, event):
global obj_i, point_i, offsets, p_scale, show_in_p
if vtk_obj.GetKeyCode() in ['b', 'B']:
p_scale /= 1.5
update_scene()
elif vtk_obj.GetKeyCode() in ['n', 'N']:
p_scale *= 1.5
update_scene()
if vtk_obj.GetKeyCode() in ['g', 'G']:
obj_i = (obj_i - 1) % len(deformed_KP)
point_i = 0
update_scene()
elif vtk_obj.GetKeyCode() in ['h', 'H']:
obj_i = (obj_i + 1) % len(deformed_KP)
point_i = 0
update_scene()
elif vtk_obj.GetKeyCode() in ['k', 'K']:
offsets = not offsets
animate_kernel()
elif vtk_obj.GetKeyCode() in ['z', 'Z']:
show_in_p = (show_in_p + 1) % 3
update_scene()
elif vtk_obj.GetKeyCode() in ['0']:
print('Saving')
# Find a new name
file_i = 0
file_name = 'KP_{:03d}.ply'.format(file_i)
files = [f for f in listdir('KP_clouds') if f.endswith('.ply')]
while file_name in files:
file_i += 1
file_name = 'KP_{:03d}.ply'.format(file_i)
KP_deform = points[obj_i][point_i] + deformed_KP[obj_i][point_i]
KP_normal = points[obj_i][point_i] + original_KP
# Save
write_ply(join('KP_clouds', file_name),
[in_points[obj_i], in_colors[obj_i]],
['x', 'y', 'z', 'red', 'green', 'blue'])
write_ply(join('KP_clouds', 'KP_{:03d}_deform.ply'.format(file_i)),
[KP_deform],
['x', 'y', 'z'])
write_ply(join('KP_clouds', 'KP_{:03d}_normal.ply'.format(file_i)),
[KP_normal],
['x', 'y', 'z'])
print('OK')
return
# Draw a first plot
pick_func = fig1.on_mouse_pick(picker_callback)
pick_func.tolerance = 0.01
update_scene()
fig1.scene.interactor.add_observer('KeyPressEvent', keyboard_callback)
mlab.show()
return
@staticmethod
def show_activation(path, relu_idx=0, save_video=False):
"""
This function show the saved input point clouds maximizing the activations. You can also directly load the files
in a visualization software like CloudCompare.
In the case of relu_idx = 0 and if gaussian mode, the associated filter is also shown. This function can only
show the filters for the last saved epoch.
"""
################
# Find the files
################
# Check visu folder
visu_path = join('visu',
'visu_' + path.split('/')[-1],
'top_activations',
'Relu{:02d}'.format(relu_idx))
if not exists(visu_path):
message = 'Relu {:d} activations of the model {:s} not found.'
raise ValueError(message.format(relu_idx, path.split('/')[-1]))
# Get the list of files
feature_files = np.sort([f for f in listdir(visu_path) if f.endswith('.ply')])
if len(feature_files) == 0:
message = 'Relu {:d} activations of the model {:s} not found.'
raise ValueError(message.format(relu_idx, path.split('/')[-1]))
# Load mode
config = Config()
config.load(path)
mode = config.convolution_mode
#################
# Get activations
#################
all_points = []
all_responses = []
for file in feature_files:
# Load points
data = read_ply(join(visu_path, file))
all_points += [np.vstack((data['x'], data['y'], data['z'])).T]
all_responses += [data['responses']]
###########################
# Interactive visualization
###########################
# Create figure for features
fig1 = mlab.figure('Features', bgcolor=(0.5, 0.5, 0.5), size=(640, 480))
fig1.scene.parallel_projection = False
# Indices
global file_i
file_i = 0
def update_scene():
# clear figure
mlab.clf(fig1)
# Plot new data feature
points = all_points[file_i]
responses = all_responses[file_i]
min_response, max_response = np.min(responses), np.max(responses)
responses = (responses - min_response) / (max_response - min_response)
# Rescale points for visu
points = (points * 1.5 / config.in_radius + np.array([1.0, 1.0, 1.0])) * 50.0
# Show point clouds colorized with activations
activations = mlab.points3d(points[:, 0],
points[:, 1],
points[:, 2],
responses,
scale_factor=3.0,
scale_mode='none',
vmin=0.1,
vmax=0.9,
figure=fig1)
# New title
mlab.title(feature_files[file_i], color=(0, 0, 0), size=0.3, height=0.01)
text = '<--- (press g for previous)' + 50*' ' + '(press h for next) --->'
mlab.text(0.01, 0.01, text, color=(0, 0, 0), width=0.98)
mlab.orientation_axes()
return
def keyboard_callback(vtk_obj, event):
global file_i
if vtk_obj.GetKeyCode() in ['g', 'G']:
file_i = (file_i - 1) % len(all_responses)
update_scene()
elif vtk_obj.GetKeyCode() in ['h', 'H']:
file_i = (file_i + 1) % len(all_responses)
update_scene()
return
# Draw a first plot
update_scene()
fig1.scene.interactor.add_observer('KeyPressEvent', keyboard_callback)
mlab.show()
return
# Utilities
# ------------------------------------------------------------------------------------------------------------------
@staticmethod
def load_last_kernels(path):
# Directories of validation error
kernel_dirs = np.array([f for f in listdir(join(path, 'kernel_points')) if f.startswith('epoch')])
# Find last epoch folder
epochs = np.array([int(f[5:]) for f in kernel_dirs])
last_dir = kernel_dirs[np.argmax(epochs)]
# Find saved files for the first layer
kernel_file = join(path, 'kernel_points', last_dir, 'layer_0_simple_0.ply')
weights_file = join(path, 'kernel_points', last_dir, 'layer_0_simple_0.npy')
# Read kernel file
data = read_ply(kernel_file)
points = np.vstack((data['x'], data['y'], data['z'])).T
extents = data['sigma'].astype(np.float32)
# Read weight file
w = np.load(weights_file)
return points, extents, w
@staticmethod
def apply_weights(points, kernel, weights, extents):
# Get all difference matrices [n_points, n_kpoints, dim]
points = np.expand_dims(points, 1)
points = np.tile(points, [1, kernel.shape[0], 1])
differences = points - kernel
# Compute distance matrices [n_points, n_kpoints]
sq_distances = np.sum(np.square(differences), axis=-1)
# Compute gaussians [n_points, n_kpoints]
gaussian_values = np.exp(-sq_distances / (2 * np.square(extents)))
# Apply weights
return np.matmul(gaussian_values, np.squeeze(weights))
def top_relu_activations_old(self, model, dataset, relu_idx=0, top_num=5):
"""
Test the model on test dataset to see which points activate the most each neurons in a relu layer
:param model: model used at training
:param dataset: dataset used at training
:param relu_idx: which features are to be visualized
:param top_num: how many top candidates are kept per features
"""
#####################################
# First choose the visualized feature
#####################################
# List all relu ops
all_ops = [op for op in tf.get_default_graph().get_operations() if op.name.startswith('KernelPointNetwork')
and op.name.endswith('LeakyRelu')]
# Non relu ops in case we want the first KPConv features
KPConv_0 = [op for op in tf.get_default_graph().get_operations() if op.name.endswith('layer_0/simple_0/Sum_1')]
# Print the chosen one
if relu_idx == 0:
features_tensor = KPConv_0[relu_idx].outputs[0]
else:
features_tensor = all_ops[relu_idx].outputs[0]
# Get parameters
layer_idx = int(features_tensor.name.split('/')[1][6:])
if 'strided' in all_ops[relu_idx].name and not ('strided' in all_ops[relu_idx+1].name):
layer_idx += 1
features_dim = int(features_tensor.shape[1])
radius = model.config.first_subsampling_dl * model.config.density_parameter * (2 ** layer_idx)
if relu_idx == 0 :
print('SPECIAL CASE : relu_idx = 0 => visualization of the fist KPConv before relu')
print('You chose to visualize the output of operation named: ' + KPConv_0[0].name)
print('It contains {:d} features.'.format(int(features_tensor.shape[1])))
else :
print('You chose to visualize the output of operation named: ' + all_ops[relu_idx].name)
print('It contains {:d} features.'.format(int(features_tensor.shape[1])))
print('\nPossible Relu indices:')
for i, t in enumerate(all_ops):
print(i, ': ', t.name)
print('\n****************************************************************************')
#####################
# Initialize containers
#####################
# Initialize containers
self.top_features = -np.ones((top_num, features_dim))
self.top_classes = -np.ones((top_num, features_dim), dtype=np.int32)
self.saving = model.config.saving
# Testing parameters
num_votes = 3
# Create visu folder
self.visu_path = None
self.fmt_str = None
if model.config.saving:
self.visu_path = join('visu',
'visu_' + model.saving_path.split('/')[-1],
'top_activations',
'Relu{:02d}'.format(relu_idx))
self.fmt_str = 'f{:04d}_top{:02d}.ply'
if not exists(self.visu_path):
makedirs(self.visu_path)
# *******************
# Network predictions
# *******************
mean_dt = np.zeros(2)
last_display = time.time()
for v in range(num_votes):
# Run model on all test examples
# ******************************
# Initialise iterator with test data
if model.config.dataset.startswith('S3DIS'):
self.sess.run(dataset.val_init_op)
else:
self.sess.run(dataset.test_init_op)
count = 0
while True:
try:
if model.config.dataset.startswith('ShapeNetPart'):
if model.config.dataset.split('_')[1] == 'multi':
label_op = model.inputs['super_labels']
else:
label_op = model.inputs['point_labels']
elif model.config.dataset.startswith('S3DIS'):
label_op = model.inputs['point_labels']
elif model.config.dataset.startswith('Scannet'):
label_op = model.inputs['point_labels']
elif model.config.dataset.startswith('ModelNet40'):
label_op = model.inputs['labels']
else:
raise ValueError('Unsupported dataset')
# Run one step of the model
t = [time.time()]
ops = (all_ops[-1].outputs[0],
features_tensor,
label_op,
model.inputs['points'],
model.inputs['pools'],
model.inputs['in_batches'])
_, stacked_features, labels, all_points, all_pools, in_batches = self.sess.run(ops, {model.dropout_prob: 1.0})
t += [time.time()]
count += in_batches.shape[0]
# Stack all batches
max_ind = np.max(in_batches)
stacked_batches = []
for b_i, b in enumerate(in_batches):
stacked_batches += [b[b < max_ind - 0.5]*0+b_i]
stacked_batches = np.hstack(stacked_batches)
# Find batches at wanted layer
for l in range(model.config.num_layers - 1):
if l >= layer_idx:
break
stacked_batches = stacked_batches[all_pools[l][:, 0]]
# Get each example and update top_activations
for b_i, b in enumerate(in_batches):
b = b[b < max_ind - 0.5]
in_points = all_points[0][b]
features = stacked_features[stacked_batches == b_i]
points = all_points[layer_idx][stacked_batches == b_i]
if model.config.dataset in ['ShapeNetPart_multi', 'ModelNet40_classif']:
l = labels[b_i]
else:
l = np.argmax(np.bincount(labels[b]))
self.update_top_activations(features, labels[b_i], points, in_points, radius)
# Average timing
t += [time.time()]
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1]))
# Display
if (t[-1] - last_display) > 1.0:
last_display = t[-1]
if model.config.dataset.startswith('S3DIS'):
completed = count / (model.config.validation_size * model.config.batch_num)
else:
completed = count / dataset.num_test
message = 'Vote {:d} : {:.1f}% (timings : {:4.2f} {:4.2f})'
print(message.format(v,
100 * completed,
1000 * (mean_dt[0]),
1000 * (mean_dt[1])))
#class_names = np.array([dataset.label_to_names[i] for i in range(dataset.num_classes)])
#print(class_names[self.top_classes[:, :20]].T)
except tf.errors.OutOfRangeError:
break
return
def show_ModelNet_models(all_points):
###########################
# Interactive visualization
###########################
# Create figure for features
fig1 = mlab.figure('Models', bgcolor=(1, 1, 1), size=(1000, 800))
fig1.scene.parallel_projection = False
# Indices
global file_i
file_i = 0
def update_scene():
# clear figure
mlab.clf(fig1)
# Plot new data feature
points = all_points[file_i]
# Rescale points for visu
points = (points * 1.5 + np.array([1.0, 1.0, 1.0])) * 50.0
# Show point clouds colorized with activations
activations = mlab.points3d(points[:, 0],
points[:, 1],
points[:, 2],
points[:, 2],
scale_factor=3.0,
scale_mode='none',
figure=fig1)
# New title
mlab.title(str(file_i), color=(0, 0, 0), size=0.3, height=0.01)
text = '<--- (press g for previous)' + 50 * ' ' + '(press h for next) --->'
mlab.text(0.01, 0.01, text, color=(0, 0, 0), width=0.98)
mlab.orientation_axes()
return
def keyboard_callback(vtk_obj, event):
global file_i
if vtk_obj.GetKeyCode() in ['g', 'G']:
file_i = (file_i - 1) % len(all_points)
update_scene()
elif vtk_obj.GetKeyCode() in ['h', 'H']:
file_i = (file_i + 1) % len(all_points)
update_scene()
return
# Draw a first plot
update_scene()
fig1.scene.interactor.add_observer('KeyPressEvent', keyboard_callback)
mlab.show()
Reference in a new issue Copy permalink