231 lines
8.7 KiB
Python
231 lines
8.7 KiB
Python
#
|
|
#
|
|
# 0=================================0
|
|
# | Kernel Point Convolutions |
|
|
# 0=================================0
|
|
#
|
|
#
|
|
# ----------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Metric utility functions
|
|
#
|
|
# ----------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Hugues THOMAS - 11/06/2018
|
|
#
|
|
|
|
|
|
# ----------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Imports and global variables
|
|
# \**********************************/
|
|
#
|
|
|
|
|
|
# Basic libs
|
|
import numpy as np
|
|
|
|
|
|
# ----------------------------------------------------------------------------------------------------------------------
|
|
#
|
|
# Utilities
|
|
# \***************/
|
|
#
|
|
|
|
def fast_confusion(true, pred, label_values=None):
|
|
"""
|
|
Fast confusion matrix (100x faster than Scikit learn). But only works if labels are la
|
|
:param true:
|
|
:param false:
|
|
:param num_classes:
|
|
:return:
|
|
"""
|
|
|
|
# Ensure data is in the right format
|
|
true = np.squeeze(true)
|
|
pred = np.squeeze(pred)
|
|
if len(true.shape) != 1:
|
|
raise ValueError('Truth values are stored in a {:d}D array instead of 1D array'. format(len(true.shape)))
|
|
if len(pred.shape) != 1:
|
|
raise ValueError('Prediction values are stored in a {:d}D array instead of 1D array'. format(len(pred.shape)))
|
|
if true.dtype not in [np.int32, np.int64]:
|
|
raise ValueError('Truth values are {:s} instead of int32 or int64'.format(true.dtype))
|
|
if pred.dtype not in [np.int32, np.int64]:
|
|
raise ValueError('Prediction values are {:s} instead of int32 or int64'.format(pred.dtype))
|
|
true = true.astype(np.int32)
|
|
pred = pred.astype(np.int32)
|
|
|
|
# Get the label values
|
|
if label_values is None:
|
|
# From data if they are not given
|
|
label_values = np.unique(np.hstack((true, pred)))
|
|
else:
|
|
# Ensure they are good if given
|
|
if label_values.dtype not in [np.int32, np.int64]:
|
|
raise ValueError('label values are {:s} instead of int32 or int64'.format(label_values.dtype))
|
|
if len(np.unique(label_values)) < len(label_values):
|
|
raise ValueError('Given labels are not unique')
|
|
|
|
# Sort labels
|
|
label_values = np.sort(label_values)
|
|
|
|
# Get the number of classes
|
|
num_classes = len(label_values)
|
|
|
|
#print(num_classes)
|
|
#print(label_values)
|
|
#print(np.max(true))
|
|
#print(np.max(pred))
|
|
#print(np.max(true * num_classes + pred))
|
|
|
|
# Start confusion computations
|
|
if label_values[0] == 0 and label_values[-1] == num_classes - 1:
|
|
|
|
# Vectorized confusion
|
|
vec_conf = np.bincount(true * num_classes + pred)
|
|
|
|
# Add possible missing values due to classes not being in pred or true
|
|
#print(vec_conf.shape)
|
|
if vec_conf.shape[0] < num_classes ** 2:
|
|
vec_conf = np.pad(vec_conf, (0, num_classes ** 2 - vec_conf.shape[0]), 'constant')
|
|
#print(vec_conf.shape)
|
|
|
|
# Reshape confusion in a matrix
|
|
return vec_conf.reshape((num_classes, num_classes))
|
|
|
|
|
|
else:
|
|
|
|
# Ensure no negative classes
|
|
if label_values[0] < 0:
|
|
raise ValueError('Unsupported negative classes')
|
|
|
|
# Get the data in [0,num_classes[
|
|
label_map = np.zeros((label_values[-1] + 1,), dtype=np.int32)
|
|
for k, v in enumerate(label_values):
|
|
label_map[v] = k
|
|
|
|
pred = label_map[pred]
|
|
true = label_map[true]
|
|
|
|
# Vectorized confusion
|
|
vec_conf = np.bincount(true * num_classes + pred)
|
|
|
|
# Add possible missing values due to classes not being in pred or true
|
|
if vec_conf.shape[0] < num_classes ** 2:
|
|
vec_conf = np.pad(vec_conf, (0, num_classes ** 2 - vec_conf.shape[0]), 'constant')
|
|
|
|
# Reshape confusion in a matrix
|
|
return vec_conf.reshape((num_classes, num_classes))
|
|
|
|
def metrics(confusions, ignore_unclassified=False):
|
|
"""
|
|
Computes different metrics from confusion matrices.
|
|
:param confusions: ([..., n_c, n_c] np.int32). Can be any dimension, the confusion matrices should be described by
|
|
the last axes. n_c = number of classes
|
|
:param ignore_unclassified: (bool). True if the the first class should be ignored in the results
|
|
:return: ([..., n_c] np.float32) precision, recall, F1 score, IoU score
|
|
"""
|
|
|
|
# If the first class (often "unclassified") should be ignored, erase it from the confusion.
|
|
if (ignore_unclassified):
|
|
confusions[..., 0, :] = 0
|
|
confusions[..., :, 0] = 0
|
|
|
|
# Compute TP, FP, FN. This assume that the second to last axis counts the truths (like the first axis of a
|
|
# confusion matrix), and that the last axis counts the predictions (like the second axis of a confusion matrix)
|
|
TP = np.diagonal(confusions, axis1=-2, axis2=-1)
|
|
TP_plus_FP = np.sum(confusions, axis=-1)
|
|
TP_plus_FN = np.sum(confusions, axis=-2)
|
|
|
|
# Compute precision and recall. This assume that the second to last axis counts the truths (like the first axis of
|
|
# a confusion matrix), and that the last axis counts the predictions (like the second axis of a confusion matrix)
|
|
PRE = TP / (TP_plus_FN + 1e-6)
|
|
REC = TP / (TP_plus_FP + 1e-6)
|
|
|
|
# Compute Accuracy
|
|
ACC = np.sum(TP, axis=-1) / (np.sum(confusions, axis=(-2, -1)) + 1e-6)
|
|
|
|
# Compute F1 score
|
|
F1 = 2 * TP / (TP_plus_FP + TP_plus_FN + 1e-6)
|
|
|
|
# Compute IoU
|
|
IoU = F1 / (2 - F1)
|
|
|
|
return PRE, REC, F1, IoU, ACC
|
|
|
|
|
|
def smooth_metrics(confusions, smooth_n=0, ignore_unclassified=False):
|
|
"""
|
|
Computes different metrics from confusion matrices. Smoothed over a number of epochs.
|
|
:param confusions: ([..., n_c, n_c] np.int32). Can be any dimension, the confusion matrices should be described by
|
|
the last axes. n_c = number of classes
|
|
:param smooth_n: (int). smooth extent
|
|
:param ignore_unclassified: (bool). True if the the first class should be ignored in the results
|
|
:return: ([..., n_c] np.float32) precision, recall, F1 score, IoU score
|
|
"""
|
|
|
|
# If the first class (often "unclassified") should be ignored, erase it from the confusion.
|
|
if ignore_unclassified:
|
|
confusions[..., 0, :] = 0
|
|
confusions[..., :, 0] = 0
|
|
|
|
# Sum successive confusions for smoothing
|
|
smoothed_confusions = confusions.copy()
|
|
if confusions.ndim > 2 and smooth_n > 0:
|
|
for epoch in range(confusions.shape[-3]):
|
|
i0 = max(epoch - smooth_n, 0)
|
|
i1 = min(epoch + smooth_n + 1, confusions.shape[-3])
|
|
smoothed_confusions[..., epoch, :, :] = np.sum(confusions[..., i0:i1, :, :], axis=-3)
|
|
|
|
# Compute TP, FP, FN. This assume that the second to last axis counts the truths (like the first axis of a
|
|
# confusion matrix), and that the last axis counts the predictions (like the second axis of a confusion matrix)
|
|
TP = np.diagonal(smoothed_confusions, axis1=-2, axis2=-1)
|
|
TP_plus_FP = np.sum(smoothed_confusions, axis=-2)
|
|
TP_plus_FN = np.sum(smoothed_confusions, axis=-1)
|
|
|
|
# Compute precision and recall. This assume that the second to last axis counts the truths (like the first axis of
|
|
# a confusion matrix), and that the last axis counts the predictions (like the second axis of a confusion matrix)
|
|
PRE = TP / (TP_plus_FN + 1e-6)
|
|
REC = TP / (TP_plus_FP + 1e-6)
|
|
|
|
# Compute Accuracy
|
|
ACC = np.sum(TP, axis=-1) / (np.sum(smoothed_confusions, axis=(-2, -1)) + 1e-6)
|
|
|
|
# Compute F1 score
|
|
F1 = 2 * TP / (TP_plus_FP + TP_plus_FN + 1e-6)
|
|
|
|
# Compute IoU
|
|
IoU = F1 / (2 - F1)
|
|
|
|
return PRE, REC, F1, IoU, ACC
|
|
|
|
|
|
def IoU_from_confusions(confusions):
|
|
"""
|
|
Computes IoU from confusion matrices.
|
|
:param confusions: ([..., n_c, n_c] np.int32). Can be any dimension, the confusion matrices should be described by
|
|
the last axes. n_c = number of classes
|
|
:param ignore_unclassified: (bool). True if the the first class should be ignored in the results
|
|
:return: ([..., n_c] np.float32) IoU score
|
|
"""
|
|
|
|
# Compute TP, FP, FN. This assume that the second to last axis counts the truths (like the first axis of a
|
|
# confusion matrix), and that the last axis counts the predictions (like the second axis of a confusion matrix)
|
|
TP = np.diagonal(confusions, axis1=-2, axis2=-1)
|
|
TP_plus_FN = np.sum(confusions, axis=-1)
|
|
TP_plus_FP = np.sum(confusions, axis=-2)
|
|
|
|
# Compute IoU
|
|
IoU = TP / (TP_plus_FP + TP_plus_FN - TP + 1e-6)
|
|
|
|
# Compute mIoU with only the actual classes
|
|
mask = TP_plus_FN < 1e-3
|
|
counts = np.sum(1 - mask, axis=-1, keepdims=True)
|
|
mIoU = np.sum(IoU, axis=-1, keepdims=True) / (counts + 1e-6)
|
|
|
|
# If class is absent, place mIoU in place of 0 IoU to get the actual mean later
|
|
IoU += mask * mIoU
|
|
|
|
return IoU
|