Skip to content

Commit

Permalink
Calibration plot: Test missing probabilities and single classes
Browse files Browse the repository at this point in the history
  • Loading branch information
@janezd
janezd committed Jun 19, 2019
1 parent 2edcb39 commit 2049afa
Show file tree
Hide file tree
Showing 2 changed files with 208 additions and 93 deletions.
159 changes: 97 additions & 62 deletions Orange/widgets/evaluate/owcalibrationplot.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,18 +74,23 @@ class Outputs:
class Error(widget.OWWidget.Error):
non_discrete_target = Msg("Calibration plot requires a discrete target")
empty_input = widget.Msg("Empty result on input. Nothing to display.")
nan_classes = \
widget.Msg("Remove test data instances with unknown classes")
all_target_class = widget.Msg(
"All data instances belong to target class")
no_target_class = widget.Msg(
"No data instances belong to target class")

class Warning(widget.OWWidget.Warning):
omitted_folds = widget.Msg(
"Test folds where all data belongs to (non)-target are not shown")
omitted_nan_prob_points = widget.Msg(
"Instance for which the model couldn't compute probabilities are"
"skipped")
no_valid_data = widget.Msg("No valid data for model(s) {}")

class Information(widget.OWWidget.Information):
no_out = "Can't output a model: "
no_output_multiple_folds = Msg(
no_out + "each training data sample produces a different model")
no_output_no_models = Msg(
no_out + "test results do not contain stored models;\n"
"try testing on separate data or on training data")
no_output_multiple_selected = Msg(
no_out + "select a single model - the widget can output only one")
no_output_non_binary_class = Msg(
no_out + "cannot calibrate non-binary classes")
no_output = Msg("Can't output a model: {}")

settingsHandler = EvaluationResultsContextHandler()
target_index = settings.ContextSetting(0)
Expand Down Expand Up @@ -179,19 +184,23 @@ def set_results(self, results):
self.clear()
self.Error.clear()
self.Information.clear()
if results is not None and not results.domain.has_discrete_class:
self.Error.non_discrete_target()
results = None
if results is not None and not results.actual.size:
self.Error.empty_input()
results = None
self.results = results
if self.results is not None:
self._initialize(results)
class_var = self.results.domain.class_var
self.target_index = int(len(class_var.values) == 2)
self.openContext(class_var, self.classifier_names)
self._replot()

self.results = None
if results is not None:
if not results.domain.has_discrete_class:
self.Error.non_discrete_target()
elif not results.actual.size:
self.Error.empty_input()
elif np.any(np.isnan(results.actual)):
self.Error.nan_classes()
else:
self.results = results
self._initialize(results)
class_var = self.results.domain.class_var
self.target_index = int(len(class_var.values) == 2)
self.openContext(class_var, self.classifier_names)
self._replot()

self.apply()

def clear(self):
Expand Down Expand Up @@ -286,9 +295,6 @@ def plot_metrics(self, data, metrics, pen_args):
return data.probs, ys

def _prob_curve(self, ytrue, probs, pen_args):
if not probs.size:
return None

xmin, xmax = probs.min(), probs.max()
x = np.linspace(xmin, xmax, 100)
if xmax != xmin:
Expand All @@ -307,16 +313,25 @@ def _setup_plot(self):
plot_folds = self.fold_curves and results.folds is not None
self.scores = []

ytrue = results.actual == target
if not self._check_class_presence(results.actual == target):
return

self.Warning.omitted_folds.clear()
self.Warning.omitted_nan_prob_points.clear()
no_valid_models = []
shadow_width = 4 + 4 * plot_folds
for clsf in self.selected_classifiers:
probs = results.probabilities[clsf, :, target]
data = Curves.from_results(results, target, clsf)
if data.tot == 0: # all probabilities are nan
no_valid_models.append(clsf)
continue
if data.tot != results.probabilities.shape[1]: # some are nan
self.Warning.omitted_nan_prob_points()

color = self.colors[clsf]
pen_args = dict(
pen=pg.mkPen(color, width=1),
shadowPen=pg.mkPen(color.lighter(160),
width=4 + 4 * plot_folds),
antiAlias=True)
data = Curves(ytrue, probs)
pen=pg.mkPen(color, width=1), antiAlias=True,
shadowPen=pg.mkPen(color.lighter(160), width=shadow_width))
self.scores.append(
(self.classifier_names[clsf],
self.plot_metrics(data, metrics, pen_args)))
Expand All @@ -330,28 +345,46 @@ def _setup_plot(self):
antiAlias=True)
for fold in range(len(results.folds)):
fold_results = results.get_fold(fold)
fold_ytrue = fold_results.actual == target
fold_probs = fold_results.probabilities[clsf, :, target]
self.plot_metrics(Curves(fold_ytrue, fold_probs),
metrics, pen_args)
fold_curve = Curves.from_results(fold_results, target, clsf)
# Can't check this before: p and n can be 0 because of
# nan probabilities
if fold_curve.p * fold_curve.n == 0:
self.Warning.omitted_folds()
self.plot_metrics(fold_curve, metrics, pen_args)

if no_valid_models:
self.Warning.no_valid_data(
", ".join(self.classifier_names[i] for i in no_valid_models))

if self.score == 0:
self.plot.plot([0, 1], [0, 1], antialias=True)

def _replot(self):
self.plot.clear()
if self.results is not None:
self._setup_plot()
if self.score != 0:
else:
self.line = pg.InfiniteLine(
pos=self.threshold, movable=True,
pen=pg.mkPen(color="k", style=Qt.DashLine, width=2),
hoverPen=pg.mkPen(color="k", style=Qt.DashLine, width=3),
bounds=(0, 1),
)
self.line.sigPositionChanged.connect(self.threshold_change)
self.line.sigPositionChangeFinished.connect(self.threshold_change_done)
self.line.sigPositionChangeFinished.connect(
self.threshold_change_done)
self.plot.addItem(self.line)

def _check_class_presence(self, ytrue):
self.Error.all_target_class.clear()
self.Error.no_target_class.clear()
if np.max(ytrue) == 0:
self.Error.no_target_class()
return False
if np.min(ytrue) == 1:
self.Error.all_target_class()
return False
return True

def _replot(self):
self.plot.clear()
if self.results is not None:
self._setup_plot()
self._update_info()

def _on_display_rug_changed(self):
Expand Down Expand Up @@ -380,10 +413,7 @@ def _update_info(self):
{"<td></td>".join(f"<td align='right'>{n}</td>"
for n in short_names)}
</tr>"""
for name, probs_curves in self.scores:
if probs_curves is None:
continue
probs, curves = probs_curves
for name, (probs, curves) in self.scores:
ind = min(np.searchsorted(probs, self.threshold),
len(probs) - 1)
text += f"<tr><th align='right'>{name}:</th>"
Expand All @@ -397,20 +427,28 @@ def threshold_change_done(self):
self.apply()

def apply(self):
info = self.Information
self.Information.no_output.clear()
wrapped = None
problems = {}
results = self.results
if results is not None:
problems = {
info.no_output_multiple_folds: len(results.folds) > 1,
info.no_output_no_models: results.models is None,
info.no_output_multiple_selected:
len(self.selected_classifiers) != 1,
info.no_output_non_binary_class:
self.score != 0
and len(results.domain.class_var.values) != 2}
if not any(problems.values()):
problems = [
msg for condition, msg in (
(len(results.folds) > 1,
"each training data sample produces a different model"),
(results.models is None,
"test results do not contain stored models - try testing on"
"separate data or on training data"),
(len(self.selected_classifiers) != 1,
"select a single model - the widget can output only one"),
(self.score != 0 and len(results.domain.class_var.values) != 2,
"cannot calibrate non-binary classes"))
if condition]
if len(problems) == 1:
self.Information.no_output(problems[0])
elif problems:
self.Information.no_output(
"".join(f"\n - {problem}" for problem in problems))
else:
clsf_idx = self.selected_classifiers[0]
model = results.models[0, clsf_idx]
if self.score == 0:
Expand All @@ -424,9 +462,6 @@ def apply(self):
wrapped = ThresholdClassifier(model, threshold)

self.Outputs.calibrated_model.send(wrapped)
for info, shown in problems.items():
if info.is_shown() != shown:
info(shown=shown)

def send_report(self):
if self.results is None:
Expand Down
Loading

0 comments on commit 2049afa

Please sign in to comment.