diff --git a/Orange/widgets/evaluate/owcalibrationplot.py b/Orange/widgets/evaluate/owcalibrationplot.py index 558a47ba41d..03affe276d6 100644 --- a/Orange/widgets/evaluate/owcalibrationplot.py +++ b/Orange/widgets/evaluate/owcalibrationplot.py @@ -5,6 +5,8 @@ """ from collections import namedtuple +import numpy as np + from AnyQt.QtWidgets import QListWidget import pyqtgraph as pg @@ -13,7 +15,6 @@ from Orange.widgets import widget, gui, settings from Orange.widgets.evaluate.utils import check_results_adequacy from Orange.widgets.utils import colorpalette, colorbrewer -from Orange.widgets.io import FileFormat from Orange.widgets.widget import Input from Orange.canvas import report @@ -122,8 +123,10 @@ def _initialize(self, results): names = ["#{}".format(i + 1) for i in range(N)] self.classifier_names = names - self.colors = colorpalette.ColorPaletteGenerator( - N, colorbrewer.colorSchemes["qualitative"]["Dark2"]) + scheme = colorbrewer.colorSchemes["qualitative"]["Dark2"] + if N > len(scheme): + scheme = colorpalette.DefaultRGBColors + self.colors = colorpalette.ColorPaletteGenerator(N, scheme) for i in range(N): item = self.classifiers_list_box.item(i) @@ -138,17 +141,17 @@ def plot_curve(self, clf_idx, target): ytrue = self.results.actual == target probs = self.results.probabilities[clf_idx, :, target] - sortind = numpy.argsort(probs) + sortind = np.argsort(probs) probs = probs[sortind] ytrue = ytrue[sortind] if probs.size: xmin, xmax = probs.min(), probs.max() - x = numpy.linspace(xmin, xmax, 100) + x = np.linspace(xmin, xmax, 100) f = gaussian_smoother(probs, ytrue, sigma=0.15 * (xmax - xmin)) observed = f(x) else: - x = numpy.array([]) - observed = numpy.array([]) + x = np.array([]) + observed = np.array([]) curve = Curve(x, observed) curve_item = pg.PlotDataItem( @@ -159,13 +162,13 @@ def plot_curve(self, clf_idx, target): ) rh = 0.025 - rug_x = numpy.c_[probs, probs] + rug_x = np.c_[probs, probs] rug_x_true = rug_x[ytrue].ravel() rug_x_false = rug_x[~ytrue].ravel() - rug_y_true = numpy.ones_like(rug_x_true) + rug_y_true = np.ones_like(rug_x_true) rug_y_true[1::2] = 1 - rh - rug_y_false = numpy.zeros_like(rug_x_false) + rug_y_false = np.zeros_like(rug_x_false) rug_y_false[1::2] = rh rug1 = pg.PlotDataItem( @@ -210,24 +213,21 @@ def send_report(self): self.report_caption(caption) -import numpy - - def gaussian_smoother(x, y, sigma=1.0): - x = numpy.asarray(x) - y = numpy.asarray(y) + x = np.asarray(x) + y = np.asarray(y) gamma = 1. / (2 * sigma ** 2) - a = 1. / (sigma * numpy.sqrt(2 * numpy.pi)) + a = 1. / (sigma * np.sqrt(2 * np.pi)) if x.shape != y.shape: raise ValueError def smoother(xs): - W = a * numpy.exp(-gamma * ((xs - x) ** 2)) - return numpy.average(y, weights=W) + W = a * np.exp(-gamma * ((xs - x) ** 2)) + return np.average(y, weights=W) - return numpy.vectorize(smoother, otypes=[numpy.float]) + return np.vectorize(smoother, otypes=[np.float]) def main(): @@ -248,7 +248,7 @@ def main(): LogisticRegressionLearner(penalty="l1"), SVMLearner(probability=True), NuSVMLearner(probability=True) - ], + ], store_data=True ) results.learner_names = ["LR l2", "LR l1", "SVM", "Nu SVM"] diff --git a/Orange/widgets/evaluate/owliftcurve.py b/Orange/widgets/evaluate/owliftcurve.py index f0cddb0062a..758956003eb 100644 --- a/Orange/widgets/evaluate/owliftcurve.py +++ b/Orange/widgets/evaluate/owliftcurve.py @@ -5,7 +5,7 @@ """ from collections import namedtuple -import numpy +import numpy as np import sklearn.metrics as skl_metrics from AnyQt import QtWidgets @@ -19,7 +19,6 @@ from Orange.widgets.evaluate.utils import check_results_adequacy from Orange.widgets.utils import colorpalette, colorbrewer from Orange.widgets.evaluate.owrocanalysis import convex_hull -from Orange.widgets.io import FileFormat from Orange.widgets.widget import Input from Orange.canvas import report @@ -37,7 +36,7 @@ LiftCurve.is_valid = property(lambda self: self.points.is_valid) -def LiftCurve_from_results(results, clf_index, target): +def liftCurve_from_results(results, clf_index, target): x, y, thresholds = lift_curve_from_results(results, target, clf_index) points = CurvePoints(x, y, thresholds) @@ -155,8 +154,10 @@ def _initialize(self, results): if names is None: names = ["#{}".format(i + 1) for i in range(N)] - self.colors = colorpalette.ColorPaletteGenerator( - N, colorbrewer.colorSchemes["qualitative"]["Dark2"]) + scheme = colorbrewer.colorSchemes["qualitative"]["Dark2"] + if N > len(scheme): + scheme = colorpalette.DefaultRGBColors + self.colors = colorpalette.ColorPaletteGenerator(N, scheme) self.classifier_names = names self.selected_classifiers = list(range(N)) @@ -168,7 +169,7 @@ def _initialize(self, results): def plot_curves(self, target, clf_idx): if (target, clf_idx) not in self._curve_data: - curve = LiftCurve_from_results(self.results, clf_idx, target) + curve = liftCurve_from_results(self.results, clf_idx, target) color = self.colors[clf_idx] pen = QPen(color, 1) pen.setCosmetic(True) @@ -182,7 +183,7 @@ def plot_curves(self, target, clf_idx): ) hull_item = pg.PlotDataItem( curve.hull[0], curve.hull[1], - pen=pen, antialias=True + pen=pen, antialias=True ) self._curve_data[target, clf_idx] = \ PlotCurve(curve, item, hull_item) @@ -243,12 +244,12 @@ def lift_curve_from_results(results, target, clf_idx, subset=slice(0, -1)): def lift_curve(ytrue, ypred, target=1): - P = numpy.sum(ytrue == target) + P = np.sum(ytrue == target) N = ytrue.size - P if P == 0 or N == 0: # Undefined TP and FP rate - return numpy.array([]), numpy.array([]), numpy.array([]) + return np.array([]), np.array([]), np.array([]) fpr, tpr, thresholds = skl_metrics.roc_curve(ytrue, ypred, target) rpp = fpr * (N / (P + N)) + tpr * (P / (P + N)) @@ -273,7 +274,7 @@ def main(): LogisticRegressionLearner(penalty="l1"), SVMLearner(probability=True), NuSVMLearner(probability=True) - ], + ], store_data=True ) results.learner_names = ["LR l2", "LR l1", "SVM", "Nu SVM"] diff --git a/Orange/widgets/evaluate/tests/base.py b/Orange/widgets/evaluate/tests/base.py new file mode 100644 index 00000000000..28f6130ade3 --- /dev/null +++ b/Orange/widgets/evaluate/tests/base.py @@ -0,0 +1,27 @@ +from Orange import classification, evaluation +from Orange.data import Table + + +class EvaluateTest: + + def test_many_evaluation_results(self): + """ + Now works with more than 9 evaluation results. + GH-2394 (ROC Analysis) + GH-2522 (Lift Curve, Calibration Plot) + """ + data = Table("iris") + learners = [ + classification.MajorityLearner(), + classification.LogisticRegressionLearner(), + classification.TreeLearner(), + classification.SVMLearner(), + classification.KNNLearner(), + classification.CN2Learner(), + classification.SGDClassificationLearner(), + classification.RandomForestLearner(), + classification.NaiveBayesLearner(), + classification.SGDClassificationLearner() + ] + res = evaluation.CrossValidation(data, learners, k=2, store_data=True) + self.send_signal("Evaluation Results", res) diff --git a/Orange/widgets/evaluate/tests/test_owcalibrationplot.py b/Orange/widgets/evaluate/tests/test_owcalibrationplot.py index 9ee78d2f161..f716948d9fd 100644 --- a/Orange/widgets/evaluate/tests/test_owcalibrationplot.py +++ b/Orange/widgets/evaluate/tests/test_owcalibrationplot.py @@ -6,11 +6,12 @@ import Orange.evaluation import Orange.classification -from Orange.widgets.tests.base import WidgetTest +from Orange.widgets.evaluate.tests.base import EvaluateTest from Orange.widgets.evaluate.owcalibrationplot import OWCalibrationPlot +from Orange.widgets.tests.base import WidgetTest -class TestOWCalibrationPlot(WidgetTest): +class TestOWCalibrationPlot(WidgetTest, EvaluateTest): @classmethod def setUpClass(cls): super().setUpClass() diff --git a/Orange/widgets/evaluate/tests/test_owliftcurve.py b/Orange/widgets/evaluate/tests/test_owliftcurve.py index a8048972a4c..6d3781063ff 100644 --- a/Orange/widgets/evaluate/tests/test_owliftcurve.py +++ b/Orange/widgets/evaluate/tests/test_owliftcurve.py @@ -6,12 +6,13 @@ import Orange.evaluation import Orange.classification +from Orange.widgets.evaluate.tests.base import EvaluateTest from Orange.widgets.tests.base import WidgetTest from Orange.widgets.tests.utils import simulate from Orange.widgets.evaluate.owliftcurve import OWLiftCurve -class TestOWLiftCurve(WidgetTest): +class TestOWLiftCurve(WidgetTest, EvaluateTest): @classmethod def setUpClass(cls): super().setUpClass() diff --git a/Orange/widgets/evaluate/tests/test_owrocanalysis.py b/Orange/widgets/evaluate/tests/test_owrocanalysis.py index b9189411281..b7891dd5860 100644 --- a/Orange/widgets/evaluate/tests/test_owrocanalysis.py +++ b/Orange/widgets/evaluate/tests/test_owrocanalysis.py @@ -1,6 +1,8 @@ +# pylint: disable=protected-access + import unittest import copy -import numpy +import numpy as np import Orange.data import Orange.evaluation @@ -8,6 +10,7 @@ from Orange.widgets.evaluate import owrocanalysis from Orange.widgets.evaluate.owrocanalysis import OWROCAnalysis +from Orange.widgets.evaluate.tests.base import EvaluateTest from Orange.widgets.tests.base import WidgetTest @@ -32,7 +35,7 @@ def test_ROCData_from_results(self): # fixed random seed because otherwise it could happen that data sample # contained only instances of two classes (and the test then fails) - data = data[numpy.random.RandomState(0).choice(len(data), size=20)] + data = data[np.random.RandomState(0).choice(len(data), size=20)] res = Orange.evaluation.LeaveOneOut(data, learners) for i, _ in enumerate(learners): @@ -62,7 +65,7 @@ def test_ROCData_from_results(self): self.assertFalse(rocdata.avg_threshold.is_valid) -class TestOWROCAnalysis(WidgetTest): +class TestOWROCAnalysis(WidgetTest, EvaluateTest): @classmethod def setUpClass(cls): @@ -109,10 +112,10 @@ def test_basic(self): def test_empty_input(self): res = Orange.evaluation.Results( data=self.lenses[:0], nmethods=2, store_data=True) - res.row_indices = numpy.array([], dtype=int) - res.actual = numpy.array([]) - res.predicted = numpy.zeros((2, 0)) - res.probabilities = numpy.zeros((2, 0, 3)) + res.row_indices = np.array([], dtype=int) + res.actual = np.array([]) + res.predicted = np.zeros((2, 0)) + res.probabilities = np.zeros((2, 0, 3)) self.send_signal(self.widget.Inputs.evaluation_results, res) self.widget.roc_averaging = OWROCAnalysis.Merge @@ -124,10 +127,10 @@ def test_empty_input(self): self.widget.roc_averaging = OWROCAnalysis.NoAveraging self.widget._replot() - res.row_indices = numpy.array([1], dtype=int) - res.actual = numpy.array([0.0]) - res.predicted = numpy.zeros((2, 1)) - res.probabilities = numpy.zeros((2, 1, 3)) + res.row_indices = np.array([1], dtype=int) + res.actual = np.array([0.0]) + res.predicted = np.zeros((2, 1)) + res.probabilities = np.zeros((2, 1, 3)) self.send_signal(self.widget.Inputs.evaluation_results, res) self.widget.roc_averaging = OWROCAnalysis.Merge @@ -145,31 +148,11 @@ def test_nan_input(self): res.predicted = res.predicted.copy() res.probabilities = res.probabilities.copy() - res.actual[0] = numpy.nan - res.predicted[:, 1] = numpy.nan - res.probabilities[0, 1, :] = numpy.nan + res.actual[0] = np.nan + res.predicted[:, 1] = np.nan + res.probabilities[0, 1, :] = np.nan self.send_signal(self.widget.Inputs.evaluation_results, res) self.assertTrue(self.widget.Error.invalid_results.is_shown()) self.send_signal(self.widget.Inputs.evaluation_results, None) self.assertFalse(self.widget.Error.invalid_results.is_shown()) - def test_many_evaluation_results(self): - """ - Now works with more than 9 evaluation results. - GH-2394 - """ - data = Orange.data.Table("iris") - learners = [ - Orange.classification.MajorityLearner(), - Orange.classification.LogisticRegressionLearner(), - Orange.classification.TreeLearner(), - Orange.classification.SVMLearner(), - Orange.classification.KNNLearner(), - Orange.classification.CN2Learner(), - Orange.classification.SGDClassificationLearner(), - Orange.classification.RandomForestLearner(), - Orange.classification.NaiveBayesLearner(), - Orange.classification.SGDClassificationLearner() - ] - res = Orange.evaluation.CrossValidation(data, learners, k=2, store_data=True) - self.send_signal("Evaluation Results", res)