From e6802f47783f96dd488c4913c2d64906fe505fc2 Mon Sep 17 00:00:00 2001 From: janezd Date: Fri, 6 Jan 2023 17:06:21 +0100 Subject: [PATCH 1/6] Scorer: Add priority and visibility; order by priority --- Orange/evaluation/scoring.py | 18 ++++++++++++++ Orange/widgets/evaluate/tests/test_utils.py | 26 ++++++++++----------- Orange/widgets/evaluate/utils.py | 16 +++++-------- 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/Orange/evaluation/scoring.py b/Orange/evaluation/scoring.py index 0f3eb47d3bb..062abae79e4 100644 --- a/Orange/evaluation/scoring.py +++ b/Orange/evaluation/scoring.py @@ -66,6 +66,9 @@ class Score(metaclass=ScoreMetaType): name = None long_name = None #: A short user-readable name (e.g. a few words) + default_visible = True + priority = 100 + def __new__(cls, results=None, **kwargs): self = super().__new__(cls) if results is not None: @@ -137,6 +140,7 @@ def is_compatible(domain: Domain) -> bool: class CA(ClassificationScore): __wraps__ = skl_metrics.accuracy_score long_name = "Classification accuracy" + priority = 20 class PrecisionRecallFSupport(ClassificationScore): @@ -185,14 +189,17 @@ def compute_score(self, results, target=None, average='binary'): class Precision(TargetScore): __wraps__ = skl_metrics.precision_score + priority = 40 class Recall(TargetScore): __wraps__ = skl_metrics.recall_score + priority = 50 class F1(TargetScore): __wraps__ = skl_metrics.f1_score + priority = 30 class AUC(ClassificationScore): @@ -211,6 +218,7 @@ class AUC(ClassificationScore): separate_folds = True is_binary = True long_name = "Area under ROC curve" + priority = 10 @staticmethod def calculate_weights(results): @@ -282,6 +290,8 @@ class LogLoss(ClassificationScore): """ __wraps__ = skl_metrics.log_loss + priority = 120 + default_visible = False def compute_score(self, results, eps=1e-15, normalize=True, sample_weight=None): @@ -297,6 +307,8 @@ def compute_score(self, results, eps=1e-15, normalize=True, class Specificity(ClassificationScore): is_binary = True + priority = 110 + default_visible = False @staticmethod def calculate_weights(results): @@ -350,6 +362,7 @@ def compute_score(self, results, target=None, average="binary"): class MSE(RegressionScore): __wraps__ = skl_metrics.mean_squared_error long_name = "Mean square error" + priority = 20 class RMSE(RegressionScore): @@ -357,21 +370,26 @@ class RMSE(RegressionScore): def compute_score(self, results): return np.sqrt(MSE(results)) + priority = 30 class MAE(RegressionScore): __wraps__ = skl_metrics.mean_absolute_error long_name = "Mean absolute error" + priority = 40 # pylint: disable=invalid-name class R2(RegressionScore): __wraps__ = skl_metrics.r2_score long_name = "Coefficient of determination" + priority = 50 class CVRMSE(RegressionScore): long_name = "Coefficient of variation of the RMSE" + priority = 110 + default_visible = False def compute_score(self, results): mean = np.nanmean(results.actual) diff --git a/Orange/widgets/evaluate/tests/test_utils.py b/Orange/widgets/evaluate/tests/test_utils.py index f696091fd7f..13eb53f5525 100644 --- a/Orange/widgets/evaluate/tests/test_utils.py +++ b/Orange/widgets/evaluate/tests/test_utils.py @@ -21,23 +21,23 @@ class TestUsableScorers(unittest.TestCase): def setUp(self): self.iris = Table("iris") self.housing = Table("housing") - self.registered_scorers = {scorer.name for scorer in scoring.Score.registry.values()} + self.registered_scorers = set(scoring.Score.registry.values()) def validate_scorer_candidates(self, scorers, class_type): - built_in_scorers = set(BUILTIN_SCORERS_ORDER[class_type]) - # scorer candidates are not all registered scorers - self.assertNotEqual(scorers, self.registered_scorers) - # scorer candidates are subset of registered scorers - self.assertTrue(scorers.issubset(self.registered_scorers)) - # builtins scorers are in fact a subset of valid candidates - self.assertTrue(built_in_scorers.issubset(scorers)) + # scorer candidates are (a proper) subset of registered scorers + self.assertTrue(set(scorers) < self.registered_scorers) + # all scorers are adequate + self.assertTrue(all(class_type in scorer.class_types + for scorer in scorers)) + # scorers are sorted + self.assertTrue(all(s1.priority <= s2.priority + for s1, s2 in zip(scorers, scorers[1:]))) def test_usable_scores(self): - classification_scorers = {scorer.name for scorer in usable_scorers(self.iris.domain)} - regression_scorers = {scorer.name for scorer in usable_scorers(self.housing.domain)} - - self.validate_scorer_candidates(classification_scorers, class_type=DiscreteVariable) - self.validate_scorer_candidates(regression_scorers, class_type=ContinuousVariable) + self.validate_scorer_candidates( + usable_scorers(self.iris.domain), class_type=DiscreteVariable) + self.validate_scorer_candidates( + usable_scorers(self.housing.domain), class_type=ContinuousVariable) class TestScoreTable(GuiTest): diff --git a/Orange/widgets/evaluate/utils.py b/Orange/widgets/evaluate/utils.py index 6584fb0a8a3..5ed22ec8f9b 100644 --- a/Orange/widgets/evaluate/utils.py +++ b/Orange/widgets/evaluate/utils.py @@ -1,6 +1,7 @@ import warnings from functools import partial from itertools import chain +from operator import attrgetter from typing import Union import numpy as np @@ -84,17 +85,12 @@ def usable_scorers(domain_or_var: Union[Variable, Domain]): if domain_or_var is None: return [] - order = {name: i - for i, name in enumerate(chain.from_iterable(BUILTIN_SCORERS_ORDER.values()))} - # 'abstract' is retrieved from __dict__ to avoid inheriting - scorer_candidates = [cls for cls in scoring.Score.registry.values() - if cls.is_scalar and not cls.__dict__.get("abstract")] - - usable = [scorer for scorer in scorer_candidates if - scorer.is_compatible(domain_or_var) and scorer.class_types] - - return sorted(usable, key=lambda cls: order.get(cls.name, 99)) + candidates = [ + scorer for scorer in scoring.Score.registry.values() + if scorer.is_scalar and not scorer.__dict__.get("abstract") + and scorer.is_compatible(domain_or_var) and scorer.class_types] + return sorted(candidates, key=attrgetter("priority")) def scorer_caller(scorer, ovr_results, target=None): From 73bd57d12007d704f12f54114bf7a0d2e01c7bce Mon Sep 17 00:00:00 2001 From: janezd Date: Fri, 6 Jan 2023 21:09:02 +0100 Subject: [PATCH 2/6] ScoreTable: Improve selection of scorers --- Orange/widgets/evaluate/owpredictions.py | 8 ++ Orange/widgets/evaluate/owtestandscore.py | 7 +- .../evaluate/tests/test_owpredictions.py | 5 + .../evaluate/tests/test_owtestandscore.py | 60 ++++++------ Orange/widgets/evaluate/tests/test_utils.py | 94 ++++++++++-------- Orange/widgets/evaluate/utils.py | 96 +++++++++---------- 6 files changed, 152 insertions(+), 118 deletions(-) diff --git a/Orange/widgets/evaluate/owpredictions.py b/Orange/widgets/evaluate/owpredictions.py index 273e31e1654..b9422246ab6 100644 --- a/Orange/widgets/evaluate/owpredictions.py +++ b/Orange/widgets/evaluate/owpredictions.py @@ -63,6 +63,8 @@ class OWPredictions(OWWidget): description = "Display predictions of models for an input dataset." keywords = [] + settings_version = 2 + want_control_area = False class Inputs: @@ -941,6 +943,12 @@ def showEvent(self, event): super().showEvent(event) QTimer.singleShot(0, self._update_splitter) + @classmethod + def migrate_settings(cls, settings, version): + if version < 2: + if "score_table" in settings: + ScoreTable.migrate_to_show_scores_hints(settings["score_table"]) + class ItemDelegate(TableDataDelegate): def initStyleOption(self, option, index): diff --git a/Orange/widgets/evaluate/owtestandscore.py b/Orange/widgets/evaluate/owtestandscore.py index 99d99d4a11f..96bd16753fa 100644 --- a/Orange/widgets/evaluate/owtestandscore.py +++ b/Orange/widgets/evaluate/owtestandscore.py @@ -150,7 +150,7 @@ class Outputs: predictions = Output("Predictions", Table) evaluations_results = Output("Evaluation Results", Results) - settings_version = 3 + settings_version = 4 buttons_area_orientation = None UserAdviceMessages = [ widget.Message( @@ -655,7 +655,7 @@ def update_stats_model(self): item.setData(float(stat.value[0]), Qt.DisplayRole) else: item.setToolTip(str(stat.exception)) - if scorer.name in self.score_table.shown_scores: + if self.score_table.show_score_hints[scorer.__name__]: has_missing_scores = True row.append(item) @@ -899,6 +899,9 @@ def migrate_settings(cls, settings_, version): settings_["context_settings"] = [ c for c in settings_.get("context_settings", ()) if not hasattr(c, 'classes')] + if version < 4: + if "score_table" in settings_: + ScoreTable.migrate_to_show_scores_hints(settings_["score_table"]) @Slot(float) def setProgressValue(self, value): diff --git a/Orange/widgets/evaluate/tests/test_owpredictions.py b/Orange/widgets/evaluate/tests/test_owpredictions.py index ee9d13dd339..fcb5ecfef9c 100644 --- a/Orange/widgets/evaluate/tests/test_owpredictions.py +++ b/Orange/widgets/evaluate/tests/test_owpredictions.py @@ -1168,6 +1168,11 @@ def test_report(self): widget.send_report() self.assertIn(value, widget.report_paragraph.call_args[0][1]) + def test_migrate_shown_scores(self): + settings = {"score_table": {"shown_scores": {"Sensitivity"}}} + self.widget.migrate_settings(settings, 1) + self.assertTrue(settings["score_table"]["show_score_hints"]["Sensitivity"]) + class SelectionModelTest(unittest.TestCase): def setUp(self): diff --git a/Orange/widgets/evaluate/tests/test_owtestandscore.py b/Orange/widgets/evaluate/tests/test_owtestandscore.py index 1897737fbef..defac18bca4 100644 --- a/Orange/widgets/evaluate/tests/test_owtestandscore.py +++ b/Orange/widgets/evaluate/tests/test_owtestandscore.py @@ -21,7 +21,6 @@ from Orange.regression import MeanLearner from Orange.widgets.evaluate.owtestandscore import ( OWTestAndScore, results_one_vs_rest) -from Orange.widgets.evaluate.utils import BUILTIN_SCORERS_ORDER from Orange.widgets.settings import ( ClassValuesContextHandler, PerfectDomainContextHandler) from Orange.widgets.tests.base import WidgetTest @@ -154,6 +153,11 @@ def test_migrate_removes_invalid_contexts(self): self.widget.migrate_settings(settings, 2) self.assertEqual(settings['context_settings'], [context_valid]) + def test_migrate_shown_scores(self): + settings = {"score_table": {"shown_scores": {"Sensitivity"}}} + self.widget.migrate_settings(settings, 3) + self.assertTrue(settings["score_table"]["show_score_hints"]["Sensitivity"]) + def test_memory_error(self): """ Handling memory error. @@ -225,38 +229,39 @@ def test_addon_scorers(self): # These classes are registered, pylint: disable=unused-variable class NewScore(Score): class_types = (DiscreteVariable, ContinuousVariable) + name = "new scorer" @staticmethod def is_compatible(domain: Domain) -> bool: return True class NewClassificationScore(ClassificationScore): - pass + name = "new classification scorer" + default_visible = False class NewRegressionScore(RegressionScore): pass - builtins = BUILTIN_SCORERS_ORDER - self.send_signal("Data", Table("iris")) - scorer_names = [scorer.name for scorer in self.widget.scorers] - self.assertEqual( - tuple(scorer_names[:len(builtins[DiscreteVariable])]), - builtins[DiscreteVariable]) - self.assertIn("NewScore", scorer_names) - self.assertIn("NewClassificationScore", scorer_names) + widget = self.create_widget(OWTestAndScore) + header = widget.score_table.view.horizontalHeader() + self.send_signal(widget.Inputs.train_data, Table("iris")) + scorer_names = [scorer.name for scorer in widget.scorers] + self.assertIn("new scorer", scorer_names) + self.assertFalse(header.isSectionHidden(3 + scorer_names.index("new scorer"))) + self.assertIn("new classification scorer", scorer_names) + self.assertTrue(header.isSectionHidden(3 + scorer_names.index("new classification scorer"))) self.assertNotIn("NewRegressionScore", scorer_names) + model = widget.score_table.model + - self.send_signal("Data", Table("housing")) - scorer_names = [scorer.name for scorer in self.widget.scorers] - self.assertEqual( - tuple(scorer_names[:len(builtins[ContinuousVariable])]), - builtins[ContinuousVariable]) - self.assertIn("NewScore", scorer_names) - self.assertNotIn("NewClassificationScore", scorer_names) + self.send_signal(widget.Inputs.train_data, Table("housing")) + scorer_names = [scorer.name for scorer in widget.scorers] + self.assertIn("new scorer", scorer_names) + self.assertNotIn("new classification scorer", scorer_names) self.assertIn("NewRegressionScore", scorer_names) - self.send_signal("Data", None) - self.assertEqual(self.widget.scorers, []) + self.send_signal(widget.Inputs.train_data, None) + self.assertEqual(widget.scorers, []) finally: del Score.registry["NewScore"] # pylint: disable=no-member del Score.registry["NewClassificationScore"] # pylint: disable=no-member @@ -752,14 +757,15 @@ def compute_score(self, results): mock_learner = Mock(spec=Learner, return_value=mock_model) mock_learner.name = 'Mockery' - self.widget.resampling = OWTestAndScore.TestOnTrain - self.send_signal(self.widget.Inputs.train_data, data) - self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0) - self.send_signal(self.widget.Inputs.learner, mock_learner, 1) - _ = self.get_output(self.widget.Outputs.evaluations_results, wait=5000) - self.assertTrue(len(self.widget.scorers) == 1) - self.assertTrue(NewScorer in self.widget.scorers) - self.assertTrue(len(self.widget._successful_slots()) == 1) + widget = self.create_widget(OWTestAndScore) + widget.resampling = OWTestAndScore.TestOnTrain + self.send_signal(widget.Inputs.train_data, data) + self.send_signal(widget.Inputs.learner, MajorityLearner(), 0) + self.send_signal(widget.Inputs.learner, mock_learner, 1) + _ = self.get_output(widget.Outputs.evaluations_results, wait=5000) + self.assertTrue(len(widget.scorers) == 1) + self.assertTrue(NewScorer in widget.scorers) + self.assertTrue(len(widget._successful_slots()) == 1) class TestHelpers(unittest.TestCase): diff --git a/Orange/widgets/evaluate/tests/test_utils.py b/Orange/widgets/evaluate/tests/test_utils.py index 13eb53f5525..76b04afccc9 100644 --- a/Orange/widgets/evaluate/tests/test_utils.py +++ b/Orange/widgets/evaluate/tests/test_utils.py @@ -3,6 +3,7 @@ import unittest import collections from distutils.version import LooseVersion +from unittest.mock import patch import numpy as np @@ -11,7 +12,8 @@ from AnyQt.QtCore import QPoint, Qt import Orange -from Orange.widgets.evaluate.utils import ScoreTable, usable_scorers, BUILTIN_SCORERS_ORDER +from Orange.evaluation.scoring import Score, RMSE, AUC, CA, F1, Specificity +from Orange.widgets.evaluate.utils import ScoreTable, usable_scorers from Orange.widgets.tests.base import GuiTest from Orange.data import Table, DiscreteVariable, ContinuousVariable from Orange.evaluation import scoring @@ -41,15 +43,26 @@ def test_usable_scores(self): class TestScoreTable(GuiTest): - def test_show_column_chooser(self): - score_table = ScoreTable(None) - view = score_table.view - all, shown = "MABDEFG", "ABDF" - header = view.horizontalHeader() - score_table.shown_scores = set(shown) - score_table.model.setHorizontalHeaderLabels(list(all)) - score_table._update_shown_columns() + def setUp(self): + class NewScore(Score): + name = "new score" + + self.orig_hints = ScoreTable.show_score_hints + hints = ScoreTable.show_score_hints = self.orig_hints.default.copy() + hints.update(dict(F1=True, CA=False, AUC=True, Recall=True, + Specificity=False, NewScore=True)) + self.name_to_qualname = {score.name: score.__name__ + for score in Score.registry.values()} + self.score_table = ScoreTable(None) + self.score_table.update_header([F1, CA, AUC, Specificity, NewScore]) + self.score_table._update_shown_columns() + + def tearDown(self): + ScoreTable.show_score_hints = self.orig_hints + del Score.registry["NewScore"] + def test_show_column_chooser(self): + hints = ScoreTable.show_score_hints actions = collections.OrderedDict() menu_add_action = QMenu.addAction @@ -59,17 +72,27 @@ def addAction(menu, a): return action def execmenu(*_): - self.assertEqual(list(actions), list(all)[1:]) - for name, action in actions.items(): - self.assertEqual(action.isChecked(), name in shown) - actions["E"].triggered.emit(True) - self.assertEqual(score_table.shown_scores, set("ABDEF")) - actions["B"].triggered.emit(False) - self.assertEqual(score_table.shown_scores, set("ADEF")) - for i, name in enumerate(all): - self.assertEqual(name == "M" or name in "ADEF", - not header.isSectionHidden(i), - msg="error in section {}({})".format(i, name)) + scores = ["F1", "CA", "AUC", "Specificity", "new score"] + self.assertEqual(list(actions)[2:], scores) + header = self.score_table.view.horizontalHeader() + for i, (name, action) in enumerate(actions.items()): + if i >= 2: + self.assertEqual(action.isChecked(), + hints[self.name_to_qualname[name]], + msg=f"error in section {name}") + self.assertEqual(header.isSectionHidden(i), + hints[self.name_to_qualname[name]], + msg=f"error in section {name}") + actions["CA"].triggered.emit(True) + hints["CA"] = True + for k, v in hints.items(): + self.assertEqual(self.score_table.show_score_hints[k], v, + msg=f"error at {k}") + actions["AUC"].triggered.emit(False) + hints["AUC"] = False + for k, v in hints.items(): + self.assertEqual(self.score_table.show_score_hints[k], v, + msg=f"error at {k}") # We must patch `QMenu.exec` because the Qt would otherwise (invisibly) # show the popup and wait for the user. @@ -78,27 +101,7 @@ def execmenu(*_): # `menuexec` finishes. with unittest.mock.patch("AnyQt.QtWidgets.QMenu.addAction", addAction), \ unittest.mock.patch("AnyQt.QtWidgets.QMenu.exec", execmenu): - score_table.show_column_chooser(QPoint(0, 0)) - - def test_update_shown_columns(self): - score_table = ScoreTable(None) - view = score_table.view - all, shown = "MABDEFG", "ABDF" - header = view.horizontalHeader() - score_table.shown_scores = set(shown) - score_table.model.setHorizontalHeaderLabels(list(all)) - score_table._update_shown_columns() - for i, name in enumerate(all): - self.assertEqual(name == "M" or name in shown, - not header.isSectionHidden(i), - msg="error in section {}({})".format(i, name)) - - score_table.shown_scores = set() - score_table._update_shown_columns() - for i, name in enumerate(all): - self.assertEqual(i == 0, - not header.isSectionHidden(i), - msg="error in section {}({})".format(i, name)) + self.score_table.show_column_chooser(QPoint(0, 0)) def test_sorting(self): def order(n=5): @@ -142,6 +145,15 @@ def order(n=5): model.sort(2, Qt.DescendingOrder) self.assertEqual(order(3), "DEC") + def test_shown_scores_backward_compatibility(self): + self.assertEqual(self.score_table.shown_scores, + {"F1", "AUC", "new score"}) + + def test_migration(self): + settings = dict(foo=False, shown_scores={"Sensitivity"}) + ScoreTable.migrate_to_show_scores_hints(settings) + self.assertTrue(settings["show_score_hints"]["Sensitivity"]) + def test_column_settings_reminder(self): if LooseVersion(Orange.__version__) >= LooseVersion("3.37"): self.fail( diff --git a/Orange/widgets/evaluate/utils.py b/Orange/widgets/evaluate/utils.py index 5ed22ec8f9b..7a0183ea4cd 100644 --- a/Orange/widgets/evaluate/utils.py +++ b/Orange/widgets/evaluate/utils.py @@ -1,8 +1,6 @@ import warnings -from functools import partial -from itertools import chain from operator import attrgetter -from typing import Union +from typing import Union, Dict, List import numpy as np @@ -13,13 +11,13 @@ QSortFilterProxyModel from sklearn.exceptions import UndefinedMetricWarning -from Orange.data import DiscreteVariable, ContinuousVariable, Domain, Variable +from Orange.data import Domain, Variable from Orange.evaluation import scoring +from Orange.evaluation.scoring import Score from Orange.widgets import gui from Orange.widgets.utils.tableview import table_selection_to_mime_data from Orange.widgets.gui import OWComponent from Orange.widgets.settings import Setting -from Orange.util import OrangeDeprecationWarning def check_results_adequacy(results, error_group, check_nan=True): @@ -70,11 +68,6 @@ def results_for_preview(data_name=""): return results -BUILTIN_SCORERS_ORDER = { - DiscreteVariable: ("AUC", "CA", "F1", "Precision", "Recall"), - ContinuousVariable: ("MSE", "RMSE", "MAE", "R2")} - - def learner_name(learner): """Return the value of `learner.name` if it exists, or the learner's type name otherwise""" @@ -135,10 +128,22 @@ def is_bad(x): return left < right +DEFAULT_HINTS = {"Model_": True, "Train": False, "Test": False} + + class ScoreTable(OWComponent, QObject): - shown_scores = Setting(set(chain(*BUILTIN_SCORERS_ORDER.values()))) + show_score_hints: Dict[str, bool] = Setting(DEFAULT_HINTS) shownScoresChanged = Signal() + # backwards compatibility + @property + def shown_scores(self): + column_names = { + self.model.horizontalHeaderItem(col).data(Qt.DisplayRole) + for col in range(1, self.model.columnCount())} + return column_names & {score.name for score in Score.registry.values() + if self.show_score_hints[score.__name__]} + class ItemDelegate(QStyledItemDelegate): def sizeHint(self, *args): size = super().sizeHint(*args) @@ -164,17 +169,8 @@ def __init__(self, master): header.setContextMenuPolicy(Qt.CustomContextMenu) header.customContextMenuRequested.connect(self.show_column_chooser) - # Currently, this component will never show scoring methods - # defined in add-ons by default. To support them properly, the - # "shown_scores" settings will need to be reworked. - # The following is a temporary solution to show the scoring method - # for survival data (it does not influence other problem types). - # It is added here so that the "C-Index" method - # will show up even if the users already have the setting defined. - # This temporary fix is here due to a paper deadline needing the feature. - # When removing, also remove TestScoreTable.test_column_settings_reminder - if isinstance(self.shown_scores, set): # TestScoreTable does not initialize settings - self.shown_scores.add("C-Index") + for score in Score.registry.values(): + self.show_score_hints.setdefault(score.__name__, score.default_visible) self.model = QStandardItemModel(master) self.model.setHorizontalHeaderLabels(["Method"]) @@ -183,46 +179,42 @@ def __init__(self, master): self.view.setModel(self.sorted_model) self.view.setItemDelegate(self.ItemDelegate()) - def _column_names(self): - return (self.model.horizontalHeaderItem(section).data(Qt.DisplayRole) - for section in range(1, self.model.columnCount())) - def show_column_chooser(self, pos): - # pylint doesn't know that self.shown_scores is a set, not a Setting - # pylint: disable=unsupported-membership-test - def update(col_name, checked): - if checked: - self.shown_scores.add(col_name) - else: - self.shown_scores.remove(col_name) - self._update_shown_columns() - menu = QMenu() header = self.view.horizontalHeader() - for col_name in self._column_names(): - action = menu.addAction(col_name) + for col in range(1, self.model.columnCount()): + item = self.model.horizontalHeaderItem(col) + action = menu.addAction(item.data(Qt.DisplayRole)) action.setCheckable(True) - action.setChecked(col_name in self.shown_scores) - action.triggered.connect(partial(update, col_name)) + qualname = item.data(Qt.UserRole) + action.setChecked(self.show_score_hints[qualname]) + + @action.triggered.connect + def update(checked, q=qualname): + self.show_score_hints[q] = checked + self._update_shown_columns() + menu.exec(header.mapToGlobal(pos)) def _update_shown_columns(self): - # pylint doesn't know that self.shown_scores is a set, not a Setting - # pylint: disable=unsupported-membership-test self.view.resizeColumnsToContents() header = self.view.horizontalHeader() - for section, col_name in enumerate(self._column_names(), start=1): - header.setSectionHidden(section, col_name not in self.shown_scores) + for section in range(1, header.count()): + qualname = self.model.horizontalHeaderItem(section).data(Qt.UserRole) + header.setSectionHidden(section, not self.show_score_hints[qualname]) self.shownScoresChanged.emit() - def update_header(self, scorers): - # Set the correct horizontal header labels on the results_model. + def update_header(self, scorers: List[Score]): self.model.setColumnCount(3 + len(scorers)) - self.model.setHorizontalHeaderItem(0, QStandardItem("Model")) - self.model.setHorizontalHeaderItem(1, QStandardItem("Train time [s]")) - self.model.setHorizontalHeaderItem(2, QStandardItem("Test time [s]")) + for i, name, id_ in ((0, "Model", "Model_"), + (1, "Train time [s]", "Train"), + (2, "Test time [s]", "Test")): + item = QStandardItem(name) + item.setData(id_, Qt.UserRole) + self.model.setHorizontalHeaderItem(i, item) for col, score in enumerate(scorers, start=3): item = QStandardItem(score.name) + item.setData(score.__name__, Qt.UserRole) item.setToolTip(score.long_name) self.model.setHorizontalHeaderItem(col, item) self._update_shown_columns() @@ -232,3 +224,11 @@ def copy_selection_to_clipboard(self): QApplication.clipboard().setMimeData( mime, QClipboard.Clipboard ) + + @staticmethod + def migrate_to_show_scores_hints(settings): + # Migration cannot disable anything because it can't know which score + # have been present when the setting was created. + settings["show_score_hints"] = DEFAULT_HINTS.copy() + settings["show_score_hints"].update( + dict.fromkeys(settings["shown_scores"], True)) From 6d5f0ac429469ec943090da1845bd08a5ce7ed54 Mon Sep 17 00:00:00 2001 From: janezd Date: Fri, 6 Jan 2023 21:37:08 +0100 Subject: [PATCH 3/6] Score: Name scorers and use names in ScoreTable --- Orange/evaluation/scoring.py | 16 ++++++++++ Orange/widgets/evaluate/owtestandscore.py | 1 + Orange/widgets/evaluate/tests/test_utils.py | 34 ++++++++++++--------- Orange/widgets/evaluate/utils.py | 11 +++++-- 4 files changed, 46 insertions(+), 16 deletions(-) diff --git a/Orange/evaluation/scoring.py b/Orange/evaluation/scoring.py index 062abae79e4..5fe943551b6 100644 --- a/Orange/evaluation/scoring.py +++ b/Orange/evaluation/scoring.py @@ -37,6 +37,7 @@ def __new__(mcs, name, bases, dict_, **kwargs): if not kwargs.get("abstract"): # Don't use inherited names, look into dict_ cls.name = dict_.get("name", name) + cls.long_name = dict_.get("long_name", cls.name) cls.registry[name] = cls else: cls.registry = {} @@ -139,6 +140,7 @@ def is_compatible(domain: Domain) -> bool: # pylint: disable=invalid-name class CA(ClassificationScore): __wraps__ = skl_metrics.accuracy_score + name = "CA" long_name = "Classification accuracy" priority = 20 @@ -189,16 +191,20 @@ def compute_score(self, results, target=None, average='binary'): class Precision(TargetScore): __wraps__ = skl_metrics.precision_score + name = "Prec" + long_name = "Precision" priority = 40 class Recall(TargetScore): __wraps__ = skl_metrics.recall_score + name = long_name = "Recall" priority = 50 class F1(TargetScore): __wraps__ = skl_metrics.f1_score + name = long_name = "F1" priority = 30 @@ -217,6 +223,7 @@ class AUC(ClassificationScore): __wraps__ = skl_metrics.roc_auc_score separate_folds = True is_binary = True + name = "AUC" long_name = "Area under ROC curve" priority = 10 @@ -291,6 +298,8 @@ class LogLoss(ClassificationScore): """ __wraps__ = skl_metrics.log_loss priority = 120 + name = "LogLoss" + long_name = "Logistic loss" default_visible = False def compute_score(self, results, eps=1e-15, normalize=True, @@ -308,6 +317,8 @@ def compute_score(self, results, eps=1e-15, normalize=True, class Specificity(ClassificationScore): is_binary = True priority = 110 + name = "Spec" + long_name = "Specificity" default_visible = False @staticmethod @@ -361,11 +372,13 @@ def compute_score(self, results, target=None, average="binary"): class MSE(RegressionScore): __wraps__ = skl_metrics.mean_squared_error + name = "MSE" long_name = "Mean square error" priority = 20 class RMSE(RegressionScore): + name = "RMSE" long_name = "Root mean square error" def compute_score(self, results): @@ -375,6 +388,7 @@ def compute_score(self, results): class MAE(RegressionScore): __wraps__ = skl_metrics.mean_absolute_error + name = "MAE" long_name = "Mean absolute error" priority = 40 @@ -382,11 +396,13 @@ class MAE(RegressionScore): # pylint: disable=invalid-name class R2(RegressionScore): __wraps__ = skl_metrics.r2_score + name = "R2" long_name = "Coefficient of determination" priority = 50 class CVRMSE(RegressionScore): + name = "CVRMSE" long_name = "Coefficient of variation of the RMSE" priority = 110 default_visible = False diff --git a/Orange/widgets/evaluate/owtestandscore.py b/Orange/widgets/evaluate/owtestandscore.py index 96bd16753fa..26d482df59a 100644 --- a/Orange/widgets/evaluate/owtestandscore.py +++ b/Orange/widgets/evaluate/owtestandscore.py @@ -655,6 +655,7 @@ def update_stats_model(self): item.setData(float(stat.value[0]), Qt.DisplayRole) else: item.setToolTip(str(stat.exception)) + # pylint: disable=unsubscriptable-object if self.score_table.show_score_hints[scorer.__name__]: has_missing_scores = True row.append(item) diff --git a/Orange/widgets/evaluate/tests/test_utils.py b/Orange/widgets/evaluate/tests/test_utils.py index 76b04afccc9..b320ea7c15d 100644 --- a/Orange/widgets/evaluate/tests/test_utils.py +++ b/Orange/widgets/evaluate/tests/test_utils.py @@ -3,6 +3,7 @@ import unittest import collections from distutils.version import LooseVersion +from itertools import count from unittest.mock import patch import numpy as np @@ -12,7 +13,7 @@ from AnyQt.QtCore import QPoint, Qt import Orange -from Orange.evaluation.scoring import Score, RMSE, AUC, CA, F1, Specificity +from Orange.evaluation.scoring import Score, AUC, CA, F1, Specificity from Orange.widgets.evaluate.utils import ScoreTable, usable_scorers from Orange.widgets.tests.base import GuiTest from Orange.data import Table, DiscreteVariable, ContinuousVariable @@ -47,12 +48,12 @@ def setUp(self): class NewScore(Score): name = "new score" + self.NewScore = NewScore # pylint: disable=invalid-name + self.orig_hints = ScoreTable.show_score_hints hints = ScoreTable.show_score_hints = self.orig_hints.default.copy() hints.update(dict(F1=True, CA=False, AUC=True, Recall=True, Specificity=False, NewScore=True)) - self.name_to_qualname = {score.name: score.__name__ - for score in Score.registry.values()} self.score_table = ScoreTable(None) self.score_table.update_header([F1, CA, AUC, Specificity, NewScore]) self.score_table._update_shown_columns() @@ -72,23 +73,28 @@ def addAction(menu, a): return action def execmenu(*_): - scores = ["F1", "CA", "AUC", "Specificity", "new score"] - self.assertEqual(list(actions)[2:], scores) + # pylint: disable=unsubscriptable-object,unsupported-assignment-operation + scorers = [F1, CA, AUC, Specificity, self.NewScore] + self.assertEqual(list(actions)[2:], ['F1', + 'Classification accuracy (CA)', + 'Area under ROC curve (AUC)', + 'Specificity (Spec)', + 'new score']) header = self.score_table.view.horizontalHeader() - for i, (name, action) in enumerate(actions.items()): + for i, action, scorer in zip(count(), actions.values(), scorers): if i >= 2: self.assertEqual(action.isChecked(), - hints[self.name_to_qualname[name]], - msg=f"error in section {name}") + hints[scorer.__name__], + msg=f"error in section {scorer.name}") self.assertEqual(header.isSectionHidden(i), - hints[self.name_to_qualname[name]], - msg=f"error in section {name}") - actions["CA"].triggered.emit(True) + hints[scorer.__name__], + msg=f"error in section {scorer.name}") + actions["Classification accuracy (CA)"].triggered.emit(True) hints["CA"] = True for k, v in hints.items(): self.assertEqual(self.score_table.show_score_hints[k], v, msg=f"error at {k}") - actions["AUC"].triggered.emit(False) + actions["Area under ROC curve (AUC)"].triggered.emit(False) hints["AUC"] = False for k, v in hints.items(): self.assertEqual(self.score_table.show_score_hints[k], v, @@ -99,8 +105,8 @@ def execmenu(*_): # Assertions are made within `menuexec` since they check the # instances of `QAction`, which are invalid (destroyed by Qt?) after # `menuexec` finishes. - with unittest.mock.patch("AnyQt.QtWidgets.QMenu.addAction", addAction), \ - unittest.mock.patch("AnyQt.QtWidgets.QMenu.exec", execmenu): + with patch("AnyQt.QtWidgets.QMenu.addAction", addAction), \ + patch("AnyQt.QtWidgets.QMenu.exec", execmenu): self.score_table.show_column_chooser(QPoint(0, 0)) def test_sorting(self): diff --git a/Orange/widgets/evaluate/utils.py b/Orange/widgets/evaluate/utils.py index 7a0183ea4cd..e98b83865ac 100644 --- a/Orange/widgets/evaluate/utils.py +++ b/Orange/widgets/evaluate/utils.py @@ -184,9 +184,16 @@ def show_column_chooser(self, pos): header = self.view.horizontalHeader() for col in range(1, self.model.columnCount()): item = self.model.horizontalHeaderItem(col) - action = menu.addAction(item.data(Qt.DisplayRole)) - action.setCheckable(True) qualname = item.data(Qt.UserRole) + if col < 3: + option = item.data(Qt.DisplayRole) + else: + score = Score.registry[qualname] + option = score.long_name + if score.name != score.long_name: + option += f" ({score.name})" + action = menu.addAction(option) + action.setCheckable(True) action.setChecked(self.show_score_hints[qualname]) @action.triggered.connect From 8aaa7086cccc7c69b8930b19388c12d92b523fb2 Mon Sep 17 00:00:00 2001 From: janezd Date: Fri, 6 Jan 2023 23:36:54 +0100 Subject: [PATCH 4/6] ScoreTable: Add indicator to select scores --- .../evaluate/tests/test_owtestandscore.py | 8 +- Orange/widgets/evaluate/tests/test_utils.py | 20 ++- Orange/widgets/evaluate/utils.py | 150 ++++++++++++------ 3 files changed, 119 insertions(+), 59 deletions(-) diff --git a/Orange/widgets/evaluate/tests/test_owtestandscore.py b/Orange/widgets/evaluate/tests/test_owtestandscore.py index defac18bca4..4f788d371a9 100644 --- a/Orange/widgets/evaluate/tests/test_owtestandscore.py +++ b/Orange/widgets/evaluate/tests/test_owtestandscore.py @@ -325,7 +325,7 @@ def __call__(self, data): header = view.horizontalHeader() p = header.rect().center() # second visible header section (after 'Model') - _, idx, *_ = (i for i in range(header.count()) + _, _, idx, *_ = (i for i in range(header.count()) if not header.isSectionHidden(i)) p.setX(header.sectionPosition(idx) + 5) QTest.mouseClick(header.viewport(), Qt.LeftButton, pos=p) @@ -723,11 +723,13 @@ def test_copy_to_clipboard(self): selection_model = view.selectionModel() selection_model.select(model.index(0, 0), selection_model.Select | selection_model.Rows) - self.widget.copy_to_clipboard() clipboard_text = QApplication.clipboard().text() + # Tests appear to register additional scorers, so we clip the list + # to what we know to be there and visible + clipboard_text = "\t".join(clipboard_text.split("\t")[:6]).strip() view_text = "\t".join([str(model.data(model.index(0, i))) - for i in (0, 3, 4, 5, 6, 7)]) + "\r\n" + for i in (0, 3, 4, 5, 6, 7)]).strip() self.assertEqual(clipboard_text, view_text) def test_multi_target_input(self): diff --git a/Orange/widgets/evaluate/tests/test_utils.py b/Orange/widgets/evaluate/tests/test_utils.py index b320ea7c15d..3408ea5b910 100644 --- a/Orange/widgets/evaluate/tests/test_utils.py +++ b/Orange/widgets/evaluate/tests/test_utils.py @@ -56,7 +56,6 @@ class NewScore(Score): Specificity=False, NewScore=True)) self.score_table = ScoreTable(None) self.score_table.update_header([F1, CA, AUC, Specificity, NewScore]) - self.score_table._update_shown_columns() def tearDown(self): ScoreTable.show_score_hints = self.orig_hints @@ -75,20 +74,19 @@ def addAction(menu, a): def execmenu(*_): # pylint: disable=unsubscriptable-object,unsupported-assignment-operation scorers = [F1, CA, AUC, Specificity, self.NewScore] - self.assertEqual(list(actions)[2:], ['F1', + self.assertEqual(list(actions)[3:], ['F1', 'Classification accuracy (CA)', 'Area under ROC curve (AUC)', 'Specificity (Spec)', 'new score']) header = self.score_table.view.horizontalHeader() - for i, action, scorer in zip(count(), actions.values(), scorers): - if i >= 2: - self.assertEqual(action.isChecked(), - hints[scorer.__name__], - msg=f"error in section {scorer.name}") - self.assertEqual(header.isSectionHidden(i), - hints[scorer.__name__], - msg=f"error in section {scorer.name}") + for i, action, scorer in zip(count(), list(actions.values())[3:], scorers): + self.assertEqual(action.isChecked(), + hints[scorer.__name__], + msg=f"error in section {scorer.name}") + self.assertEqual(header.isSectionHidden(3 + i), + not hints[scorer.__name__], + msg=f"error in section {scorer.name}") actions["Classification accuracy (CA)"].triggered.emit(True) hints["CA"] = True for k, v in hints.items(): @@ -107,7 +105,7 @@ def execmenu(*_): # `menuexec` finishes. with patch("AnyQt.QtWidgets.QMenu.addAction", addAction), \ patch("AnyQt.QtWidgets.QMenu.exec", execmenu): - self.score_table.show_column_chooser(QPoint(0, 0)) + self.score_table.view.horizontalHeader().show_column_chooser(QPoint(0, 0)) def test_sorting(self): def order(n=5): diff --git a/Orange/widgets/evaluate/utils.py b/Orange/widgets/evaluate/utils.py index e98b83865ac..ed69286ab82 100644 --- a/Orange/widgets/evaluate/utils.py +++ b/Orange/widgets/evaluate/utils.py @@ -3,13 +3,15 @@ from typing import Union, Dict, List import numpy as np +from sklearn.exceptions import UndefinedMetricWarning from AnyQt.QtWidgets import QHeaderView, QStyledItemDelegate, QMenu, \ - QApplication -from AnyQt.QtGui import QStandardItemModel, QStandardItem, QClipboard + QApplication, QToolButton +from AnyQt.QtGui import QStandardItemModel, QStandardItem, QClipboard, QColor from AnyQt.QtCore import Qt, QSize, QObject, pyqtSignal as Signal, \ QSortFilterProxyModel -from sklearn.exceptions import UndefinedMetricWarning + +from orangewidget.gui import OrangeUserRole from Orange.data import Domain, Variable from Orange.evaluation import scoring @@ -128,7 +130,84 @@ def is_bad(x): return left < right -DEFAULT_HINTS = {"Model_": True, "Train": False, "Test": False} +DEFAULT_HINTS = {"Model_": True, "Train_": False, "Test_": False} + + +class PersistentMenu(QMenu): + def mouseReleaseEvent(self, e): + action = self.activeAction() + if action: + action.setEnabled(False) + super().mouseReleaseEvent(e) + action.setEnabled(True) + action.trigger() + else: + super().mouseReleaseEvent(e) + + +class SelectableColumnsHeader(QHeaderView): + SelectMenuRole = next(OrangeUserRole) + ShownHintRole = next(OrangeUserRole) + sectionVisibleChanged = Signal(int, bool) + + def __init__(self, shown_columns_hints, *args, **kwargs): + super().__init__(Qt.Horizontal, *args, **kwargs) + self.show_column_hints = shown_columns_hints + self.button = QToolButton(self) + self.button.setArrowType(Qt.DownArrow) + self.button.setFixedSize(24, 12) + col = self.button.palette().color(self.button.backgroundRole()) + self.button.setStyleSheet( + f"border: none; background-color: {col.name(QColor.NameFormat.HexRgb)}") + self.setContextMenuPolicy(Qt.CustomContextMenu) + self.customContextMenuRequested.connect(self.show_column_chooser) + self.button.clicked.connect(self._on_button_clicked) + + def showEvent(self, e): + self._set_pos() + self.button.show() + super().showEvent(e) + + def resizeEvent(self, e): + self._set_pos() + super().resizeEvent(e) + + def _set_pos(self): + w, h = self.button.width(), self.button.height() + vw, vh = self.viewport().width(), self.viewport().height() + self.button.setGeometry(vw - w, (vh - h) // 2, w, h) + + def __data(self, section, role): + return self.model().headerData(section, Qt.Horizontal, role) + + def show_column_chooser(self, pos): + # pylint: disable=unsubscriptable-object, unsupported-assignment-operation + menu = PersistentMenu() + for section in range(self.count()): + name, enabled = self.__data(section, self.SelectMenuRole) + hint_id = self.__data(section, self.ShownHintRole) + action = menu.addAction(name) + action.setDisabled(not enabled) + action.setCheckable(True) + action.setChecked(self.show_column_hints[hint_id]) + + @action.triggered.connect # pylint: disable=cell-var-from-loop + def update(checked, q=hint_id, section=section): + self.show_column_hints[q] = checked + self.setSectionHidden(section, not checked) + self.sectionVisibleChanged.emit(section, checked) + self.resizeSections(self.ResizeToContents) + + pos.setY(self.viewport().height()) + menu.exec(self.mapToGlobal(pos)) + + def _on_button_clicked(self): + self.show_column_chooser(self.button.pos()) + + def update_shown_columns(self): + for section in range(self.count()): + hint_id = self.__data(section, self.ShownHintRole) + self.setSectionHidden(section, not self.show_column_hints[hint_id]) class ScoreTable(OWComponent, QObject): @@ -138,6 +217,7 @@ class ScoreTable(OWComponent, QObject): # backwards compatibility @property def shown_scores(self): + # pylint: disable=unsubscriptable-object column_names = { self.model.horizontalHeaderItem(col).data(Qt.DisplayRole) for col in range(1, self.model.columnCount())} @@ -166,65 +246,45 @@ def __init__(self, master): header.setSectionResizeMode(QHeaderView.ResizeToContents) header.setDefaultAlignment(Qt.AlignCenter) header.setStretchLastSection(False) - header.setContextMenuPolicy(Qt.CustomContextMenu) - header.customContextMenuRequested.connect(self.show_column_chooser) for score in Score.registry.values(): self.show_score_hints.setdefault(score.__name__, score.default_visible) self.model = QStandardItemModel(master) - self.model.setHorizontalHeaderLabels(["Method"]) + header = SelectableColumnsHeader(self.show_score_hints) + header.setSectionsClickable(True) + self.view.setHorizontalHeader(header) self.sorted_model = ScoreModel() self.sorted_model.setSourceModel(self.model) self.view.setModel(self.sorted_model) self.view.setItemDelegate(self.ItemDelegate()) - - def show_column_chooser(self, pos): - menu = QMenu() - header = self.view.horizontalHeader() - for col in range(1, self.model.columnCount()): - item = self.model.horizontalHeaderItem(col) - qualname = item.data(Qt.UserRole) - if col < 3: - option = item.data(Qt.DisplayRole) - else: - score = Score.registry[qualname] - option = score.long_name - if score.name != score.long_name: - option += f" ({score.name})" - action = menu.addAction(option) - action.setCheckable(True) - action.setChecked(self.show_score_hints[qualname]) - - @action.triggered.connect - def update(checked, q=qualname): - self.show_score_hints[q] = checked - self._update_shown_columns() - - menu.exec(header.mapToGlobal(pos)) - - def _update_shown_columns(self): - self.view.resizeColumnsToContents() - header = self.view.horizontalHeader() - for section in range(1, header.count()): - qualname = self.model.horizontalHeaderItem(section).data(Qt.UserRole) - header.setSectionHidden(section, not self.show_score_hints[qualname]) - self.shownScoresChanged.emit() + header.sectionVisibleChanged.connect(self.shownScoresChanged.emit) + self.sorted_model.dataChanged.connect(self.view.resizeColumnsToContents) def update_header(self, scorers: List[Score]): self.model.setColumnCount(3 + len(scorers)) - for i, name, id_ in ((0, "Model", "Model_"), - (1, "Train time [s]", "Train"), - (2, "Test time [s]", "Test")): + SelectMenuRole = SelectableColumnsHeader.SelectMenuRole + ShownHintRole = SelectableColumnsHeader.ShownHintRole + for i, name, long_name, id_, in ((0, "Model", "Model", "Model_"), + (1, "Train", "Train time [s]", "Train_"), + (2, "Test", "Test time [s]", "Test_")): item = QStandardItem(name) - item.setData(id_, Qt.UserRole) + item.setData((long_name, i != 0), SelectMenuRole) + item.setData(id_, ShownHintRole) + item.setToolTip(long_name) self.model.setHorizontalHeaderItem(i, item) for col, score in enumerate(scorers, start=3): item = QStandardItem(score.name) - item.setData(score.__name__, Qt.UserRole) + name = score.long_name + if name != score.name: + name += f" ({score.name})" + item.setData((name, True), SelectMenuRole) + item.setData(score.__name__, ShownHintRole) item.setToolTip(score.long_name) self.model.setHorizontalHeaderItem(col, item) - self._update_shown_columns() + + self.view.horizontalHeader().update_shown_columns() + self.view.resizeColumnsToContents() def copy_selection_to_clipboard(self): mime = table_selection_to_mime_data(self.view) From 5d5a44313f626d5962d806dfca41fb3eb663c976 Mon Sep 17 00:00:00 2001 From: Marko Toplak Date: Mon, 9 Jan 2023 14:13:36 +0100 Subject: [PATCH 5/6] Remove deprecation reminder as the feature was just fixed --- Orange/widgets/evaluate/tests/test_utils.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Orange/widgets/evaluate/tests/test_utils.py b/Orange/widgets/evaluate/tests/test_utils.py index 3408ea5b910..3aa7e463f61 100644 --- a/Orange/widgets/evaluate/tests/test_utils.py +++ b/Orange/widgets/evaluate/tests/test_utils.py @@ -158,13 +158,6 @@ def test_migration(self): ScoreTable.migrate_to_show_scores_hints(settings) self.assertTrue(settings["show_score_hints"]["Sensitivity"]) - def test_column_settings_reminder(self): - if LooseVersion(Orange.__version__) >= LooseVersion("3.37"): - self.fail( - "Orange 3.32 added a workaround to show C-Index into ScoreTable.__init__. " - "This should have been properly fixed long ago." - ) - if __name__ == "__main__": unittest.main() From 1df2e31e7d2739af6b5b85143f142e16ad0ef579 Mon Sep 17 00:00:00 2001 From: Marko Toplak Date: Mon, 9 Jan 2023 14:31:42 +0100 Subject: [PATCH 6/6] owtestandscore: hint not needed anymore --- Orange/widgets/evaluate/owtestandscore.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Orange/widgets/evaluate/owtestandscore.py b/Orange/widgets/evaluate/owtestandscore.py index 26d482df59a..f14bbd96f7e 100644 --- a/Orange/widgets/evaluate/owtestandscore.py +++ b/Orange/widgets/evaluate/owtestandscore.py @@ -152,10 +152,6 @@ class Outputs: settings_version = 4 buttons_area_orientation = None - UserAdviceMessages = [ - widget.Message( - "Click on the table header to select shown columns", - "click_header")] settingsHandler = settings.PerfectDomainContextHandler() score_table = settings.SettingProvider(ScoreTable)