From 6037f9e25829c92e4a87f640024e7d026e5b9796 Mon Sep 17 00:00:00 2001 From: ceprio Date: Sat, 7 Oct 2017 21:35:47 -0400 Subject: [PATCH 1/7] Update owsql.py Added functionality to the widget: - The widget now saves the settings of the selected table name and SQL Query - Restoring of table name and SQL query is also now working when opening a workflow - Connection to the database is done earlier to allow the tables list to be fetched - Table name is saved and restored as text (or name of table) instead of Id (as previously) - Empty username and password are now saved as '' (empty string) instead of None. This allows the use of SQL Server's Windows authentication (use of the current user credentials if both fields are empty). - Placing 'Custom SQL' in second in the table list in order to see it faster when the list of tables is long. - Execute SQL query only if minimum number of chars entered --- Orange/widgets/data/owsql.py | 66 ++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/Orange/widgets/data/owsql.py b/Orange/widgets/data/owsql.py index 585017ff8d6..7575d574c3b 100644 --- a/Orange/widgets/data/owsql.py +++ b/Orange/widgets/data/owsql.py @@ -1,3 +1,5 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- import sys from collections import OrderedDict @@ -20,7 +22,7 @@ MAX_DL_LIMIT = 1000000 -class TableModel(PyListModel): +class TableModels(PyListModel): def data(self, index, role=Qt.DisplayRole): row = index.row() if role == Qt.DisplayRole: @@ -28,7 +30,7 @@ def data(self, index, role=Qt.DisplayRole): return super().data(index, role) -class BackendModel(PyListModel): +class BackendModels(PyListModel): def data(self, index, role=Qt.DisplayRole): row = index.row() if role == Qt.DisplayRole: @@ -43,7 +45,7 @@ class OWSql(OWWidget): icon = "icons/SQLTable.svg" priority = 30 category = "Data" - keywords = ["data", "file", "load", "read"] + keywords = ["data", "file", "load", "read", "SQL"] class Outputs: data = Output("Data", Table, doc="Attribute-valued data set read from the input file.") @@ -85,10 +87,10 @@ def __init__(self): vbox = gui.vBox(self.controlArea, "Server", addSpace=True) box = gui.vBox(vbox) - self.backendmodel = BackendModel(Backend.available_backends()) + self.backendmodels = BackendModels(Backend.available_backends()) self.backendcombo = QComboBox(box) - if len(self.backendmodel): - self.backendcombo.setModel(self.backendmodel) + if len(self.backendmodels): + self.backendcombo.setModel(self.backendmodels) else: self.Error.no_backends() box.setEnabled(False) @@ -124,17 +126,23 @@ def __init__(self): box.layout().addWidget(self.passwordtext) self._load_credentials() + self.tablemodels = TableModels() tables = gui.hBox(box) - self.tablemodel = TableModel() self.tablecombo = QComboBox( minimumContentsLength=35, sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLength ) - self.tablecombo.setModel(self.tablemodel) + self.tablecombo.setModel(self.tablemodels) self.tablecombo.setToolTip('table') tables.layout().addWidget(self.tablecombo) - self.tablecombo.activated[int].connect(self.select_table) + self.connect() + + index = self.tablecombo.findText(str(self.table)) + if index != -1: + self.tablecombo.setCurrentIndex(index) + self.tablecombo.activated[int].connect(self.select_table) # set up the callback to select_table in case of selection change + self.connectbutton = gui.button( tables, self, '↻', callback=self.connect) self.connectbutton.setSizePolicy( @@ -167,7 +175,8 @@ def __init__(self): callback=self.open_table) gui.rubber(self.buttonsArea) - QTimer.singleShot(0, self.connect) + + QTimer.singleShot(0, self.select_table) def _load_credentials(self): self._parse_host_port() @@ -182,8 +191,8 @@ def _load_credentials(self): def _save_credentials(self): cm = self._credential_manager(self.host, self.port) - cm.username = self.username - cm.password = self.password + cm.username = self.username or '' + cm.password = self.password or '' def _credential_manager(self, host, port): return CredentialManager("SQL Table: {}:{}".format(host, port)) @@ -217,7 +226,7 @@ def connect(self): try: if self.backendcombo.currentIndex() < 0: return - backend = self.backendmodel[self.backendcombo.currentIndex()] + backend = self.backendmodels[self.backendcombo.currentIndex()] self.backend = backend(dict( host=self.host, port=self.port, @@ -232,7 +241,6 @@ def connect(self): ("Database", self.database), ("User name", self.username) )) self.refresh_tables() - self.select_table() except BackendError as err: error = str(err).split('\n')[0] self.Error.connection(error) @@ -240,17 +248,18 @@ def connect(self): self.tablecombo.clear() def refresh_tables(self): - self.tablemodel.clear() + self.tablemodels.clear() self.Error.missing_extension.clear() if self.backend is None: self.data_desc_table = None return - self.tablemodel.append("Select a table") - self.tablemodel.extend(self.backend.list_tables(self.schema)) - self.tablemodel.append("Custom SQL") + self.tablemodels.append("Select a table") + self.tablemodels.append("Custom SQL") + self.tablemodels.extend(self.backend.list_tables(self.schema)) def select_table(self): + "Called on tablecombo selection change" curIdx = self.tablecombo.currentIndex() if self.tablecombo.itemText(curIdx) != "Custom SQL": self.custom_sql.setVisible(False) @@ -260,6 +269,8 @@ def select_table(self): self.data_desc_table = None self.database_desc["Table"] = "(None)" self.table = None + if len(str(self.sql)) > 14: + return self.open_table() #self.Error.missing_extension( # 's' if len(missing) > 1 else '', @@ -272,19 +283,22 @@ def open_table(self): self.Outputs.data.send(table) def get_table(self): - if self.tablecombo.currentIndex() <= 0: + curIdx = self.tablecombo.currentIndex() + if curIdx <= 0: if self.database_desc: self.database_desc["Table"] = "(None)" self.data_desc_table = None return - if self.tablecombo.currentIndex() < self.tablecombo.count() - 1: - self.table = self.tablemodel[self.tablecombo.currentIndex()] + if self.tablecombo.itemText(curIdx) != "Custom SQL": + self.table = self.tablemodels[self.tablecombo.currentIndex()] self.database_desc["Table"] = self.table if "Query" in self.database_desc: del self.database_desc["Query"] + what = self.table else: - self.sql = self.table = self.sqltext.toPlainText() + what = self.sql = self.sqltext.toPlainText() + self.table = "Custom SQL" if self.materialize: import psycopg2 if not self.materialize_table_name: @@ -297,11 +311,10 @@ def get_table(self): pass with self.backend.execute_sql_query("CREATE TABLE " + self.materialize_table_name + - " AS " + self.table): + " AS " + self.sql): pass with self.backend.execute_sql_query("ANALYZE " + self.materialize_table_name): pass - self.table = self.materialize_table_name except (psycopg2.ProgrammingError, BackendError) as ex: self.Error.connection(str(ex)) return @@ -312,7 +325,7 @@ def get_table(self): database=self.database, user=self.username, password=self.password), - self.table, + what, backend=type(self.backend), inspect_values=False) except BackendError as ex: @@ -321,8 +334,8 @@ def get_table(self): self.Error.connection.clear() - sample = False + if table.approx_len() > LARGE_TABLE and self.guess_values: confirm = QMessageBox(self) confirm.setIcon(QMessageBox.Warning) @@ -347,6 +360,7 @@ def get_table(self): domain = s.get_domain(inspect_values=True) self.Information.data_sampled() else: +# domain = table.get_domain(inspect_values=True) QApplication.restoreOverrideCursor() table.domain = domain From 8ae9a1a1acc212e92cc02612b7014c78c70efedb Mon Sep 17 00:00:00 2001 From: ceprio Date: Wed, 11 Oct 2017 15:38:11 -0400 Subject: [PATCH 2/7] Changed file according to reviewer comments and bug fix for SQL Server 2012 --- Orange/data/sql/backend/mssql.py | 23 ++++++++++++++++------- Orange/widgets/data/owsql.py | 20 ++++++++++---------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/Orange/data/sql/backend/mssql.py b/Orange/data/sql/backend/mssql.py index 2dbeb46d4d1..e6b19c75ded 100644 --- a/Orange/data/sql/backend/mssql.py +++ b/Orange/data/sql/backend/mssql.py @@ -117,17 +117,26 @@ def _guess_variable(self, field_name, field_metadata, inspect_table): EST_ROWS_RE = re.compile(r'StatementEstRows="(\d+)"') def count_approx(self, query): - try: - with self.connection.cursor() as cur: + with self.connection.cursor() as cur: + try: cur.execute("SET SHOWPLAN_XML ON") try: cur.execute(query) result = cur.fetchone() return int(self.EST_ROWS_RE.search(result[0]).group(1)) + except AttributeError: + # This is to catch a bug from SQL Server 2012 + # (the StatementEstRows=float instead of an int) + pass finally: cur.execute("SET SHOWPLAN_XML OFF") - except pymssql.Error as ex: - if "SHOWPLAN permission denied" in str(ex): - warnings.warn("SHOWPLAN permission denied, count approximates will not be used") - return None - raise BackendError(str(ex)) from ex + except pymssql.Error as ex: + if "SHOWPLAN permission denied" in str(ex): + warnings.warn("SHOWPLAN permission denied, count approximates will not be used") + return None + raise BackendError(str(ex)) from ex + # In case of an AttributeError, give a second chance: + # Use the long method for counting (or upgrade SQL Server version :) ) + cur.execute("SELECT count(*) FROM ( {} ) x".format(query)) + result = cur.fetchone() + return result[0] diff --git a/Orange/widgets/data/owsql.py b/Orange/widgets/data/owsql.py index 7575d574c3b..e0b5d61d212 100644 --- a/Orange/widgets/data/owsql.py +++ b/Orange/widgets/data/owsql.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # -*- coding: utf-8 -*- import sys from collections import OrderedDict @@ -30,7 +29,7 @@ def data(self, index, role=Qt.DisplayRole): return super().data(index, role) -class BackendModels(PyListModel): +class BackendModel(PyListModel): def data(self, index, role=Qt.DisplayRole): row = index.row() if role == Qt.DisplayRole: @@ -87,10 +86,10 @@ def __init__(self): vbox = gui.vBox(self.controlArea, "Server", addSpace=True) box = gui.vBox(vbox) - self.backendmodels = BackendModels(Backend.available_backends()) + self.backends = BackendModel(Backend.available_backends()) self.backendcombo = QComboBox(box) - if len(self.backendmodels): - self.backendcombo.setModel(self.backendmodels) + if len(self.backends): + self.backendcombo.setModel(self.backends) else: self.Error.no_backends() box.setEnabled(False) @@ -141,7 +140,8 @@ def __init__(self): index = self.tablecombo.findText(str(self.table)) if index != -1: self.tablecombo.setCurrentIndex(index) - self.tablecombo.activated[int].connect(self.select_table) # set up the callback to select_table in case of selection change + # set up the callback to select_table in case of selection change + self.tablecombo.activated[int].connect(self.select_table) self.connectbutton = gui.button( tables, self, '↻', callback=self.connect) @@ -226,7 +226,7 @@ def connect(self): try: if self.backendcombo.currentIndex() < 0: return - backend = self.backendmodels[self.backendcombo.currentIndex()] + backend = self.backends[self.backendcombo.currentIndex()] self.backend = backend(dict( host=self.host, port=self.port, @@ -258,8 +258,8 @@ def refresh_tables(self): self.tablemodels.append("Custom SQL") self.tablemodels.extend(self.backend.list_tables(self.schema)) + # Called on tablecombo selection change: def select_table(self): - "Called on tablecombo selection change" curIdx = self.tablecombo.currentIndex() if self.tablecombo.itemText(curIdx) != "Custom SQL": self.custom_sql.setVisible(False) @@ -360,7 +360,7 @@ def get_table(self): domain = s.get_domain(inspect_values=True) self.Information.data_sampled() else: -# + domain = table.get_domain(inspect_values=True) QApplication.restoreOverrideCursor() table.domain = domain @@ -409,4 +409,4 @@ def migrate_settings(cls, settings, version): ow = OWSql() ow.show() a.exec_() - ow.saveSettings() + ow.saveSettings() \ No newline at end of file From a1e6411d03463e4de26daebfa49db6051c6fc47f Mon Sep 17 00:00:00 2001 From: ceprio Date: Wed, 11 Oct 2017 16:06:57 -0400 Subject: [PATCH 3/7] Correction following comments from Travis CI --- Orange/data/sql/backend/mssql.py | 6 +++--- Orange/widgets/data/owsql.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Orange/data/sql/backend/mssql.py b/Orange/data/sql/backend/mssql.py index e6b19c75ded..15b5a7469cc 100644 --- a/Orange/data/sql/backend/mssql.py +++ b/Orange/data/sql/backend/mssql.py @@ -124,9 +124,9 @@ def count_approx(self, query): cur.execute(query) result = cur.fetchone() return int(self.EST_ROWS_RE.search(result[0]).group(1)) - except AttributeError: - # This is to catch a bug from SQL Server 2012 - # (the StatementEstRows=float instead of an int) + except AttributeError: + # This is to catch a bug from SQL Server 2012 + # resulting StatementEstRows is float instead of int pass finally: cur.execute("SET SHOWPLAN_XML OFF") diff --git a/Orange/widgets/data/owsql.py b/Orange/widgets/data/owsql.py index e0b5d61d212..72831e8e5b9 100644 --- a/Orange/widgets/data/owsql.py +++ b/Orange/widgets/data/owsql.py @@ -141,7 +141,7 @@ def __init__(self): if index != -1: self.tablecombo.setCurrentIndex(index) # set up the callback to select_table in case of selection change - self.tablecombo.activated[int].connect(self.select_table) + self.tablecombo.activated[int].connect(self.select_table) self.connectbutton = gui.button( tables, self, '↻', callback=self.connect) @@ -409,4 +409,4 @@ def migrate_settings(cls, settings, version): ow = OWSql() ow.show() a.exec_() - ow.saveSettings() \ No newline at end of file + ow.saveSettings() From 8ba75840c49db89490d030e55715bd622190adb6 Mon Sep 17 00:00:00 2001 From: ceprio Date: Sat, 14 Oct 2017 13:06:55 -0400 Subject: [PATCH 4/7] Cosmetic changes following reviewer's comment --- Orange/data/sql/backend/mssql.py | 6 +++--- Orange/widgets/data/owsql.py | 17 ++++++++--------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Orange/data/sql/backend/mssql.py b/Orange/data/sql/backend/mssql.py index 15b5a7469cc..009fd5e3cf4 100644 --- a/Orange/data/sql/backend/mssql.py +++ b/Orange/data/sql/backend/mssql.py @@ -125,8 +125,8 @@ def count_approx(self, query): result = cur.fetchone() return int(self.EST_ROWS_RE.search(result[0]).group(1)) except AttributeError: - # This is to catch a bug from SQL Server 2012 - # resulting StatementEstRows is float instead of int + # This is to catch a float received in StatementEstRows + # a float is received when the server's statistics are out of date. pass finally: cur.execute("SET SHOWPLAN_XML OFF") @@ -136,7 +136,7 @@ def count_approx(self, query): return None raise BackendError(str(ex)) from ex # In case of an AttributeError, give a second chance: - # Use the long method for counting (or upgrade SQL Server version :) ) + # Use the long method for counting cur.execute("SELECT count(*) FROM ( {} ) x".format(query)) result = cur.fetchone() return result[0] diff --git a/Orange/widgets/data/owsql.py b/Orange/widgets/data/owsql.py index 72831e8e5b9..6c88091d413 100644 --- a/Orange/widgets/data/owsql.py +++ b/Orange/widgets/data/owsql.py @@ -21,7 +21,7 @@ MAX_DL_LIMIT = 1000000 -class TableModels(PyListModel): +class TableModel(PyListModel): def data(self, index, role=Qt.DisplayRole): row = index.row() if role == Qt.DisplayRole: @@ -125,14 +125,14 @@ def __init__(self): box.layout().addWidget(self.passwordtext) self._load_credentials() - self.tablemodels = TableModels() + self.tables = TableModel() tables = gui.hBox(box) self.tablecombo = QComboBox( minimumContentsLength=35, sizeAdjustPolicy=QComboBox.AdjustToMinimumContentsLength ) - self.tablecombo.setModel(self.tablemodels) + self.tablecombo.setModel(self.tables) self.tablecombo.setToolTip('table') tables.layout().addWidget(self.tablecombo) self.connect() @@ -248,15 +248,15 @@ def connect(self): self.tablecombo.clear() def refresh_tables(self): - self.tablemodels.clear() + self.tables.clear() self.Error.missing_extension.clear() if self.backend is None: self.data_desc_table = None return - self.tablemodels.append("Select a table") - self.tablemodels.append("Custom SQL") - self.tablemodels.extend(self.backend.list_tables(self.schema)) + self.tables.append("Select a table") + self.tables.append("Custom SQL") + self.tables.extend(self.backend.list_tables(self.schema)) # Called on tablecombo selection change: def select_table(self): @@ -291,7 +291,7 @@ def get_table(self): return if self.tablecombo.itemText(curIdx) != "Custom SQL": - self.table = self.tablemodels[self.tablecombo.currentIndex()] + self.table = self.tables[self.tablecombo.currentIndex()] self.database_desc["Table"] = self.table if "Query" in self.database_desc: del self.database_desc["Query"] @@ -360,7 +360,6 @@ def get_table(self): domain = s.get_domain(inspect_values=True) self.Information.data_sampled() else: - domain = table.get_domain(inspect_values=True) QApplication.restoreOverrideCursor() table.domain = domain From 34433be9485d649b89babf08e61392359330eb1f Mon Sep 17 00:00:00 2001 From: ceprio Date: Sat, 14 Oct 2017 14:51:08 -0400 Subject: [PATCH 5/7] Removed white space --- Orange/data/sql/backend/mssql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Orange/data/sql/backend/mssql.py b/Orange/data/sql/backend/mssql.py index 009fd5e3cf4..2d9e61831f9 100644 --- a/Orange/data/sql/backend/mssql.py +++ b/Orange/data/sql/backend/mssql.py @@ -136,7 +136,7 @@ def count_approx(self, query): return None raise BackendError(str(ex)) from ex # In case of an AttributeError, give a second chance: - # Use the long method for counting + # Use the long method for counting cur.execute("SELECT count(*) FROM ( {} ) x".format(query)) result = cur.fetchone() return result[0] From f7918571a928c79775268d138023fc1cc3749246 Mon Sep 17 00:00:00 2001 From: ceprio Date: Mon, 16 Oct 2017 16:50:07 -0400 Subject: [PATCH 6/7] Code change to allow exact count be made by the callee. --- Orange/data/sql/backend/mssql.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Orange/data/sql/backend/mssql.py b/Orange/data/sql/backend/mssql.py index 2d9e61831f9..d63e59e606f 100644 --- a/Orange/data/sql/backend/mssql.py +++ b/Orange/data/sql/backend/mssql.py @@ -123,11 +123,17 @@ def count_approx(self, query): try: cur.execute(query) result = cur.fetchone() - return int(self.EST_ROWS_RE.search(result[0]).group(1)) - except AttributeError: - # This is to catch a float received in StatementEstRows - # a float is received when the server's statistics are out of date. - pass + match = self.EST_ROWS_RE.search(result[0]) + if not match: + # Either StatementEstRows was not found or + # a float is received. + # If it is a float then it is most probable + # that the server's statistics are out of date + # and the result is false. In that case + # it is preferable to return None so + # an exact count be used. + return None + return int(match.group(1)) finally: cur.execute("SET SHOWPLAN_XML OFF") except pymssql.Error as ex: @@ -135,8 +141,3 @@ def count_approx(self, query): warnings.warn("SHOWPLAN permission denied, count approximates will not be used") return None raise BackendError(str(ex)) from ex - # In case of an AttributeError, give a second chance: - # Use the long method for counting - cur.execute("SELECT count(*) FROM ( {} ) x".format(query)) - result = cur.fetchone() - return result[0] From 6e702a09bb9f949f9e41497e2ec430a60b0afc4b Mon Sep 17 00:00:00 2001 From: ceprio Date: Mon, 16 Oct 2017 17:32:17 -0400 Subject: [PATCH 7/7] Removed spaces at end of lines Eclipse does not seem to be able to take care of these... --- Orange/data/sql/backend/mssql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Orange/data/sql/backend/mssql.py b/Orange/data/sql/backend/mssql.py index d63e59e606f..6d136043472 100644 --- a/Orange/data/sql/backend/mssql.py +++ b/Orange/data/sql/backend/mssql.py @@ -127,10 +127,10 @@ def count_approx(self, query): if not match: # Either StatementEstRows was not found or # a float is received. - # If it is a float then it is most probable + # If it is a float then it is most probable # that the server's statistics are out of date # and the result is false. In that case - # it is preferable to return None so + # it is preferable to return None so # an exact count be used. return None return int(match.group(1))