[core][cleanup] Refactor and remove old Py2 code checks and methods (#…

…3683)

## What changes were proposed in this pull request?

- Remove all Py2 check conditionals.
- Remove some not-used imports.
- Streamline file opening code section and remove py2 support.
- Remove `smart_unicode` method and change its usage with `smart_str` wherever required. This also triggered adding an encoding param in the `smart_str` method for compatibility with `smart_unicode` switch.


- Because of all the files modified with above actions, we are also fixing Ruff violations on these files **which made this change big.**
- **The idea is to merge this as a single commit and have as low regression as possible and if something major comes up, we can try simply reverting to previous state. For minor regressions, we can try fixing on top of this change.**

## How was this patch tested?

- Manually and basic Hue testing.
- Running existing full unit test suite.
- E2E cluster setup

apps/about/src/about/templates/admin_wizard.mako

@@ -23,7 +23,6 @@ from metadata.conf import OPTIMIZER, has_optimizer
 
 from desktop.auth.backend import is_admin
 from desktop.conf import has_connectors
-from desktop.lib.i18n import smart_unicode
 from desktop.views import commonheader, commonfooter
 
 if sys.version_info[0] > 2:

apps/about/src/about/urls.py

@@ -15,15 +15,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
+from django.urls import re_path
 
 from about import views as about_views
 
-if sys.version_info[0] > 2:
-  from django.urls import re_path
-else:
-  from django.conf.urls import url as re_path
-
 urlpatterns = [
   re_path(r'^$', about_views.admin_wizard, name='index'),
   re_path(r'^admin_wizard$', about_views.admin_wizard, name='admin_wizard'),

apps/about/src/about/views.py

@@ -15,9 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from builtins import str
-import logging
-import sys
+from django.utils.translation import gettext as _
 
 from desktop import appmanager
 from desktop.auth.backend import is_hue_admin
@@ -26,11 +24,6 @@
 from desktop.models import Settings, hue_version
 from desktop.views import collect_usage
 
-if sys.version_info[0] > 2:
-  from django.utils.translation import gettext as _
-else:
-  from django.utils.translation import ugettext as _
-
 
 def admin_wizard(request):
   if is_hue_admin(request.user):

apps/beeswax/src/beeswax/api.py

@@ -16,7 +16,6 @@
 # limitations under the License.
 
 import re
-import sys
 import json
 import logging
 from builtins import zip

apps/beeswax/src/beeswax/api_tests.py

@@ -16,27 +16,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
-import logging
-import pytest
 import sys
+import logging
+from unittest.mock import Mock, patch
 
+import pytest
 from django.test import TestCase
 from requests.exceptions import ReadTimeout
 
+from beeswax.api import _autocomplete, get_functions
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import add_to_group, grant_access
 from useradmin.models import User
 
-from beeswax.api import _autocomplete, get_functions
-
-
-if sys.version_info[0] > 2:
-  from unittest.mock import patch, Mock
-else:
-  from mock import patch, Mock
-
-
 LOG = logging.getLogger()
 
 
@@ -47,9 +39,8 @@ def setup_method(self):
     self.client = make_logged_in_client(username="test", groupname="default", recreate=True, is_superuser=False)
     self.user = User.objects.get(username="test")
 
-
   def test_autocomplete_time_out(self):
-    get_tables_meta=Mock(
+    get_tables_meta = Mock(
       side_effect=ReadTimeout("HTTPSConnectionPool(host='gethue.com', port=10001): Read timed out. (read timeout=120)")
     )
     db = Mock(
@@ -65,7 +56,6 @@ def test_autocomplete_time_out(self):
         'error': "HTTPSConnectionPool(host='gethue.com', port=10001): Read timed out. (read timeout=120)"
       })
 
-
   def test_get_functions(self):
     db = Mock(
       get_functions=Mock(
@@ -83,7 +73,6 @@ def test_get_functions(self):
       resp ==
       [{'name': 'f1'}, {'name': 'f2'}])
 
-
   def test_get_functions(self):
     with patch('beeswax.api._get_functions') as _get_functions:
       db = Mock()
@@ -97,12 +86,11 @@ def test_get_functions(self):
         resp['functions'] ==
         [{'name': 'f1'}, {'name': 'f2'}, {'name': 'f3'}])
 
-
   def test_get_function(self):
     db = Mock()
-    db.client = Mock(query_server = {'dialect': 'hive'})
+    db.client = Mock(query_server={'dialect': 'hive'})
     db.get_function = Mock(
-      return_value = [
+      return_value=[
         ['floor_month(param) - Returns the timestamp at a month granularity'],
         ['param needs to be a timestamp value'],
         ['Example:'],
@@ -123,8 +111,7 @@ def test_get_function(self):
             '> SELECT floor_month(CAST(\'yyyy-MM-dd HH:mm:ss\' AS TIMESTAMP)) FROM src;\nyyyy-MM-01 00:00:00'
       })
 
-
-    db.client = Mock(query_server = {'dialect': 'impala'})
+    db.client = Mock(query_server={'dialect': 'impala'})
     data = _autocomplete(db, operation='function')
 
     assert data['function'] == {}

apps/beeswax/src/beeswax/conf.py

@@ -15,13 +15,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import division
-
-import sys
 import math
 import logging
 import os.path
-from builtins import str
 
 from django.utils.translation import gettext as _, gettext_lazy as _t
 

apps/beeswax/src/beeswax/create_table.py

@@ -15,54 +15,47 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-from __future__ import division
-from builtins import str
-from builtins import range
-from builtins import object
+import re
 import csv
 import gzip
 import json
-import logging
 import math
-import re
-import sys
+import logging
 
-from django.urls import reverse
 from django.http import QueryDict
+from django.urls import reverse
+from django.utils.translation import gettext as _
 
 from aws.s3.s3fs import S3FileSystemException
+from beeswax.common import TERMINATORS
+from beeswax.design import hql_query
+from beeswax.forms import (
+  TERMINATOR_CHOICES,
+  ColumnTypeFormSet,
+  CreateByImportDelimForm,
+  CreateByImportFileForm,
+  CreateTableForm,
+  PartitionTypeFormSet,
+)
+from beeswax.server import dbms
+from beeswax.server.dbms import QueryServerException
+from beeswax.views import execute_directly
 from desktop.context_processors import get_app_name
 from desktop.lib import django_mako, i18n
+from desktop.lib.django_forms import MultiForm
 from desktop.lib.django_util import render
 from desktop.lib.exceptions_renderable import PopupException
-from desktop.lib.django_forms import MultiForm
 from desktop.models import _get_apps
 from hadoop.fs import hadoopfs
 
-from beeswax.common import TERMINATORS
-from beeswax.design import hql_query
-from beeswax.forms import CreateTableForm, ColumnTypeFormSet,\
-  PartitionTypeFormSet, CreateByImportFileForm, CreateByImportDelimForm,\
-  TERMINATOR_CHOICES
-from beeswax.server import dbms
-from beeswax.server.dbms import QueryServerException
-from beeswax.views import execute_directly
-
-if sys.version_info[0] > 2:
-  from django.utils.translation import gettext as _
-else:
-  from django.utils.translation import ugettext as _
-
-
 LOG = logging.getLogger()
 
 
 def create_table(request, database='default'):
   """Create a table by specifying its attributes manually"""
   db = dbms.get(request.user)
   dbs = db.get_databases()
-  databases = [{'name':db, 'url':reverse('beeswax:create_table', kwargs={'database': db})} for db in dbs]
+  databases = [{'name': db, 'url': reverse('beeswax:create_table', kwargs={'database': db})} for db in dbs]
 
   form = MultiForm(
       table=CreateTableForm,
@@ -77,8 +70,8 @@ def create_table(request, database='default'):
 
     if request.POST.get('create'):
       if form.is_valid():
-        columns = [ f.cleaned_data for f in form.columns.forms ]
-        partition_columns = [ f.cleaned_data for f in form.partitions.forms ]
+        columns = [f.cleaned_data for f in form.columns.forms]
+        partition_columns = [f.cleaned_data for f in form.partitions.forms]
         proposed_query = django_mako.render_to_string("create_table_statement.mako", {
             'databases': databases,
             'database': database,
@@ -109,15 +102,16 @@ def create_table(request, database='default'):
 
 IMPORT_PEEK_SIZE = 5 * 1024**2
 IMPORT_PEEK_NLINES = 10
-DELIMITERS = [ hive_val for hive_val, desc, ascii in TERMINATORS ]
-DELIMITER_READABLE = {'\\001' : _('ctrl-As'),
-                      '\\002' : _('ctrl-Bs'),
-                      '\\003' : _('ctrl-Cs'),
-                      '\\t'   : _('tabs'),
-                      ','     : _('commas'),
-                      ' '     : _('spaces')}
+DELIMITERS = [hive_val for hive_val, desc, ascii in TERMINATORS]
+DELIMITER_READABLE = {'\\001': _('ctrl-As'),
+                      '\\002': _('ctrl-Bs'),
+                      '\\003': _('ctrl-Cs'),
+                      '\\t': _('tabs'),
+                      ',': _('commas'),
+                      ' ': _('spaces')}
 FILE_READERS = []
 
+
 def import_wizard(request, database='default'):
   """
   Help users define table and based on a file they want to import to Hive.
@@ -133,7 +127,7 @@ def import_wizard(request, database='default'):
 
   db = dbms.get(request.user)
   dbs = db.get_databases()
-  databases = [{'name':db, 'url':reverse('beeswax:import_wizard', kwargs={'database': db})} for db in dbs]
+  databases = [{'name': db, 'url': reverse('beeswax:import_wizard', kwargs={'database': db})} for db in dbs]
 
   if request.method == 'POST':
     #
@@ -164,7 +158,7 @@ def import_wizard(request, database='default'):
       cancel_s3_column_def = request.POST.get('cancel_create')  # Step 3 -> 2
 
       # Exactly one of these should be True
-      if len([_f for _f in (do_s2_auto_delim, do_s2_user_delim, do_s3_column_def, do_hive_create, cancel_s2_user_delim, cancel_s3_column_def) if _f]) != 1:
+      if len([_f for _f in (do_s2_auto_delim, do_s2_user_delim, do_s3_column_def, do_hive_create, cancel_s2_user_delim, cancel_s3_column_def) if _f]) != 1:  # noqa: E501
         raise PopupException(_('Invalid form submission'))
 
       if not do_s2_auto_delim:
@@ -198,7 +192,8 @@ def import_wizard(request, database='default'):
           raise PopupException(_('Path location "%s" is invalid: %s') % (path, e))
 
         delim_is_auto = True
-        fields_list, n_cols, s2_delim_form = _delim_preview(request.fs, s1_file_form, encoding, [reader.TYPE for reader in FILE_READERS], DELIMITERS)
+        fields_list, n_cols, s2_delim_form = _delim_preview(
+          request.fs, s1_file_form, encoding, [reader.TYPE for reader in FILE_READERS], DELIMITERS)
 
       if (do_s2_user_delim or do_s3_column_def or cancel_s3_column_def) and s2_delim_form.is_valid():
         # Delimit based on input
@@ -236,7 +231,7 @@ def import_wizard(request, database='default'):
         try:
           fields_list_for_json = list(fields_list)
           if fields_list_for_json:
-            fields_list_for_json[0] = [re.sub('[^\w]', '', a) for a in fields_list_for_json[0]] # Cleaning headers
+            fields_list_for_json[0] = [re.sub(r'[^\w]', '', a) for a in fields_list_for_json[0]]  # Cleaning headers
           apps_list = _get_apps(request.user, '')
           return render('import_wizard_define_columns.mako', request, {
             'apps': apps_list,
@@ -251,7 +246,8 @@ def import_wizard(request, database='default'):
             'databases': databases
           })
         except Exception as e:
-          raise PopupException(_("The selected delimiter is creating an un-even number of columns. Please make sure you don't have empty columns."), detail=e)
+          raise PopupException(_(
+            "The selected delimiter is creating an un-even number of columns. Please make sure you don't have empty columns."), detail=e)
 
       #
       # Final: Execute
@@ -271,7 +267,7 @@ def import_wizard(request, database='default'):
                 'path': path,
                 'skip_header': request.GET.get('removeHeader', 'off').lower() == 'on'
              },
-            'columns': [ f.cleaned_data for f in s3_col_formset.forms ],
+            'columns': [f.cleaned_data for f in s3_col_formset.forms],
             'partition_columns': [],
             'database': database,
             'databases': databases
@@ -337,7 +333,7 @@ def _delim_preview(fs, file_form, encoding, file_types, delimiters):
     LOG.exception(msg)
     raise PopupException(msg)
 
-  n_cols = max([ len(row) for row in fields_list ])
+  n_cols = max([len(row) for row in fields_list])
   # ``delimiter`` is a MultiValueField. delimiter_0 and delimiter_1 are the sub-fields.
   delimiter_0 = delim
   delimiter_1 = ''
@@ -409,13 +405,12 @@ def score_delim(fields_list):
     avg_n_fields = math.floor(sum(len_list) / n_lines)
     sq_of_exp = avg_n_fields * avg_n_fields
 
-    len_list_sq = [l * l for l in len_list]
+    len_list_sq = [len * len for len in len_list]
     exp_of_sq = math.floor(sum(len_list_sq) / n_lines)
     var = exp_of_sq - sq_of_exp
     # Favour more fields
     return (1000.0 / (var + 1)) + avg_n_fields
 
-
   max_score = -1
   res = (None, None)
 
@@ -424,7 +419,7 @@ def score_delim(fields_list):
     delimiter = delim.decode('string_escape')
     try:
       fields_list = _get_rows(lines, delimiter)
-    except:
+    except Exception:
       LOG.exception('failed to get rows')
       fields_list = [line.split(delimiter) for line in lines if line]
 
@@ -472,6 +467,7 @@ def readlines(fileobj, encoding):
     except UnicodeError:
       return None
 
+
 FILE_READERS.append(GzipFileReader)
 
 
@@ -488,6 +484,7 @@ def readlines(fileobj, encoding):
     except UnicodeError:
       return None
 
+
 FILE_READERS.append(TextFileReader)