Manually set up gsutil path for newer versions of Python

utils/clone.py

@@ -47,13 +47,15 @@
 """
 
 
-def clone(args): # pylint: disable=too-many-branches, too-many-statements
+def clone(args): # pylint: disable=too-many-branches, too-many-locals, too-many-statements
     """Clones, downloads, and generates the required sources"""
     get_logger().info('Setting up cloning environment')
     iswin = sys.platform.startswith('win')
     chromium_version = get_chromium_version()
     ucstaging = args.output / 'uc_staging'
     dtpath = ucstaging / 'depot_tools'
+    gsuver = '5.30'
+    gsupath = dtpath / 'external_bin' / 'gsutil' / ('gsutil_%s' % gsuver) / 'gsutil'
     gnpath = ucstaging / 'gn'
     environ['GCLIENT_FILE'] = str(ucstaging / '.gclient')
     environ['PATH'] += pathsep + str(dtpath)
@@ -109,11 +111,36 @@ def clone(args): # pylint: disable=too-many-branches, too-many-statements
     # Apply changes to gclient
     run(['git', 'apply'],
         input=Path(__file__).with_name('depot_tools.patch').read_text().replace(
-            'UC_OUT', str(args.output)).replace('UC_STAGING', str(ucstaging)),
+            'UC_OUT', str(args.output)).replace('UC_STAGING',
+                                                str(ucstaging)).replace('GSUVER', gsuver),
         cwd=dtpath,
         check=True,
         universal_newlines=True)
 
+    # Manualy set up the gsutil directory for newer versions of Python
+    get_logger().info('Cloning gsutil')
+    if not gsupath.exists():
+        gsupath.mkdir(parents=True)
+        run(['git', 'init', '-q'], cwd=gsupath, check=True)
+        run(['git', 'remote', 'add', 'origin', 'https://github.com/GoogleCloudPlatform/gsutil'],
+            cwd=gsupath,
+            check=True)
+    run(['git', 'fetch', '--depth=1', 'origin', 'v%s' % gsuver], cwd=gsupath, check=True)
+    run(['git', 'reset', '--hard', 'FETCH_HEAD'], cwd=gsupath, check=True)
+    run(['git', 'clean', '-ffdx'], cwd=gsupath, check=True)
+    get_logger().info('Updating gsutil submodules')
+    run(['git', 'submodule', 'update', '--init', '--recursive', '--depth=1', '-q'],
+        cwd=gsupath,
+        check=True)
+    # apitools needs to be set to a newer commit for newer versions of Python
+    run(['git', 'fetch', 'origin', 'f0dfa4e3fcb510d7d27389e011198d9f176026e2'],
+        cwd=(gsupath / 'third_party' / 'apitools'),
+        check=True)
+    run(['git', 'reset', '--hard', 'FETCH_HEAD'],
+        cwd=(gsupath / 'third_party' / 'apitools'),
+        check=True)
+    (gsupath / 'install.flag').write_text('This flag file is dropped by clone.py')
+
     # gn requires full history to be able to generate last_commit_position.h
     get_logger().info('Cloning gn')
     if gnpath.exists():

utils/depot_tools.patch

@@ -67,7 +67,7 @@
 
 --- a/gclient_scm.py
 +++ b/gclient_scm.py
-@@ -925,8 +925,7 @@ class GitWrapper(SCMWrapper):
+@@ -952,8 +952,7 @@ class GitWrapper(SCMWrapper):
          self._SetFetchConfig(options)
 
          # Fetch upstream if we don't already have |revision|.
@@ -77,7 +77,7 @@
              self._Fetch(options, prune=options.force)
 
              if not scm.GIT.IsValidRevision(
-@@ -942,7 +941,7 @@ class GitWrapper(SCMWrapper):
+@@ -969,7 +968,7 @@ class GitWrapper(SCMWrapper):
 
          # This is a big hammer, debatable if it should even be here...
          if options.force or options.reset:
@@ -86,15 +86,15 @@
              if options.upstream and upstream_branch:
                  target = upstream_branch
              self._Scrub(target, options)
-@@ -957,7 +956,6 @@ class GitWrapper(SCMWrapper):
+@@ -984,7 +983,6 @@ class GitWrapper(SCMWrapper):
              # to the checkout step.
              if not (options.force or options.reset):
                  self._CheckClean(revision)
 -            self._CheckDetachedHead(revision, options)
 
              if not current_revision:
                  current_revision = self._Capture(
-@@ -1637,8 +1635,7 @@ class GitWrapper(SCMWrapper):
+@@ -1668,8 +1666,7 @@ class GitWrapper(SCMWrapper):
              fetch_cmd.append('--no-tags')
          elif quiet:
              fetch_cmd.append('--quiet')
@@ -111,7 +111,7 @@
  IS_WINDOWS = os.name == 'nt'
 
 -VERSION = '4.68'
-+VERSION = '5.28'
++VERSION = 'GSUVER'
 
  # Google OAuth Context required by gsutil.
  LUCI_AUTH_SCOPES = [