commit 585b7e8e555a217be32213b8d667421f4c9e729d
author Primiano Tucci <primiano@google.com> Thu May 14 11:19:10 2020 +0100
committer Primiano Tucci <primiano@google.com> Thu May 14 11:19:10 2020 +0100
tree 752d7d8109d8b6efb2cbc9e10264fa5597e45cd9
parent d8cb80d7a786d4896fd1ac22ff35470187d3a933

Use exec(curl) for downloading artifacts

Due to the union of an openssl bug, python inconsistencies
and Google Cloud changing CA, there doesn't seem to be a way
to consistently download an artifact over https in pure python
that works both on Linux and Mac (At least for Google HW).

On Mac:
  python3 -c 'import urllib.request;  urllib.request.urlopen("https://storage.googleapis.com/")'
  Fails with CERTIFICATE_VERIFY_FAILED because python3 on mac
  bundles its own set of CA roots which seems
  now obsolete (https://bugs.python.org/msg276516)

  However

  python2 -c 'from urllib import urlretrieve; urlretrieve("https://storage.googleapis.com/")'
  works

On Linux:
  python3 -c 'import urllib.request;  urllib.request.urlopen("https://storage.googleapis.com/")'
  Works

  However

  python2 -c 'from urllib import urlretrieve; urlretrieve("https://storage.googleapis.com/")'
  Fails with
  ssl.SSLError: [SSL: KRB5_S_TKT_NYV] unexpected eof while reading (_ssl.c:1946)
  Because of a bung introduced by OpenSSL 1.1.1e (https://bugs.python.org/issue40018)

Other option (like requests) are not viable because require the user to perform
some manual actions (pip install...) which is a bad UX
The world failed us. exec(curl) to the rescue.

Bug: many
Test: manually tested both install-build-deps and heap_profile on mac
Change-Id: I064c539fd15f7cac87bd0050631c6e008acaff77

tools/heap_profile

diff --git a/tools/heap_profile b/tools/heap_profile
index f2aaf0f..a969de4 100755
--- a/tools/heap_profile
+++ b/tools/heap_profile
@@ -30,12 +30,6 @@
 import time
 import uuid
 
-try:
-    # Python 2.x
-    from urllib import urlretrieve
-except ImportError:
-    # Python 3.x
-    from urllib.request import urlretrieve
 
 TRACE_TO_TEXT_SHAS = {
     'linux': 'aba0e660818bfc249992ebbceb13a2e4c9a62c3a',
@@ -71,7 +65,7 @@
       return local_file
 
   url = TRACE_TO_TEXT_BASE_URL + file_name
-  urlretrieve(url, local_file)
+  subprocess.check_call(['curl', '-#', '-o', local_file, url])
   if not check_hash(local_file, sha_value):
     os.remove(local_file)
     raise ValueError("Invalid signature.")

tools/install-build-deps

diff --git a/tools/install-build-deps b/tools/install-build-deps
index b189463..7bd693c 100755
--- a/tools/install-build-deps
+++ b/tools/install-build-deps
@@ -26,7 +26,6 @@
 from collections import namedtuple
 from platform import system
 
-from compat import urlretrieve
 
 # The format for the deps below is the following:
 # (target_folder, source_url, sha1, target_platform)
@@ -232,6 +231,10 @@
 NODE_MODULES_STATUS_FILE = os.path.join(UI_DIR, 'node_modules', '.last_install')
 
 
+def DownloadURL(url, out_file):
+  subprocess.check_call(['curl', '-#', '-o', out_file, url])
+
+
 def ReadFile(path):
   if not os.path.exists(path):
     return None
@@ -315,7 +318,7 @@
       logging.info('Downloading %s from %s', rel_path, url)
       with tempfile.NamedTemporaryFile(delete=False) as f:
         f.close()
-        urlretrieve(url, f.name)
+        DownloadURL(url, f.name)
         actual_sha1 = HashLocalFile(f.name)
         os.unlink(f.name)
         if (actual_sha1 != expected_sha1):
@@ -361,7 +364,7 @@
     if HashLocalFile(local_path) != expected_sha1:
       download_path = local_path + '.tmp'
       logging.info('Downloading %s from %s', local_path, url)
-      urlretrieve(url, download_path)
+      DownloadURL(url, download_path)
       os.chmod(download_path, 0o755)
       actual_sha1 = HashLocalFile(download_path)
       if (actual_sha1 != expected_sha1):