Update to latest catapult (be85e5fd)android-cts_7.1_r1android-cts-7.1_r9android-cts-7.1_r8android-cts-7.1_r7android-cts-7.1_r6android-cts-7.1_r5android-cts-7.1_r4android-cts-7.1_r3android-cts-7.1_r29android-cts-7.1_r28android-cts-7.1_r27android-cts-7.1_r26android-cts-7.1_r25android-cts-7.1_r24android-cts-7.1_r23android-cts-7.1_r22android-cts-7.1_r21android-cts-7.1_r20android-cts-7.1_r2android-cts-7.1_r19android-cts-7.1_r18android-cts-7.1_r17android-cts-7.1_r16android-cts-7.1_r15android-cts-7.1_r14android-cts-7.1_r13android-cts-7.1_r12android-cts-7.1_r11android-cts-7.1_r10android-cts-7.1_r1android-7.1.1_r9android-7.1.1_r8android-7.1.1_r7android-7.1.1_r61android-7.1.1_r60android-7.1.1_r6android-7.1.1_r59android-7.1.1_r58android-7.1.1_r57android-7.1.1_r56android-7.1.1_r55android-7.1.1_r54android-7.1.1_r53android-7.1.1_r52android-7.1.1_r51android-7.1.1_r50android-7.1.1_r49android-7.1.1_r48android-7.1.1_r47android-7.1.1_r46android-7.1.1_r45android-7.1.1_r44android-7.1.1_r43android-7.1.1_r42android-7.1.1_r41android-7.1.1_r40android-7.1.1_r4android-7.1.1_r39android-7.1.1_r38android-7.1.1_r35android-7.1.1_r33android-7.1.1_r32android-7.1.1_r31android-7.1.1_r3android-7.1.1_r28android-7.1.1_r27android-7.1.1_r26android-7.1.1_r25android-7.1.1_r24android-7.1.1_r23android-7.1.1_r22android-7.1.1_r21android-7.1.1_r20android-7.1.1_r2android-7.1.1_r17android-7.1.1_r16android-7.1.1_r15android-7.1.1_r14android-7.1.1_r13android-7.1.1_r12android-7.1.1_r11android-7.1.1_r10android-7.1.1_r1nougat-mr1.8-releasenougat-mr1.7-releasenougat-mr1.6-releasenougat-mr1.5-releasenougat-mr1.4-releasenougat-mr1.3-releasenougat-mr1.2-releasenougat-mr1.1-releasenougat-mr1-volantis-releasenougat-mr1-security-releasenougat-mr1-releasenougat-mr1-flounder-releasenougat-mr1-devnougat-mr1-cts-release

bug:31335915
bug:31340203
bug:31551133

Change-Id: I844b83711c56f3ea8a6d0bf254a1187b62c33f81

11 files changed, 50 insertions, 13 deletions

diff --git a/catapult/dependency_manager/PRESUBMIT.py b/catapult/dependency_manager/PRESUBMIT.py
index 3db32cec..a34480c7 100644
--- a/catapult/dependency_manager/PRESUBMIT.py
+++ b/catapult/dependency_manager/PRESUBMIT.py
@@ -25,7 +25,7 @@ def _GetPathsToPrepend(input_api):
   return [
       project_dir,
 
-      input_api.os_path.join(catapult_dir, 'catapult_base'),
+      input_api.os_path.join(catapult_dir, 'common', 'py_utils'),
 
       input_api.os_path.join(catapult_dir, 'third_party', 'mock'),
       input_api.os_path.join(catapult_dir, 'third_party', 'pyfakefs'),
diff --git a/catapult/dependency_manager/dependency_manager/__init__.py b/catapult/dependency_manager/dependency_manager/__init__.py
index f7908c0b..296eb04a 100644
--- a/catapult/dependency_manager/dependency_manager/__init__.py
+++ b/catapult/dependency_manager/dependency_manager/__init__.py
@@ -18,7 +18,7 @@ def _AddDirToPythonPath(*path_parts):
     sys.path.append(path)
 
 
-_AddDirToPythonPath(CATAPULT_PATH, 'catapult_base')
+_AddDirToPythonPath(CATAPULT_PATH, 'common', 'py_utils')
 _AddDirToPythonPath(CATAPULT_THIRD_PARTY_PATH, 'mock')
 _AddDirToPythonPath(CATAPULT_THIRD_PARTY_PATH, 'pyfakefs')
 _AddDirToPythonPath(DEPENDENCY_MANAGER_PATH)
diff --git a/catapult/dependency_manager/dependency_manager/base_config.py b/catapult/dependency_manager/dependency_manager/base_config.py
index 47897c78..840ddf8f 100644
--- a/catapult/dependency_manager/dependency_manager/base_config.py
+++ b/catapult/dependency_manager/dependency_manager/base_config.py
@@ -6,7 +6,7 @@ import json
 import logging
 import os
 
-from catapult_base import cloud_storage
+from py_utils import cloud_storage
 from dependency_manager import archive_info
 from dependency_manager import cloud_storage_info
 from dependency_manager import dependency_info
diff --git a/catapult/dependency_manager/dependency_manager/base_config_unittest.py b/catapult/dependency_manager/dependency_manager/base_config_unittest.py
index 6f1140ce..2cffbd8a 100755
--- a/catapult/dependency_manager/dependency_manager/base_config_unittest.py
+++ b/catapult/dependency_manager/dependency_manager/base_config_unittest.py
@@ -7,7 +7,7 @@
 import os
 import unittest
 
-from catapult_base import cloud_storage
+from py_utils import cloud_storage
 import mock
 from pyfakefs import fake_filesystem_unittest
 from pyfakefs import fake_filesystem
diff --git a/catapult/dependency_manager/dependency_manager/cloud_storage_info.py b/catapult/dependency_manager/dependency_manager/cloud_storage_info.py
index e8b73c85..376c311b 100644
--- a/catapult/dependency_manager/dependency_manager/cloud_storage_info.py
+++ b/catapult/dependency_manager/dependency_manager/cloud_storage_info.py
@@ -6,7 +6,7 @@ import errno
 import os
 import stat
 
-from catapult_base import cloud_storage
+from py_utils import cloud_storage
 
 from dependency_manager import exceptions
 
diff --git a/catapult/dependency_manager/dependency_manager/cloud_storage_info_unittest.py b/catapult/dependency_manager/dependency_manager/cloud_storage_info_unittest.py
index 5c03d9af..be44fc18 100644
--- a/catapult/dependency_manager/dependency_manager/cloud_storage_info_unittest.py
+++ b/catapult/dependency_manager/dependency_manager/cloud_storage_info_unittest.py
@@ -8,7 +8,7 @@ import unittest
 
 import mock
 from pyfakefs import fake_filesystem_unittest
-from catapult_base import cloud_storage
+from py_utils import cloud_storage
 
 from dependency_manager import archive_info
 from dependency_manager import cloud_storage_info
@@ -127,7 +127,7 @@ class TestGetRemotePath(fake_filesystem_unittest.TestCase):
     self.tearDownPyfakefs()
 
   @mock.patch(
-      'catapult_base.cloud_storage.GetIfHashChanged')
+      'py_utils.cloud_storage.GetIfHashChanged')
   def testGetRemotePathNoArchive(self, cs_get_mock):
     def _GetIfHashChangedMock(cs_path, download_path, bucket, file_hash):
       del cs_path, bucket, file_hash
@@ -198,7 +198,7 @@ class TestGetRemotePath(fake_filesystem_unittest.TestCase):
 
 
   @mock.patch(
-      'catapult_base.cloud_storage.GetIfHashChanged')
+      'py_utils.cloud_storage.GetIfHashChanged')
   def testGetRemotePathCloudStorageErrors(self, cs_get_mock):
     cs_get_mock.side_effect = cloud_storage.CloudStorageError
     self.assertRaises(cloud_storage.CloudStorageError,
diff --git a/catapult/dependency_manager/dependency_manager/dependency_manager_unittest.py b/catapult/dependency_manager/dependency_manager/dependency_manager_unittest.py
index 1d55afe6..86d17f79 100644
--- a/catapult/dependency_manager/dependency_manager/dependency_manager_unittest.py
+++ b/catapult/dependency_manager/dependency_manager/dependency_manager_unittest.py
@@ -4,9 +4,10 @@
 
 # pylint: disable=unused-argument
 
-from catapult_base import cloud_storage
 import mock
+
 from pyfakefs import fake_filesystem_unittest
+from py_utils import cloud_storage
 
 import dependency_manager
 from dependency_manager import exceptions
diff --git a/catapult/dependency_manager/dependency_manager/dependency_manager_util.py b/catapult/dependency_manager/dependency_manager/dependency_manager_util.py
index afa15f3c..ed65db79 100644
--- a/catapult/dependency_manager/dependency_manager/dependency_manager_util.py
+++ b/catapult/dependency_manager/dependency_manager/dependency_manager_util.py
@@ -20,6 +20,9 @@ def _WinReadOnlyHandler(func, path, execinfo):
 
 
 def RemoveDir(dir_path):
+  assert os.path.isabs(dir_path)
+  if sys.platform.startswith('win'):
+    dir_path = u'\\\\?\\' + dir_path
   if os.path.isdir(dir_path):
     shutil.rmtree(dir_path, onerror=_WinReadOnlyHandler)
 
@@ -87,9 +90,14 @@ def UnzipArchive(archive_path, unzip_path):
   try:
     with zipfile.ZipFile(archive_path, 'r') as archive:
       VerifySafeArchive(archive)
+      assert os.path.isabs(unzip_path)
+      unzip_path_without_prefix = unzip_path
+      if sys.platform.startswith('win'):
+        unzip_path = u'\\\\?\\' + unzip_path
       archive.extractall(path=unzip_path)
       SetUnzippedDirPermissions(archive, unzip_path)
   except:
-    if unzip_path and os.path.isdir(unzip_path):
-      RemoveDir(unzip_path)
+    # Hack necessary because isdir doesn't work with escaped paths on Windows.
+    if unzip_path_without_prefix and os.path.isdir(unzip_path_without_prefix):
+      RemoveDir(unzip_path_without_prefix)
     raise
diff --git a/catapult/dependency_manager/dependency_manager/dependency_manager_util_unittest.py b/catapult/dependency_manager/dependency_manager/dependency_manager_util_unittest.py
index 19889ee3..c62bfd5e 100644
--- a/catapult/dependency_manager/dependency_manager/dependency_manager_util_unittest.py
+++ b/catapult/dependency_manager/dependency_manager/dependency_manager_util_unittest.py
@@ -135,6 +135,34 @@ class DependencyManagerUtilTest(unittest.TestCase):
     self.AssertExpectedDirFiles(self.tmp_dir)
     dependency_manager_util.RemoveDir(unzip_path)
 
+  def testUnzipFileContainingLongPath(self):
+    try:
+      dir_path = self.tmp_dir
+      if sys.platform.startswith('win'):
+        dir_path = u'\\\\?\\' + dir_path
+
+      archive_suffix = ''
+      # 260 is the Windows API path length limit.
+      while len(archive_suffix) < 260:
+        archive_suffix = os.path.join(archive_suffix, 'really')
+      contents_dir_path = os.path.join(dir_path, archive_suffix)
+      os.makedirs(contents_dir_path)
+      filename = os.path.join(contents_dir_path, 'longpath.txt')
+      open(filename, 'a').close()
+
+      base_path = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
+      archive_path = shutil.make_archive(base_path, 'zip', dir_path)
+      self.assertTrue(os.path.exists(archive_path))
+      self.assertTrue(zipfile.is_zipfile(archive_path))
+    except:
+      if os.path.isfile(archive_path):
+        os.remove(archive_path)
+      raise
+
+    unzip_path = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
+    dependency_manager_util.UnzipArchive(archive_path, unzip_path)
+    dependency_manager_util.RemoveDir(unzip_path)
+
   def testUnzipFileFailure(self):
     unzip_path = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
     self.assertFalse(os.path.exists(unzip_path))
diff --git a/catapult/dependency_manager/dependency_manager/exceptions.py b/catapult/dependency_manager/dependency_manager/exceptions.py
index 96cb4e65..d7863db7 100644
--- a/catapult/dependency_manager/dependency_manager/exceptions.py
+++ b/catapult/dependency_manager/dependency_manager/exceptions.py
@@ -2,7 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-from catapult_base import cloud_storage
+from py_utils import cloud_storage
 
 
 CloudStorageError = cloud_storage.CloudStorageError
diff --git a/catapult/dependency_manager/dependency_manager/uploader.py b/catapult/dependency_manager/dependency_manager/uploader.py
index 34b91f56..e88b7c04 100644
--- a/catapult/dependency_manager/dependency_manager/uploader.py
+++ b/catapult/dependency_manager/dependency_manager/uploader.py
@@ -5,7 +5,7 @@
 import logging
 import os
 
-from catapult_base import cloud_storage
+from py_utils import cloud_storage
 
 from dependency_manager import exceptions