diff options
Diffstat (limited to 'tools/telemetry/catapult_base/dependency_manager/base_config_unittest.py')
| -rw-r--r-- | tools/telemetry/catapult_base/dependency_manager/base_config_unittest.py | 1374 |
1 files changed, 1334 insertions, 40 deletions
diff --git a/tools/telemetry/catapult_base/dependency_manager/base_config_unittest.py b/tools/telemetry/catapult_base/dependency_manager/base_config_unittest.py index 5beee8c..e4fce4a 100644 --- a/tools/telemetry/catapult_base/dependency_manager/base_config_unittest.py +++ b/tools/telemetry/catapult_base/dependency_manager/base_config_unittest.py @@ -6,12 +6,1346 @@ import os import unittest import mock +from pyfakefs import fake_filesystem_unittest +from pyfakefs import fake_filesystem +from catapult_base import cloud_storage from catapult_base.dependency_manager import base_config from catapult_base.dependency_manager import dependency_info from catapult_base.dependency_manager import exceptions +from catapult_base.dependency_manager import uploader +class BaseConfigCreationAndUpdateUnittests(fake_filesystem_unittest.TestCase): + def setUp(self): + self.addTypeEqualityFunc(uploader.CloudStorageUploader, + uploader.CloudStorageUploader.__eq__) + self.setUpPyfakefs() + self.dependencies = { + 'dep1': {'cloud_storage_bucket': 'bucket1', + 'cloud_storage_base_folder': 'dependencies_folder', + 'file_info': { + 'plat1': { + 'cloud_storage_hash': 'hash11', + 'download_path': '../../relative/dep1/path1'}, + 'plat2': { + 'cloud_storage_hash': 'hash12', + 'download_path': '../../relative/dep1/path2'}}}, + 'dep2': {'cloud_storage_bucket': 'bucket2', + 'file_info': { + 'plat1': { + 'cloud_storage_hash': 'hash21', + 'download_path': '../../relative/dep2/path1'}, + 'plat2': { + 'cloud_storage_hash': 'hash22', + 'download_path': '../../relative/dep2/path2'}}}} + + self.expected_file_lines = [ + '{', '"config_type": "BaseConfig",', '"dependencies": {', + '"dep1": {', '"cloud_storage_base_folder": "dependencies_folder",', + '"cloud_storage_bucket": "bucket1",', '"file_info": {', + '"plat1": {', '"cloud_storage_hash": "hash11",', + '"download_path": "../../relative/dep1/path1"', '},', + '"plat2": {', '"cloud_storage_hash": "hash12",', + '"download_path": "../../relative/dep1/path2"', '}', '}', '},', + '"dep2": {', '"cloud_storage_bucket": "bucket2",', '"file_info": {', + '"plat1": {', '"cloud_storage_hash": "hash21",', + '"download_path": "../../relative/dep2/path1"', '},', + '"plat2": {', '"cloud_storage_hash": "hash22",', + '"download_path": "../../relative/dep2/path2"', '}', '}', '}', + '}', '}'] + + self.file_path = os.path.abspath(os.path.join( + 'path', 'to', 'config', 'file')) + + self.new_dep_path = 'path/to/new/dep' + self.fs.CreateFile(self.new_dep_path) + self.new_dep_hash = 'A23B56B7F23E798601F' + self.new_dependencies = { + 'dep1': {'cloud_storage_bucket': 'bucket1', + 'cloud_storage_base_folder': 'dependencies_folder', + 'file_info': { + 'plat1': { + 'cloud_storage_hash': 'hash11', + 'download_path': '../../relative/dep1/path1'}, + 'plat2': { + 'cloud_storage_hash': self.new_dep_hash, + 'download_path': '../../relative/dep1/path2'}}}, + 'dep2': {'cloud_storage_bucket': 'bucket2', + 'file_info': { + 'plat1': { + 'cloud_storage_hash': 'hash21', + 'download_path': '../../relative/dep2/path1'}, + 'plat2': { + 'cloud_storage_hash': 'hash22', + 'download_path': '../../relative/dep2/path2'}}}} + self.new_bucket = 'bucket1' + self.new_remote_path = 'dependencies_folder/dep1_%s' % self.new_dep_hash + self.new_pending_upload = uploader.CloudStorageUploader( + self.new_bucket, self.new_remote_path, self.new_dep_path) + self.expected_new_backup_path = '.'.join([self.new_remote_path, 'old']) + self.new_expected_file_lines = [ + '{', '"config_type": "BaseConfig",', '"dependencies": {', + '"dep1": {', '"cloud_storage_base_folder": "dependencies_folder",', + '"cloud_storage_bucket": "bucket1",', '"file_info": {', + '"plat1": {', '"cloud_storage_hash": "hash11",', + '"download_path": "../../relative/dep1/path1"', '},', + '"plat2": {', '"cloud_storage_hash": "%s",' % self.new_dep_hash, + '"download_path": "../../relative/dep1/path2"', '}', '}', '},', + '"dep2": {', '"cloud_storage_bucket": "bucket2",', '"file_info": {', + '"plat1": {', '"cloud_storage_hash": "hash21",', + '"download_path": "../../relative/dep2/path1"', '},', + '"plat2": {', '"cloud_storage_hash": "hash22",', + '"download_path": "../../relative/dep2/path2"', '}', '}', '}', + '}', '}'] + + self.final_dep_path = 'path/to/final/dep' + self.fs.CreateFile(self.final_dep_path) + self.final_dep_hash = 'B34662F23B56B7F98601F' + self.final_bucket = 'bucket2' + self.final_remote_path = 'dep1_%s' % self.final_dep_hash + self.final_pending_upload = uploader.CloudStorageUploader( + self.final_bucket, self.final_remote_path, self.final_dep_path) + self.expected_final_backup_path = '.'.join([self.final_remote_path, + 'old']) + self.final_dependencies = { + 'dep1': {'cloud_storage_bucket': 'bucket1', + 'cloud_storage_base_folder': 'dependencies_folder', + 'file_info': { + 'plat1': { + 'cloud_storage_hash': 'hash11', + 'download_path': '../../relative/dep1/path1'}, + 'plat2': { + 'cloud_storage_hash': self.new_dep_hash, + 'download_path': '../../relative/dep1/path2'}}}, + 'dep2': {'cloud_storage_bucket': 'bucket2', + 'file_info': { + 'plat1': { + 'cloud_storage_hash': self.final_dep_hash, + 'download_path': '../../relative/dep2/path1'}, + 'plat2': { + 'cloud_storage_hash': 'hash22', + 'download_path': '../../relative/dep2/path2'}}}} + self.final_expected_file_lines = [ + '{', '"config_type": "BaseConfig",', '"dependencies": {', + '"dep1": {', '"cloud_storage_base_folder": "dependencies_folder",', + '"cloud_storage_bucket": "bucket1",', '"file_info": {', + '"plat1": {', '"cloud_storage_hash": "hash11",', + '"download_path": "../../relative/dep1/path1"', '},', + '"plat2": {', '"cloud_storage_hash": "%s",' % self.new_dep_hash, + '"download_path": "../../relative/dep1/path2"', '}', '}', '},', + '"dep2": {', '"cloud_storage_bucket": "bucket2",', '"file_info": {', + '"plat1": {', '"cloud_storage_hash": "%s",' % self.final_dep_hash, + '"download_path": "../../relative/dep2/path1"', '},', + '"plat2": {', '"cloud_storage_hash": "hash22",', + '"download_path": "../../relative/dep2/path2"', '}', '}', '}', + '}', '}'] + + + def tearDown(self): + self.tearDownPyfakefs() + + # Init is not meant to be overridden, so we should be mocking the + # base_config's json module, even in subclasses. + def testCreateEmptyConfig(self): + expected_file_lines = ['{', + '"config_type": "BaseConfig",', + '"dependencies": {}', + '}'] + config = base_config.BaseConfig(self.file_path, writable=True) + + file_module = fake_filesystem.FakeFileOpen(self.fs) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + self.assertEqual({}, config._config_data) + self.assertEqual(self.file_path, config._config_path) + + def testCreateEmptyConfigError(self): + self.assertRaises(exceptions.EmptyConfigError, + base_config.BaseConfig, self.file_path) + + def testCloudStorageRemotePath(self): + dependency = 'dep_name' + cs_hash = self.new_dep_hash + cs_base_folder = 'dependency_remote_folder' + expected_remote_path = '%s/%s_%s' % (cs_base_folder, dependency, cs_hash) + remote_path = base_config.BaseConfig._CloudStorageRemotePath( + dependency, cs_hash, cs_base_folder) + self.assertEqual(expected_remote_path, remote_path) + + cs_base_folder = 'dependency_remote_folder' + expected_remote_path = '%s_%s' % (dependency, cs_hash) + remote_path = base_config.BaseConfig._CloudStorageRemotePath( + dependency, cs_hash, cs_base_folder) + + def testGetEmptyJsonDict(self): + expected_json_dict = {'config_type': 'BaseConfig', + 'dependencies': {}} + json_dict = base_config.BaseConfig._GetJsonDict() + self.assertEqual(expected_json_dict, json_dict) + + def testGetNonEmptyJsonDict(self): + expected_json_dict = {"config_type": "BaseConfig", + "dependencies": self.dependencies} + json_dict = base_config.BaseConfig._GetJsonDict(self.dependencies) + self.assertEqual(expected_json_dict, json_dict) + + def testWriteEmptyConfigToFile(self): + expected_file_lines = ['{', '"config_type": "BaseConfig",', + '"dependencies": {}', '}'] + self.assertFalse(os.path.exists(self.file_path)) + base_config.BaseConfig._WriteConfigToFile(self.file_path) + self.assertTrue(os.path.exists(self.file_path)) + file_module = fake_filesystem.FakeFileOpen(self.fs) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + + def testWriteNonEmptyConfigToFile(self): + self.assertFalse(os.path.exists(self.file_path)) + base_config.BaseConfig._WriteConfigToFile(self.file_path, self.dependencies) + self.assertTrue(os.path.exists(self.file_path)) + expected_file_lines = list(self.expected_file_lines) + file_module = fake_filesystem.FakeFileOpen(self.fs) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testExecuteUpdateJobsNoOp(self, base_config_cs_mock, uploader_cs_mock): + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + + self.assertFalse(config.ExecuteUpdateJobs()) + self.assertFalse(config._is_dirty) + self.assertFalse(config._pending_uploads) + self.assertEqual(self.dependencies, config._config_data) + file_module = fake_filesystem.FakeFileOpen(self.fs) + expected_file_lines = list(self.expected_file_lines) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testExecuteUpdateJobsFailureOnInsertNoCSCollision( + self, base_config_cs_mock, uploader_cs_mock): + uploader_cs_mock.Exists.return_value = False + uploader_cs_mock.Insert.side_effect = cloud_storage.CloudStorageError + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + config._config_data = self.new_dependencies.copy() + config._is_dirty = True + config._pending_uploads = [self.new_pending_upload] + self.assertEqual(self.new_dependencies, config._config_data) + self.assertTrue(config._is_dirty) + self.assertEqual(1, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path)] + expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, + self.new_dep_path)] + expected_copy_calls = [] + expected_delete_calls = [] + + self.assertRaises(cloud_storage.CloudStorageError, + config.ExecuteUpdateJobs) + self.assertTrue(config._is_dirty) + self.assertEqual(1, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.new_dependencies, config._config_data) + file_module = fake_filesystem.FakeFileOpen(self.fs) + expected_file_lines = list(self.expected_file_lines) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + self.assertEqual(1, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(expected_insert_calls, + uploader_cs_mock.Insert.call_args_list) + self.assertEqual(expected_exists_calls, + uploader_cs_mock.Exists.call_args_list) + self.assertEqual(expected_copy_calls, + uploader_cs_mock.Copy.call_args_list) + self.assertEqual(expected_delete_calls, + uploader_cs_mock.Delete.call_args_list) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testExecuteUpdateJobsFailureOnInsertCSCollisionForce( + self, base_config_cs_mock, uploader_cs_mock): + uploader_cs_mock.Exists.return_value = True + uploader_cs_mock.Insert.side_effect = cloud_storage.CloudStorageError + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + config._config_data = self.new_dependencies.copy() + config._is_dirty = True + config._pending_uploads = [self.new_pending_upload] + self.assertEqual(self.new_dependencies, config._config_data) + self.assertTrue(config._is_dirty) + self.assertEqual(1, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path)] + expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, + self.new_dep_path)] + expected_copy_calls = [mock.call(self.new_bucket, self.new_bucket, + self.new_remote_path, + self.expected_new_backup_path), + mock.call(self.new_bucket, self.new_bucket, + self.expected_new_backup_path, + self.new_remote_path)] + expected_delete_calls = [] + + self.assertRaises(cloud_storage.CloudStorageError, + config.ExecuteUpdateJobs, force=True) + self.assertTrue(config._is_dirty) + self.assertEqual(1, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.new_dependencies, config._config_data) + file_module = fake_filesystem.FakeFileOpen(self.fs) + expected_file_lines = list(self.expected_file_lines) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + self.assertEqual(1, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(expected_insert_calls, + uploader_cs_mock.Insert.call_args_list) + self.assertEqual(expected_exists_calls, + uploader_cs_mock.Exists.call_args_list) + self.assertEqual(expected_copy_calls, + uploader_cs_mock.Copy.call_args_list) + self.assertEqual(expected_delete_calls, + uploader_cs_mock.Delete.call_args_list) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testExecuteUpdateJobsFailureOnInsertCSCollisionNoForce( + self, base_config_cs_mock, uploader_cs_mock): + uploader_cs_mock.Exists.return_value = True + uploader_cs_mock.Insert.side_effect = cloud_storage.CloudStorageError + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + config._config_data = self.new_dependencies.copy() + config._is_dirty = True + config._pending_uploads = [self.new_pending_upload] + self.assertEqual(self.new_dependencies, config._config_data) + self.assertTrue(config._is_dirty) + self.assertEqual(1, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path)] + expected_insert_calls = [] + expected_copy_calls = [] + expected_delete_calls = [] + + self.assertRaises(cloud_storage.CloudStorageError, + config.ExecuteUpdateJobs) + self.assertTrue(config._is_dirty) + self.assertEqual(1, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.new_dependencies, config._config_data) + file_module = fake_filesystem.FakeFileOpen(self.fs) + expected_file_lines = list(self.expected_file_lines) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + self.assertEqual(1, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(expected_insert_calls, + uploader_cs_mock.Insert.call_args_list) + self.assertEqual(expected_exists_calls, + uploader_cs_mock.Exists.call_args_list) + self.assertEqual(expected_copy_calls, + uploader_cs_mock.Copy.call_args_list) + self.assertEqual(expected_delete_calls, + uploader_cs_mock.Delete.call_args_list) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testExecuteUpdateJobsFailureOnCopy( + self, base_config_cs_mock, uploader_cs_mock): + uploader_cs_mock.Exists.return_value = True + uploader_cs_mock.Copy.side_effect = cloud_storage.CloudStorageError + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + config._config_data = self.new_dependencies.copy() + config._is_dirty = True + config._pending_uploads = [self.new_pending_upload] + self.assertEqual(self.new_dependencies, config._config_data) + self.assertTrue(config._is_dirty) + self.assertEqual(1, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path)] + expected_insert_calls = [] + expected_copy_calls = [mock.call(self.new_bucket, self.new_bucket, + self.new_remote_path, + self.expected_new_backup_path)] + expected_delete_calls = [] + + self.assertRaises(cloud_storage.CloudStorageError, + config.ExecuteUpdateJobs, force=True) + self.assertTrue(config._is_dirty) + self.assertEqual(1, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.new_dependencies, config._config_data) + file_module = fake_filesystem.FakeFileOpen(self.fs) + expected_file_lines = list(self.expected_file_lines) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + self.assertEqual(1, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(expected_insert_calls, + uploader_cs_mock.Insert.call_args_list) + self.assertEqual(expected_exists_calls, + uploader_cs_mock.Exists.call_args_list) + self.assertEqual(expected_copy_calls, + uploader_cs_mock.Copy.call_args_list) + self.assertEqual(expected_delete_calls, + uploader_cs_mock.Delete.call_args_list) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testExecuteUpdateJobsFailureOnSecondInsertNoCSCollision( + self, base_config_cs_mock, uploader_cs_mock): + uploader_cs_mock.Exists.return_value = False + uploader_cs_mock.Insert.side_effect = [ + True, cloud_storage.CloudStorageError] + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + config._config_data = self.new_dependencies.copy() + config._is_dirty = True + config._pending_uploads = [self.new_pending_upload, + self.final_pending_upload] + self.assertEqual(self.new_dependencies, config._config_data) + self.assertTrue(config._is_dirty) + self.assertEqual(2, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) + expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path), + mock.call(self.final_bucket, + self.final_remote_path)] + expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, + self.new_dep_path), + mock.call(self.final_bucket, + self.final_remote_path, + self.final_dep_path)] + expected_copy_calls = [] + expected_delete_calls = [mock.call(self.new_bucket, self.new_remote_path)] + + self.assertRaises(cloud_storage.CloudStorageError, + config.ExecuteUpdateJobs) + self.assertTrue(config._is_dirty) + self.assertEqual(2, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) + self.assertEqual(self.new_dependencies, config._config_data) + file_module = fake_filesystem.FakeFileOpen(self.fs) + expected_file_lines = list(self.expected_file_lines) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + self.assertEqual(expected_insert_calls, + uploader_cs_mock.Insert.call_args_list) + self.assertEqual(expected_exists_calls, + uploader_cs_mock.Exists.call_args_list) + self.assertEqual(expected_copy_calls, + uploader_cs_mock.Copy.call_args_list) + self.assertEqual(expected_delete_calls, + uploader_cs_mock.Delete.call_args_list) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testExecuteUpdateJobsFailureOnSecondInsertCSCollisionForce( + self, base_config_cs_mock, uploader_cs_mock): + uploader_cs_mock.Exists.return_value = True + uploader_cs_mock.Insert.side_effect = [ + True, cloud_storage.CloudStorageError] + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + config._config_data = self.new_dependencies.copy() + config._is_dirty = True + config._pending_uploads = [self.new_pending_upload, + self.final_pending_upload] + self.assertEqual(self.new_dependencies, config._config_data) + self.assertTrue(config._is_dirty) + self.assertEqual(2, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) + expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path), + mock.call(self.final_bucket, + self.final_remote_path)] + expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, + self.new_dep_path), + mock.call(self.final_bucket, + self.final_remote_path, + self.final_dep_path)] + expected_copy_calls = [mock.call(self.new_bucket, self.new_bucket, + self.new_remote_path, + self.expected_new_backup_path), + mock.call(self.final_bucket, self.final_bucket, + self.final_remote_path, + self.expected_final_backup_path), + mock.call(self.final_bucket, self.final_bucket, + self.expected_final_backup_path, + self.final_remote_path), + mock.call(self.new_bucket, self.new_bucket, + self.expected_new_backup_path, + self.new_remote_path)] + expected_delete_calls = [] + + self.assertRaises(cloud_storage.CloudStorageError, + config.ExecuteUpdateJobs, force=True) + self.assertTrue(config._is_dirty) + self.assertEqual(2, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) + self.assertEqual(self.new_dependencies, config._config_data) + file_module = fake_filesystem.FakeFileOpen(self.fs) + expected_file_lines = list(self.expected_file_lines) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + self.assertEqual(expected_insert_calls, + uploader_cs_mock.Insert.call_args_list) + self.assertEqual(expected_exists_calls, + uploader_cs_mock.Exists.call_args_list) + self.assertEqual(expected_copy_calls, + uploader_cs_mock.Copy.call_args_list) + self.assertEqual(expected_delete_calls, + uploader_cs_mock.Delete.call_args_list) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testExecuteUpdateJobsFailureOnSecondInsertFirstCSCollisionForce( + self, base_config_cs_mock, uploader_cs_mock): + uploader_cs_mock.Exists.side_effect = [True, False, True] + uploader_cs_mock.Insert.side_effect = [ + True, cloud_storage.CloudStorageError] + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + config._config_data = self.new_dependencies.copy() + config._is_dirty = True + config._pending_uploads = [self.new_pending_upload, + self.final_pending_upload] + self.assertEqual(self.new_dependencies, config._config_data) + self.assertTrue(config._is_dirty) + self.assertEqual(2, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) + expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path), + mock.call(self.final_bucket, + self.final_remote_path)] + expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, + self.new_dep_path), + mock.call(self.final_bucket, + self.final_remote_path, + self.final_dep_path)] + expected_copy_calls = [mock.call(self.new_bucket, self.new_bucket, + self.new_remote_path, + self.expected_new_backup_path), + mock.call(self.new_bucket, self.new_bucket, + self.expected_new_backup_path, + self.new_remote_path)] + expected_delete_calls = [] + + self.assertRaises(cloud_storage.CloudStorageError, + config.ExecuteUpdateJobs, force=True) + self.assertTrue(config._is_dirty) + self.assertEqual(2, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) + self.assertEqual(self.new_dependencies, config._config_data) + file_module = fake_filesystem.FakeFileOpen(self.fs) + expected_file_lines = list(self.expected_file_lines) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + self.assertEqual(expected_insert_calls, + uploader_cs_mock.Insert.call_args_list) + self.assertEqual(expected_exists_calls, + uploader_cs_mock.Exists.call_args_list) + self.assertEqual(expected_copy_calls, + uploader_cs_mock.Copy.call_args_list) + self.assertEqual(expected_delete_calls, + uploader_cs_mock.Delete.call_args_list) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testExecuteUpdateJobsFailureOnFirstCSCollisionNoForce( + self, base_config_cs_mock, uploader_cs_mock): + uploader_cs_mock.Exists.side_effect = [True, False, True] + uploader_cs_mock.Insert.side_effect = [ + True, cloud_storage.CloudStorageError] + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + config._config_data = self.new_dependencies.copy() + config._is_dirty = True + config._pending_uploads = [self.new_pending_upload, + self.final_pending_upload] + self.assertEqual(self.new_dependencies, config._config_data) + self.assertTrue(config._is_dirty) + self.assertEqual(2, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) + expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path)] + expected_insert_calls = [] + expected_copy_calls = [] + expected_delete_calls = [] + + self.assertRaises(cloud_storage.CloudStorageError, + config.ExecuteUpdateJobs) + self.assertTrue(config._is_dirty) + self.assertEqual(2, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) + self.assertEqual(self.new_dependencies, config._config_data) + file_module = fake_filesystem.FakeFileOpen(self.fs) + expected_file_lines = list(self.expected_file_lines) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + self.assertEqual(expected_insert_calls, + uploader_cs_mock.Insert.call_args_list) + self.assertEqual(expected_exists_calls, + uploader_cs_mock.Exists.call_args_list) + self.assertEqual(expected_copy_calls, + uploader_cs_mock.Copy.call_args_list) + self.assertEqual(expected_delete_calls, + uploader_cs_mock.Delete.call_args_list) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testExecuteUpdateJobsFailureOnSecondCopyCSCollision( + self, base_config_cs_mock, uploader_cs_mock): + uploader_cs_mock.Exists.return_value = True + uploader_cs_mock.Insert.return_value = True + uploader_cs_mock.Copy.side_effect = [ + True, cloud_storage.CloudStorageError, True] + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + config._config_data = self.new_dependencies.copy() + config._is_dirty = True + config._pending_uploads = [self.new_pending_upload, + self.final_pending_upload] + self.assertEqual(self.new_dependencies, config._config_data) + self.assertTrue(config._is_dirty) + self.assertEqual(2, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) + expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path), + mock.call(self.final_bucket, + self.final_remote_path)] + expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, + self.new_dep_path)] + expected_copy_calls = [mock.call(self.new_bucket, self.new_bucket, + self.new_remote_path, + self.expected_new_backup_path), + mock.call(self.final_bucket, self.final_bucket, + self.final_remote_path, + self.expected_final_backup_path), + mock.call(self.new_bucket, self.new_bucket, + self.expected_new_backup_path, + self.new_remote_path)] + expected_delete_calls = [] + + self.assertRaises(cloud_storage.CloudStorageError, + config.ExecuteUpdateJobs, force=True) + self.assertTrue(config._is_dirty) + self.assertEqual(2, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) + self.assertEqual(self.new_dependencies, config._config_data) + file_module = fake_filesystem.FakeFileOpen(self.fs) + expected_file_lines = list(self.expected_file_lines) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + self.assertEqual(expected_insert_calls, + uploader_cs_mock.Insert.call_args_list) + self.assertEqual(expected_exists_calls, + uploader_cs_mock.Exists.call_args_list) + self.assertEqual(expected_copy_calls, + uploader_cs_mock.Copy.call_args_list) + self.assertEqual(expected_delete_calls, + uploader_cs_mock.Delete.call_args_list) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testExecuteUpdateJobsFailureOnSecondCopyNoCSCollisionForce( + self, base_config_cs_mock, uploader_cs_mock): + uploader_cs_mock.Exists.side_effect = [False, True, False] + uploader_cs_mock.Copy.side_effect = cloud_storage.CloudStorageError + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + config._config_data = self.new_dependencies.copy() + config._is_dirty = True + config._pending_uploads = [self.new_pending_upload, + self.final_pending_upload] + self.assertEqual(self.new_dependencies, config._config_data) + self.assertTrue(config._is_dirty) + self.assertEqual(2, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) + expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path), + mock.call(self.final_bucket, + self.final_remote_path)] + expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, + self.new_dep_path)] + expected_copy_calls = [mock.call(self.final_bucket, self.final_bucket, + self.final_remote_path, + self.expected_final_backup_path)] + expected_delete_calls = [mock.call(self.new_bucket, self.new_remote_path)] + + self.assertRaises(cloud_storage.CloudStorageError, + config.ExecuteUpdateJobs, force=True) + self.assertTrue(config._is_dirty) + self.assertEqual(2, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) + self.assertEqual(self.new_dependencies, config._config_data) + file_module = fake_filesystem.FakeFileOpen(self.fs) + expected_file_lines = list(self.expected_file_lines) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + self.assertEqual(expected_insert_calls, + uploader_cs_mock.Insert.call_args_list) + self.assertEqual(expected_exists_calls, + uploader_cs_mock.Exists.call_args_list) + self.assertEqual(expected_copy_calls, + uploader_cs_mock.Copy.call_args_list) + self.assertEqual(expected_delete_calls, + uploader_cs_mock.Delete.call_args_list) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testExecuteUpdateJobsFailureOnSecondCopyNoCSCollisionNoForce( + self, base_config_cs_mock, uploader_cs_mock): + uploader_cs_mock.Exists.side_effect = [False, True, False] + uploader_cs_mock.Copy.side_effect = cloud_storage.CloudStorageError + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + config._config_data = self.new_dependencies.copy() + config._is_dirty = True + config._pending_uploads = [self.new_pending_upload, + self.final_pending_upload] + self.assertEqual(self.new_dependencies, config._config_data) + self.assertTrue(config._is_dirty) + self.assertEqual(2, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) + expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path), + mock.call(self.final_bucket, + self.final_remote_path)] + expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, + self.new_dep_path)] + expected_copy_calls = [] + expected_delete_calls = [mock.call(self.new_bucket, self.new_remote_path)] + + self.assertRaises(cloud_storage.CloudStorageError, + config.ExecuteUpdateJobs) + self.assertTrue(config._is_dirty) + self.assertEqual(2, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) + self.assertEqual(self.new_dependencies, config._config_data) + file_module = fake_filesystem.FakeFileOpen(self.fs) + expected_file_lines = list(self.expected_file_lines) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + self.assertEqual(expected_insert_calls, + uploader_cs_mock.Insert.call_args_list) + self.assertEqual(expected_exists_calls, + uploader_cs_mock.Exists.call_args_list) + self.assertEqual(expected_copy_calls, + uploader_cs_mock.Copy.call_args_list) + self.assertEqual(expected_delete_calls, + uploader_cs_mock.Delete.call_args_list) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testExecuteUpdateJobsSuccessOnePendingDepNoCloudStorageCollision( + self, base_config_cs_mock, uploader_cs_mock): + uploader_cs_mock.Exists.return_value = False + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + config._config_data = self.new_dependencies.copy() + config._is_dirty = True + config._pending_uploads = [self.new_pending_upload] + self.assertEqual(self.new_dependencies, config._config_data) + self.assertTrue(config._is_dirty) + self.assertEqual(1, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path)] + expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, + self.new_dep_path)] + expected_copy_calls = [] + expected_delete_calls = [] + + self.assertTrue(config.ExecuteUpdateJobs()) + self.assertFalse(config._is_dirty) + self.assertFalse(config._pending_uploads) + self.assertEqual(self.new_dependencies, config._config_data) + file_module = fake_filesystem.FakeFileOpen(self.fs) + expected_file_lines = list(self.new_expected_file_lines) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + self.assertFalse(config._pending_uploads) + self.assertEqual(expected_insert_calls, + uploader_cs_mock.Insert.call_args_list) + self.assertEqual(expected_exists_calls, + uploader_cs_mock.Exists.call_args_list) + self.assertEqual(expected_copy_calls, + uploader_cs_mock.Copy.call_args_list) + self.assertEqual(expected_delete_calls, + uploader_cs_mock.Delete.call_args_list) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testExecuteUpdateJobsSuccessOnePendingDepCloudStorageCollision( + self, base_config_cs_mock, uploader_cs_mock): + uploader_cs_mock.Exists.return_value = True + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + config._config_data = self.new_dependencies.copy() + config._is_dirty = True + config._pending_uploads = [self.new_pending_upload] + self.assertEqual(self.new_dependencies, config._config_data) + self.assertTrue(config._is_dirty) + self.assertEqual(1, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path)] + expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, + self.new_dep_path)] + expected_copy_calls = [mock.call(self.new_bucket, self.new_bucket, + self.new_remote_path, + self.expected_new_backup_path)] + + self.assertTrue(config.ExecuteUpdateJobs(force=True)) + self.assertFalse(config._is_dirty) + self.assertFalse(config._pending_uploads) + self.assertEqual(self.new_dependencies, config._config_data) + file_module = fake_filesystem.FakeFileOpen(self.fs) + expected_file_lines = list(self.new_expected_file_lines) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + self.assertFalse(config._pending_uploads) + self.assertEqual(expected_insert_calls, + uploader_cs_mock.Insert.call_args_list) + self.assertEqual(expected_exists_calls, + uploader_cs_mock.Exists.call_args_list) + self.assertEqual(expected_copy_calls, + uploader_cs_mock.Copy.call_args_list) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testExecuteUpdateJobsErrorOnePendingDepCloudStorageCollisionNoForce( + self, base_config_cs_mock, uploader_cs_mock): + uploader_cs_mock.Exists.return_value = True + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + config._config_data = self.new_dependencies.copy() + config._is_dirty = True + config._pending_uploads = [self.new_pending_upload] + self.assertEqual(self.new_dependencies, config._config_data) + self.assertTrue(config._is_dirty) + self.assertEqual(1, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path)] + expected_insert_calls = [] + expected_copy_calls = [] + + self.assertRaises(exceptions.CloudStorageUploadConflictError, + config.ExecuteUpdateJobs) + self.assertTrue(config._is_dirty) + self.assertTrue(config._pending_uploads) + self.assertEqual(self.new_dependencies, config._config_data) + self.assertEqual(1, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + file_module = fake_filesystem.FakeFileOpen(self.fs) + expected_file_lines = list(self.expected_file_lines) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + self.assertEqual(expected_insert_calls, + uploader_cs_mock.Insert.call_args_list) + self.assertEqual(expected_exists_calls, + uploader_cs_mock.Exists.call_args_list) + self.assertEqual(expected_copy_calls, + uploader_cs_mock.Copy.call_args_list) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testExecuteUpdateJobsSuccessMultiplePendingDepsOneCloudStorageCollision( + self, base_config_cs_mock, uploader_cs_mock): + uploader_cs_mock.Exists.side_effect = [False, True] + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + config._config_data = self.final_dependencies.copy() + config._is_dirty = True + config._pending_uploads = [self.new_pending_upload, + self.final_pending_upload] + self.assertEqual(self.final_dependencies, config._config_data) + self.assertTrue(config._is_dirty) + self.assertEqual(2, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(self.final_pending_upload, config._pending_uploads[1]) + + expected_exists_calls = [mock.call(self.new_bucket, self.new_remote_path), + mock.call(self.final_bucket, + self.final_remote_path)] + expected_insert_calls = [mock.call(self.new_bucket, self.new_remote_path, + self.new_dep_path), + mock.call(self.final_bucket, + self.final_remote_path, + self.final_dep_path)] + expected_copy_calls = [mock.call(self.final_bucket, self.final_bucket, + self.final_remote_path, + self.expected_final_backup_path)] + + self.assertTrue(config.ExecuteUpdateJobs(force=True)) + self.assertFalse(config._is_dirty) + self.assertFalse(config._pending_uploads) + self.assertEqual(self.final_dependencies, config._config_data) + file_module = fake_filesystem.FakeFileOpen(self.fs) + expected_file_lines = list(self.final_expected_file_lines) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + self.assertFalse(config._pending_uploads) + self.assertEqual(expected_insert_calls, + uploader_cs_mock.Insert.call_args_list) + self.assertEqual(expected_exists_calls, + uploader_cs_mock.Exists.call_args_list) + self.assertEqual(expected_copy_calls, + uploader_cs_mock.Copy.call_args_list) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testUpdateCloudStorageDependenciesReadOnlyConfig( + self, base_config_cs_mock, uploader_cs_mock): + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path) + self.assertRaises( + exceptions.ReadWriteError, config.AddCloudStorageDependencyUpdateJob, + 'dep', 'plat', 'path') + self.assertRaises( + exceptions.ReadWriteError, config.AddCloudStorageDependencyUpdateJob, + 'dep', 'plat', 'path', version='1.2.3') + self.assertRaises( + exceptions.ReadWriteError, config.AddCloudStorageDependencyUpdateJob, + 'dep', 'plat', 'path', execute_job=False) + self.assertRaises( + exceptions.ReadWriteError, config.AddCloudStorageDependencyUpdateJob, + 'dep', 'plat', 'path', version='1.2.3', execute_job=False) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testUpdateCloudStorageDependenciesMissingDependency( + self, base_config_cs_mock, uploader_cs_mock): + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + self.assertRaises(ValueError, config.AddCloudStorageDependencyUpdateJob, + 'dep', 'plat', 'path') + self.assertRaises(ValueError, config.AddCloudStorageDependencyUpdateJob, + 'dep', 'plat', 'path', version='1.2.3') + self.assertRaises(ValueError, config.AddCloudStorageDependencyUpdateJob, + 'dep', 'plat', 'path', execute_job=False) + self.assertRaises(ValueError, config.AddCloudStorageDependencyUpdateJob, + 'dep', 'plat', 'path', version='1.2.3', execute_job=False) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testUpdateCloudStorageDependenciesWrite( + self, base_config_cs_mock, uploader_cs_mock): + expected_dependencies = self.dependencies + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + self.assertFalse(config._is_dirty) + self.assertEqual(expected_dependencies, config._config_data) + + base_config_cs_mock.CalculateHash.return_value = self.new_dep_hash + uploader_cs_mock.Exists.return_value = False + expected_dependencies = self.new_dependencies + config.AddCloudStorageDependencyUpdateJob( + 'dep1', 'plat2', self.new_dep_path, execute_job=True) + self.assertFalse(config._is_dirty) + self.assertFalse(config._pending_uploads) + self.assertEqual(expected_dependencies, config._config_data) + # check that file contents has been updated + file_module = fake_filesystem.FakeFileOpen(self.fs) + expected_file_lines = list(self.new_expected_file_lines) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + + expected_dependencies = self.final_dependencies + base_config_cs_mock.CalculateHash.return_value = self.final_dep_hash + config.AddCloudStorageDependencyUpdateJob( + 'dep2', 'plat1', self.final_dep_path, execute_job=True) + self.assertFalse(config._is_dirty) + self.assertFalse(config._pending_uploads) + self.assertEqual(expected_dependencies, config._config_data) + # check that file contents has been updated + expected_file_lines = list(self.final_expected_file_lines) + file_module = fake_filesystem.FakeFileOpen(self.fs) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + + @mock.patch('catapult_base.dependency_manager.uploader.cloud_storage') + @mock.patch('catapult_base.dependency_manager.base_config.cloud_storage') + def testUpdateCloudStorageDependenciesNoWrite( + self, base_config_cs_mock, uploader_cs_mock): + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + config = base_config.BaseConfig(self.file_path, writable=True) + + self.assertRaises(ValueError, config.AddCloudStorageDependencyUpdateJob, + 'dep', 'plat', 'path') + self.assertRaises(ValueError, config.AddCloudStorageDependencyUpdateJob, + 'dep', 'plat', 'path', version='1.2.3') + + expected_dependencies = self.dependencies + config = base_config.BaseConfig(self.file_path, writable=True) + self.assertFalse(config._is_dirty) + self.assertFalse(config._pending_uploads) + self.assertEqual(expected_dependencies, config._config_data) + + base_config_cs_mock.CalculateHash.return_value = self.new_dep_hash + uploader_cs_mock.Exists.return_value = False + expected_dependencies = self.new_dependencies + config.AddCloudStorageDependencyUpdateJob( + 'dep1', 'plat2', self.new_dep_path, execute_job=False) + self.assertTrue(config._is_dirty) + self.assertEqual(1, len(config._pending_uploads)) + self.assertEqual(self.new_pending_upload, config._pending_uploads[0]) + self.assertEqual(expected_dependencies, config._config_data) + # check that file contents have not been updated. + expected_file_lines = list(self.expected_file_lines) + file_module = fake_filesystem.FakeFileOpen(self.fs) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + + expected_dependencies = self.final_dependencies + base_config_cs_mock.CalculateHash.return_value = self.final_dep_hash + config.AddCloudStorageDependencyUpdateJob( + 'dep2', 'plat1', self.final_dep_path, execute_job=False) + self.assertTrue(config._is_dirty) + self.assertEqual(expected_dependencies, config._config_data) + # check that file contents have not been updated. + expected_file_lines = list(self.expected_file_lines) + file_module = fake_filesystem.FakeFileOpen(self.fs) + for line in file_module(self.file_path): + self.assertEqual(expected_file_lines.pop(0), line.strip()) + self.fs.CloseOpenFile(file_module(self.file_path)) + + +class BaseConfigDataManipulationUnittests(fake_filesystem_unittest.TestCase): + def setUp(self): + self.addTypeEqualityFunc(uploader.CloudStorageUploader, + uploader.CloudStorageUploader.__eq__) + self.setUpPyfakefs() + + self.cs_bucket = 'bucket1' + self.cs_base_folder = 'dependencies_folder' + self.cs_hash = 'hash12' + self.download_path = '../../relative/dep1/path2' + self.local_paths = ['../../../relative/local/path21', + '../../../relative/local/path22'] + self.platform_dict = {'cloud_storage_hash': self.cs_hash, + 'download_path': self.download_path, + 'local_paths': self.local_paths} + self.dependencies = { + 'dep1': {'cloud_storage_bucket': self.cs_bucket, + 'cloud_storage_base_folder': self.cs_base_folder, + 'file_info': { + 'plat1': { + 'cloud_storage_hash': 'hash11', + 'download_path': '../../relative/dep1/path1', + 'local_paths': ['../../../relative/local/path11', + '../../../relative/local/path12']}, + 'plat2': self.platform_dict + } + }, + 'dep2': {'cloud_storage_bucket': 'bucket2', + 'file_info': { + 'plat1': { + 'cloud_storage_hash': 'hash21', + 'download_path': '../../relative/dep2/path1', + 'local_paths': ['../../../relative/local/path31', + '../../../relative/local/path32']}, + 'plat2': { + 'cloud_storage_hash': 'hash22', + 'download_path': '../../relative/dep2/path2'}}}} + + self.file_path = os.path.abspath(os.path.join( + 'path', 'to', 'config', 'file')) + + + self.expected_file_lines = [ + '{', '"config_type": "BaseConfig",', '"dependencies": {', + '"dep1": {', '"cloud_storage_base_folder": "dependencies_folder",', + '"cloud_storage_bucket": "bucket1",', '"file_info": {', + '"plat1": {', '"cloud_storage_hash": "hash11",', + '"download_path": "../../relative/dep1/path1",', + '"local_paths": [', '"../../../relative/local/path11",', + '"../../../relative/local/path12"', ']', '},', + '"plat2": {', '"cloud_storage_hash": "hash12",', + '"download_path": "../../relative/dep1/path2",', + '"local_paths": [', '"../../../relative/local/path21",', + '"../../../relative/local/path22"', ']', + '}', '}', '},', + '"dep2": {', '"cloud_storage_bucket": "bucket2",', '"file_info": {', + '"plat1": {', '"cloud_storage_hash": "hash21",', + '"download_path": "../../relative/dep2/path1",', + '"local_paths": [', '"../../../relative/local/path31",', + '"../../../relative/local/path32"', ']', '},', + '"plat2": {', '"cloud_storage_hash": "hash22",', + '"download_path": "../../relative/dep2/path2"', '}', '}', '}', + '}', '}'] + self.fs.CreateFile(self.file_path, + contents='\n'.join(self.expected_file_lines)) + + + def testSetPlatformDataFailureNotWritable(self): + config = base_config.BaseConfig(self.file_path) + self.assertRaises(exceptions.ReadWriteError, config._SetPlatformData, + 'dep1', 'plat1', 'cloud_storage_bucket', 'new_bucket') + self.assertEqual(self.dependencies, config._config_data) + + def testSetPlatformDataFailure(self): + config = base_config.BaseConfig(self.file_path, writable=True) + self.assertRaises(ValueError, config._SetPlatformData, 'missing_dep', + 'plat2', 'cloud_storage_bucket', 'new_bucket') + self.assertEqual(self.dependencies, config._config_data) + self.assertRaises(ValueError, config._SetPlatformData, 'dep1', + 'missing_plat', 'cloud_storage_bucket', 'new_bucket') + self.assertEqual(self.dependencies, config._config_data) + + + def testSetPlatformDataCloudStorageBucketSuccess(self): + config = base_config.BaseConfig(self.file_path, writable=True) + updated_cs_dependencies = { + 'dep1': {'cloud_storage_bucket': 'new_bucket', + 'cloud_storage_base_folder': 'dependencies_folder', + 'file_info': { + 'plat1': { + 'cloud_storage_hash': 'hash11', + 'download_path': '../../relative/dep1/path1', + 'local_paths': ['../../../relative/local/path11', + '../../../relative/local/path12']}, + 'plat2': { + 'cloud_storage_hash': 'hash12', + 'download_path': '../../relative/dep1/path2', + 'local_paths': ['../../../relative/local/path21', + '../../../relative/local/path22']}}}, + 'dep2': {'cloud_storage_bucket': 'bucket2', + 'file_info': { + 'plat1': { + 'cloud_storage_hash': 'hash21', + 'download_path': '../../relative/dep2/path1', + 'local_paths': ['../../../relative/local/path31', + '../../../relative/local/path32']}, + 'plat2': { + 'cloud_storage_hash': 'hash22', + 'download_path': '../../relative/dep2/path2'}}}} + config._SetPlatformData('dep1', 'plat2', 'cloud_storage_bucket', + 'new_bucket') + self.assertEqual(updated_cs_dependencies, config._config_data) + + def testSetPlatformDataCloudStorageBaseFolderSuccess(self): + config = base_config.BaseConfig(self.file_path, writable=True) + updated_cs_dependencies = { + 'dep1': {'cloud_storage_bucket': 'bucket1', + 'cloud_storage_base_folder': 'new_dependencies_folder', + 'file_info': { + 'plat1': { + 'cloud_storage_hash': 'hash11', + 'download_path': '../../relative/dep1/path1', + 'local_paths': ['../../../relative/local/path11', + '../../../relative/local/path12']}, + 'plat2': { + 'cloud_storage_hash': 'hash12', + 'download_path': '../../relative/dep1/path2', + 'local_paths': ['../../../relative/local/path21', + '../../../relative/local/path22']}}}, + 'dep2': {'cloud_storage_bucket': 'bucket2', + 'file_info': { + 'plat1': { + 'cloud_storage_hash': 'hash21', + 'download_path': '../../relative/dep2/path1', + 'local_paths': ['../../../relative/local/path31', + '../../../relative/local/path32']}, + 'plat2': { + 'cloud_storage_hash': 'hash22', + 'download_path': '../../relative/dep2/path2'}}}} + config._SetPlatformData('dep1', 'plat2', 'cloud_storage_base_folder', + 'new_dependencies_folder') + self.assertEqual(updated_cs_dependencies, config._config_data) + + def testSetPlatformDataHashSuccess(self): + self.maxDiff = None + config = base_config.BaseConfig(self.file_path, writable=True) + updated_cs_dependencies = { + 'dep1': {'cloud_storage_bucket': 'bucket1', + 'cloud_storage_base_folder': 'dependencies_folder', + 'file_info': { + 'plat1': { + 'cloud_storage_hash': 'hash11', + 'download_path': '../../relative/dep1/path1', + 'local_paths': ['../../../relative/local/path11', + '../../../relative/local/path12']}, + 'plat2': { + 'cloud_storage_hash': 'new_hash', + 'download_path': '../../relative/dep1/path2', + 'local_paths': ['../../../relative/local/path21', + '../../../relative/local/path22']}}}, + 'dep2': {'cloud_storage_bucket': 'bucket2', + 'file_info': { + 'plat1': { + 'cloud_storage_hash': 'hash21', + 'download_path': '../../relative/dep2/path1', + 'local_paths': ['../../../relative/local/path31', + '../../../relative/local/path32']}, + 'plat2': { + 'cloud_storage_hash': 'hash22', + 'download_path': '../../relative/dep2/path2'}}}} + config._SetPlatformData('dep1', 'plat2', 'cloud_storage_hash', + 'new_hash') + self.assertEqual(updated_cs_dependencies, config._config_data) + + def testSetPlatformDataDownloadPathSuccess(self): + config = base_config.BaseConfig(self.file_path, writable=True) + updated_cs_dependencies = { + 'dep1': {'cloud_storage_bucket': 'bucket1', + 'cloud_storage_base_folder': 'dependencies_folder', + 'file_info': { + 'plat1': { + 'cloud_storage_hash': 'hash11', + 'download_path': '../../relative/dep1/path1', + 'local_paths': ['../../../relative/local/path11', + '../../../relative/local/path12']}, + 'plat2': { + 'cloud_storage_hash': 'hash12', + 'download_path': '../../new/dep1/path2', + 'local_paths': ['../../../relative/local/path21', + '../../../relative/local/path22']}}}, + 'dep2': {'cloud_storage_bucket': 'bucket2', + 'file_info': { + 'plat1': { + 'cloud_storage_hash': 'hash21', + 'download_path': '../../relative/dep2/path1', + 'local_paths': ['../../../relative/local/path31', + '../../../relative/local/path32']}, + 'plat2': { + 'cloud_storage_hash': 'hash22', + 'download_path': '../../relative/dep2/path2'}}}} + config._SetPlatformData('dep1', 'plat2', 'download_path', + '../../new/dep1/path2') + self.assertEqual(updated_cs_dependencies, config._config_data) + + def testSetPlatformDataLocalPathSuccess(self): + config = base_config.BaseConfig(self.file_path, writable=True) + updated_cs_dependencies = { + 'dep1': {'cloud_storage_bucket': 'bucket1', + 'cloud_storage_base_folder': 'dependencies_folder', + 'file_info': { + 'plat1': { + 'cloud_storage_hash': 'hash11', + 'download_path': '../../relative/dep1/path1', + 'local_paths': ['../../../relative/local/path11', + '../../../relative/local/path12']}, + 'plat2': { + 'cloud_storage_hash': 'hash12', + 'download_path': '../../relative/dep1/path2', + 'local_paths': ['../../new/relative/local/path21', + '../../new/relative/local/path22']}}}, + 'dep2': {'cloud_storage_bucket': 'bucket2', + 'file_info': { + 'plat1': { + 'cloud_storage_hash': 'hash21', + 'download_path': '../../relative/dep2/path1', + 'local_paths': ['../../../relative/local/path31', + '../../../relative/local/path32']}, + 'plat2': { + 'cloud_storage_hash': 'hash22', + 'download_path': '../../relative/dep2/path2'}}}} + config._SetPlatformData('dep1', 'plat2', 'local_paths', + ['../../new/relative/local/path21', + '../../new/relative/local/path22']) + self.assertEqual(updated_cs_dependencies, config._config_data) + + def testGetPlatformDataFailure(self): + config = base_config.BaseConfig(self.file_path, writable=True) + self.assertRaises(ValueError, config._GetPlatformData, 'missing_dep', + 'plat2', 'cloud_storage_bucket') + self.assertEqual(self.dependencies, config._config_data) + self.assertRaises(ValueError, config._GetPlatformData, 'dep1', + 'missing_plat', 'cloud_storage_bucket') + self.assertEqual(self.dependencies, config._config_data) + + def testGetPlatformDataDictSuccess(self): + config = base_config.BaseConfig(self.file_path, writable=True) + self.assertEqual(self.platform_dict, + config._GetPlatformData('dep1', 'plat2')) + self.assertEqual(self.dependencies, config._config_data) + + def testGetPlatformDataCloudStorageBucketSuccess(self): + config = base_config.BaseConfig(self.file_path, writable=True) + self.assertEqual(self.cs_bucket, config._GetPlatformData( + 'dep1', 'plat2', 'cloud_storage_bucket')) + self.assertEqual(self.dependencies, config._config_data) + + def testGetPlatformDataCloudStorageBaseFolderSuccess(self): + config = base_config.BaseConfig(self.file_path, writable=True) + self.assertEqual(self.cs_base_folder, config._GetPlatformData( + 'dep1', 'plat2', 'cloud_storage_base_folder')) + self.assertEqual(self.dependencies, config._config_data) + + def testGetPlatformDataHashSuccess(self): + config = base_config.BaseConfig(self.file_path, writable=True) + self.assertEqual(self.cs_hash, config._GetPlatformData( + 'dep1', 'plat2', 'cloud_storage_hash')) + self.assertEqual(self.dependencies, config._config_data) + + def testGetPlatformDataDownloadPathSuccess(self): + config = base_config.BaseConfig(self.file_path, writable=True) + self.assertEqual(self.download_path, config._GetPlatformData( + 'dep1', 'plat2', 'download_path')) + self.assertEqual(self.dependencies, config._config_data) + + def testGetPlatformDataLocalPathSuccess(self): + config = base_config.BaseConfig(self.file_path, writable=True) + self.assertEqual(self.local_paths, config._GetPlatformData( + 'dep1', 'plat2', 'local_paths')) + self.assertEqual(self.dependencies, config._config_data) + class BaseConfigTest(unittest.TestCase): """ Subclassable unittests for BaseConfig. For subclasses: override setUp, GetConfigDataFromDict, @@ -67,46 +1401,6 @@ class BaseConfigTest(unittest.TestCase): def GetConfigDataFromDict(self, config_dict): return config_dict.get('dependencies', {}) - - # Init is not meant to be overridden, so we should be mocking the - # base_config's json module, even in subclasses. - @mock.patch('catapult_base.dependency_manager.base_config.json.dump') - @mock.patch('os.path.exists') - @mock.patch('__builtin__.open') - def testCreateEmptyConfig(self, open_mock, exists_mock, dump_mock): - exists_mock.return_value = False - expected_dump = mock.call(self.empty_dict, mock.ANY, sort_keys=True, - indent=2) - expected_open = mock.call('file_path', 'w') - config_dict = self.config_class.CreateEmptyConfig('file_path') - self.assertEqual(dump_mock.call_args, expected_dump) - self.assertEqual(expected_open, open_mock.call_args) - self.assertEqual(self.empty_dict, config_dict) - - exists_mock.return_value = True - self.assertRaises(ValueError, - self.config_class.CreateEmptyConfig, 'file_path') - - - # Init is not meant to be overridden, so we should be mocking the - # base_config's json module, even in subclasses. - @mock.patch( - 'catapult_base.dependency_manager.base_config.BaseConfig.CreateEmptyConfig') #pylint: disable=line-too-long - @mock.patch('catapult_base.dependency_manager.base_config.json') - @mock.patch('os.path') - @mock.patch('__builtin__.open') - def testInitNoFile(self, open_mock, path_mock, json_mock, create_config_mock): - path_mock.exists.return_value = False - # Writable config. - config = self.config_class('file_path', writable=True) - self.assertEqual(self.GetConfigDataFromDict(self.empty_dict), - config._config_data) - # Not writable config. - self.assertRaises(exceptions.EmptyConfigError, - self.config_class, 'file_path') - create_config_mock.assert_called_once_with('file_path') - - @mock.patch('os.path') @mock.patch('__builtin__.open') def testInitBaseProperties(self, open_mock, path_mock): |