# Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. from third_party.cloudstorage import cloudstorage_api from third_party.cloudstorage import common from third_party.cloudstorage import errors from docs_server_utils import StringIdentity from file_system import FileSystem, FileNotFoundError, StatInfo from future import Future from path_util import ( AssertIsDirectory, AssertIsFile, AssertIsValid, IsDirectory, Join) import logging import traceback # See gcs_file_system_provider.py for documentation on using Google Cloud # Storage as a filesystem. # # Note that the path requirements for GCS are different for the docserver; # GCS requires that paths start with a /, we require that they don't. # Name of the file containing the Git hash of the latest commit sync'ed # to Cloud Storage. This file is generated by the Github->GCS sync script LAST_COMMIT_HASH_FILENAME = '.__lastcommit.txt' def _ReadFile(filename): AssertIsFile(filename) try: with cloudstorage_api.open('/' + filename, 'r') as f: return f.read() except errors.Error: raise FileNotFoundError('Read failed for %s: %s' % (filename, traceback.format_exc())) def _ListDir(dir_name, recursive=False): AssertIsDirectory(dir_name) try: # The listbucket method uses a prefix approach to simulate hierarchy. # Calling it with the "delimiter" argument set to '/' gets only files # directly inside the directory, not all recursive content. delimiter = None if recursive else '/' files = cloudstorage_api.listbucket('/' + dir_name, delimiter=delimiter) return [os_path.filename.lstrip('/')[len(dir_name):] for os_path in files] except errors.Error: raise FileNotFoundError('cloudstorage.listbucket failed for %s: %s' % (dir_name, traceback.format_exc())) def _CreateStatInfo(bucket, path): full_path = Join(bucket, path) last_commit_file = Join(bucket, LAST_COMMIT_HASH_FILENAME) try: last_commit = _ReadFile(last_commit_file) if IsDirectory(full_path): child_versions = dict((filename, last_commit) for filename in _ListDir(full_path)) else: child_versions = None return StatInfo(last_commit, child_versions) except (TypeError, errors.Error): raise FileNotFoundError('cloudstorage.stat failed for %s: %s' % (path, traceback.format_exc())) class CloudStorageFileSystem(FileSystem): '''FileSystem implementation which fetches resources from Google Cloud Storage. ''' def __init__(self, bucket, debug_access_token=None, debug_bucket_prefix=None): self._bucket = bucket if debug_access_token: logging.debug('gcs: using debug access token: %s' % debug_access_token) common.set_access_token(debug_access_token) if debug_bucket_prefix: logging.debug('gcs: prefixing all bucket names with %s' % debug_bucket_prefix) self._bucket = debug_bucket_prefix + self._bucket AssertIsValid(self._bucket) def Read(self, paths, skip_not_found=False): def resolve(): try: result = {} for path in paths: full_path = Join(self._bucket, path) logging.debug('gcs: requested path "%s", reading "%s"' % (path, full_path)) if IsDirectory(path): result[path] = _ListDir(full_path) else: result[path] = _ReadFile(full_path) return result except errors.AuthorizationError: self._warnAboutAuthError() raise return Future(callback=resolve) def Refresh(self): return Future(value=()) def Stat(self, path): AssertIsValid(path) try: return _CreateStatInfo(self._bucket, path) except errors.AuthorizationError: self._warnAboutAuthError() raise def GetIdentity(self): return '@'.join((self.__class__.__name__, StringIdentity(self._bucket))) def __repr__(self): return 'CloudStorageFileSystem(%s)' % self._bucket def _warnAboutAuthError(self): logging.warn(('Authentication error on Cloud Storage. Check if your' ' appengine project has permissions to Read the GCS' ' buckets. If you are running a local appengine server,' ' you need to set an access_token in' ' local_debug/gcs_debug.conf.' ' Remember that this token expires in less than 10' ' minutes, so keep it updated. See' ' gcs_file_system_provider.py for instructions.')); logging.debug(traceback.format_exc())