lib/storage.dart

// Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

/// This library provides access to Google Cloud Storage.
///
/// Google Cloud Storage is an object store for binary objects. Each
/// object has a set of metadata attached to it. For more information on
/// Google Cloud Sorage see https://developers.google.com/storage/.
///
/// There are two main concepts in Google Cloud Storage: Buckets and Objects.
/// A bucket is a container for objects and objects are the actual binary
/// objects.
///
/// The API has two main classes for dealing with buckets and objects.
///
/// The class `Storage` is the main API class providing access to working
/// with buckets. This is the 'bucket service' interface.
///
/// The class `Bucket` provide access to working with objcts in a specific
/// bucket. This is the 'object service' interface.
///
/// Both buckets have objects, have names. The bucket namespace is flat and
/// global across all projects. This means that a bucket is always
/// addressable using its name without requiring further context.
///
/// Within buckets the object namespace is also flat. Object are *not*
/// organized hierachical. However, as object names allow the slash `/`
/// character this is often used to simulate a hierachical structure
/// based on common prefixes.
///
/// This package uses relative and absolute names to refer to objects. A
/// relative name is just the object name within a bucket, and requires the
/// context of a bucket to be used. A relative name just looks like this:
///
///     object_name
///
/// An absolute name includes the bucket name and uses the `gs://` prefix
/// also used by the `gsutil` tool. An absolute name looks like this.
///
///     gs://bucket_name/object_name
///
/// In most cases relative names are used. Absolute names are typically
/// only used for operations involving objects in different buckets.
///
/// For most of the APIs in ths library which take instances of other classes
/// from this library it is the assumption that the actual implementations
/// provided here are used.
library gcloud.storage;

import 'dart:async';
import 'dart:collection' show UnmodifiableListView, UnmodifiableMapView;

import 'package:http/http.dart' as http;

import 'package:crypto/crypto.dart' as crypto;
import 'package:googleapis/storage/v1.dart' as storage_api;

import 'service_scope.dart' as ss;

import 'common.dart';
export 'common.dart';

part 'src/storage_impl.dart';

const Symbol _storageKey = #gcloud.storage;

/// Access the [Storage] object available in the current service scope.
///
/// The returned object will be the one which was previously registered with
/// [registerStorageService] within the current (or a parent) service scope.
///
/// Accessing this getter outside of a service scope will result in an error.
/// See the `package:gcloud/service_scope.dart` library for more information.
Storage get storageService => ss.lookup(_storageKey);

/// Registers the [storage] object within the current service scope.
///
/// The provided `storage` object will be available via the top-level
/// `storageService` getter.
///
/// Calling this function outside of a service scope will result in an error.
/// Calling this function more than once inside the same service scope is not
/// allowed.
void registerStorageService(Storage storage) {
  ss.register(_storageKey, storage);
}

int _jenkinsHash(List e) {
  const _HASH_MASK = 0x3fffffff;
  int hash = 0;
  for (int i = 0; i < e.length; i++) {
    int c = e[i].hashCode;
    hash = (hash + c) & _HASH_MASK;
    hash = (hash + (hash << 10)) & _HASH_MASK;
    hash ^= (hash >> 6);
  }
  hash = (hash + (hash << 3)) & _HASH_MASK;
  hash ^= (hash >> 11);
  hash = (hash + (hash << 15)) & _HASH_MASK;
  return hash;
}

/// An ACL (Access Control List) describes access rights to buckets and
/// objects.
///
/// An ACL is a prioritized sequence of access control specifications,
/// which individually prevent or grant access.
/// The access controls are described by [AclEntry] objects.
class Acl {
  final List<AclEntry> _entries;
  int _cachedHashCode;

  /// The entries in the ACL.
  List<AclEntry> get entries => new UnmodifiableListView<AclEntry>(_entries);

  /// Create a new ACL with a list of ACL entries.
  Acl(Iterable<AclEntry> entries) : _entries = new List.from(entries);

  Acl._fromBucketAcl(storage_api.Bucket bucket)
      : _entries = new List(bucket.acl == null ? 0 : bucket.acl.length) {
    if (bucket.acl != null) {
      for (int i = 0; i < bucket.acl.length; i++) {
        _entries[i] = new AclEntry(_aclScopeFromEntity(bucket.acl[i].entity),
                                   _aclPermissionFromRole(bucket.acl[i].role));
      }
    }
  }

  Acl._fromObjectAcl(storage_api.Object object)
      : _entries = new List(object.acl == null ? 0 : object.acl.length) {
    if (object.acl != null) {
      for (int i = 0; i < object.acl.length; i++) {
        _entries[i] = new AclEntry(_aclScopeFromEntity(object.acl[i].entity),
                                   _aclPermissionFromRole(object.acl[i].role));
      }
    }
  }

  AclScope _aclScopeFromEntity(String entity) {
    if (entity.startsWith('user-')) {
      String tmp = entity.substring(5);
      int at = tmp.indexOf('@');
      if (at != -1) {
        return new AccountScope(tmp);
      } else {
        return new StorageIdScope(tmp);
      }
    } else if (entity.startsWith('group-')) {
      return new GroupScope(entity.substring(6));
    } else if (entity.startsWith('domain-')) {
      return new DomainScope(entity.substring(7));
    } else if (entity.startsWith('allAuthenticatedUsers-')) {
      return AclScope.allAuthenticated;
    } else if (entity.startsWith('allUsers-')) {
      return AclScope.allUsers;
    } else if (entity.startsWith('project-')) {
      String tmp = entity.substring(8);
      int dash = tmp.indexOf('-');
      if (dash != -1) {
        return new ProjectScope(tmp.substring(dash + 1),
                                tmp.substring(0, dash));
      }
    }
    return new OpaqueScope(entity);
  }

  AclPermission _aclPermissionFromRole(String role) {
    if (role == 'READER') return AclPermission.READ;
    if (role == 'WRITER') return AclPermission.WRITE;
    if (role == 'OWNER') return AclPermission.FULL_CONTROL;
    throw new UnsupportedError(
        "Server returned a unsupported permission role '$role'");
  }

  List<storage_api.BucketAccessControl> _toBucketAccessControlList() {
    return _entries.map((entry) => entry._toBucketAccessControl()).toList();
  }

  List<storage_api.ObjectAccessControl> _toObjectAccessControlList() {
    return _entries.map((entry) => entry._toObjectAccessControl()).toList();
  }

  int get hashCode {
    return _cachedHashCode != null
        ? _cachedHashCode
        : _cachedHashCode = _jenkinsHash(_entries);
  }

  bool operator==(Object other) {
    if (other is Acl) {
      List entries = _entries;
      List otherEntries = other._entries;
      if (entries.length != otherEntries.length) return false;
      for (int i = 0; i < entries.length; i++) {
        if (entries[i] != otherEntries[i]) return false;
      }
      return true;
    } else {
      return false;
    }
  }

  String toString() => 'Acl($_entries)';
}

/// An ACL entry specifies that an entity has a specific access permission.
///
/// A permission grants a specific permission to the entity.
class AclEntry {
  final AclScope scope;
  final AclPermission permission;
  int _cachedHashCode;

  AclEntry(this.scope, this.permission);

  storage_api.BucketAccessControl _toBucketAccessControl() {
    var acl = new storage_api.BucketAccessControl();
    acl.entity = scope._storageEntity;
    acl.role = permission._storageBucketRole;
    return acl;
  }

  storage_api.ObjectAccessControl _toObjectAccessControl() {
    var acl = new storage_api.ObjectAccessControl();
    acl.entity = scope._storageEntity;
    acl.role = permission._storageObjectRole;
    return acl;
  }

  int get hashCode {
    return _cachedHashCode != null
        ? _cachedHashCode
        : _cachedHashCode = _jenkinsHash([scope, permission]);
  }

  bool operator==(Object other) {
    return other is AclEntry &&
           scope == other.scope &&
           permission == other.permission;
  }

  String toString() => 'AclEntry($scope, $permission)';
}

/// An ACL scope specifies an entity for which a permission applies.
///
/// A scope can be one of:
///
///   * Google Storage ID
///   * Google account email address
///   * Google group email address
///   * Google Apps domain
///   * Special identifier for all Google account holders
///   * Special identifier for all users
///
/// See https://cloud.google.com/storage/docs/accesscontrol for more details.
abstract class AclScope {
  int _cachedHashCode;

  /// ACL type for scope representing a Google Storage id.
  static const int _TYPE_STORAGE_ID = 0;

  /// ACL type for scope representing a project entity.
  static const int _TYPE_PROJECT = 1;

  /// ACL type for scope representing an account holder.
  static const int _TYPE_ACCOUNT = 2;

  /// ACL type for scope representing a group.
  static const int _TYPE_GROUP = 3;

  /// ACL type for scope representing a domain.
  static const int _TYPE_DOMAIN = 4;

  /// ACL type for scope representing all authenticated users.
  static const int _TYPE_ALL_AUTHENTICATED = 5;

  /// ACL type for scope representing all users.
  static const int _TYPE_ALL_USERS = 6;

  /// ACL type for scope representing an unsupported scope.
  static const int _TYPE_OPAQUE = 7;

  /// The id of the actual entity this ACL scope represents. The actual values
  /// are set in the different subclasses.
  final String _id;

  /// The type of this acope this ACL scope represents.
  final int _type;

  /// ACL scope for all authenticated users.
  static AllAuthenticatedScope allAuthenticated = new AllAuthenticatedScope();

  /// ACL scope for all users.
  static AllUsersScope allUsers = new AllUsersScope();

  AclScope._(this._type, this._id);

  int get hashCode {
    return _cachedHashCode != null
        ? _cachedHashCode
        : _cachedHashCode = _jenkinsHash([_type, _id]);
  }

  bool operator==(Object other) {
    return other is AclScope && _type == other._type && _id == other._id;
  }

  String toString() => 'AclScope($_storageEntity)';

  String get _storageEntity;
}

/// An ACL scope for an entity identified by a 'Google Storage ID'.
///
/// The [storageId] is a string of 64 hexadecimal digits that identifies a
/// specific Google account holder or a specific Google group.
class StorageIdScope extends AclScope {
  StorageIdScope(String storageId)
      : super._(AclScope._TYPE_STORAGE_ID, storageId);

  /// Google Storage ID.
  String get storageId => _id;

  String get _storageEntity => 'user-$_id';
}

/// An ACL scope for an entity identified by an individual email address.
class AccountScope extends AclScope {
  AccountScope(String email): super._(AclScope._TYPE_ACCOUNT, email);

  /// Email address.
  String get email => _id;

  String get _storageEntity => 'user-$_id';
}

/// An ACL scope for an entity identified by an Google Groups email.
class GroupScope extends AclScope {
  GroupScope(String group): super._(AclScope._TYPE_GROUP, group);

  /// Group name.
  String get group => _id;

  String get _storageEntity => 'group-$_id';
}

/// An ACL scope for an entity identified by a domain name.
class DomainScope extends AclScope {
  DomainScope(String domain): super._(AclScope._TYPE_DOMAIN, domain);

  /// Domain name.
  String get domain => _id;

  String get _storageEntity => 'domain-$_id';
}

/// An ACL scope for an project related entity.
class ProjectScope extends AclScope {
  /// Project role.
  ///
  /// Possible values are `owners`, `editors` and `viewers`.
  final String role;

  ProjectScope(String project, String this.role)
      : super._(AclScope._TYPE_PROJECT, project);

  /// Project ID.
  String get project => _id;

  String get _storageEntity => 'project-$role-$_id';
}

/// An ACL scope for an unsupported scope.
class OpaqueScope extends AclScope {
  OpaqueScope(String id) : super._(AclScope._TYPE_OPAQUE, id);

  String get _storageEntity => _id;
}

/// ACL scope for a all authenticated users.
class AllAuthenticatedScope extends AclScope {
  AllAuthenticatedScope()
      : super._(AclScope._TYPE_ALL_AUTHENTICATED, null);

  String get _storageEntity => 'allAuthenticatedUsers';
}

/// ACL scope for a all users.
class AllUsersScope extends AclScope {
  AllUsersScope(): super._(AclScope._TYPE_ALL_USERS, null);

  String get _storageEntity => 'allUsers';
}

/// Permissions for individual scopes in an ACL.
class AclPermission {
  /// Provide read access.
  static const READ = const AclPermission._('READER');

  /// Provide write access.
  ///
  /// For objects this permission is the same as [FULL_CONTROL].
  static const WRITE = const AclPermission._('WRITER');

  /// Provide full control.
  ///
  /// For objects this permission is the same as [WRITE].
  static const FULL_CONTROL = const AclPermission._('OWNER');

  final String _id;

  const AclPermission._(this._id);

  String get _storageBucketRole => _id;

  String get _storageObjectRole => this == WRITE ? FULL_CONTROL._id : _id;

  int get hashCode => _id.hashCode;

  bool operator==(Object other) {
    return other is AclPermission && _id == other._id;
  }

  String toString() => 'AclPermission($_id)';
}

/// Definition of predefined ACLs.
///
/// There is a convenient way of referring to number of _predefined_ ACLs. These
/// predefined ACLs have explicit names, and can _only_ be used to set an ACL,
/// when either creating or updating a bucket or object. This set of predefined
/// ACLs are expanded on the server to their actual list of [AclEntry] objects.
/// When information is retrieved on a bucket or object, this expanded list will
/// be present. For a description of these predefined ACLs see:
/// https://cloud.google.com/storage/docs/accesscontrol#extension.
class PredefinedAcl {
  final String _name;
  const PredefinedAcl._(this._name);

  /// Predefined ACL for the 'authenticated-read' ACL. Applies to both buckets
  /// and objects.
  static const PredefinedAcl authenticatedRead =
      const PredefinedAcl._('authenticatedRead');

  /// Predefined ACL for the 'private' ACL. Applies to both buckets
  /// and objects.
  static const PredefinedAcl private = const PredefinedAcl._('private');

  /// Predefined ACL for the 'project-private' ACL. Applies to both buckets
  /// and objects.
  static const PredefinedAcl projectPrivate =
      const PredefinedAcl._('projectPrivate');

  /// Predefined ACL for the 'public-read' ACL. Applies to both buckets
  /// and objects.
  static const PredefinedAcl publicRead = const PredefinedAcl._('publicRead');

  /// Predefined ACL for the 'public-read-write' ACL. Applies only to buckets.
  static const PredefinedAcl publicReadWrite =
      const PredefinedAcl._('publicReadWrite');

  /// Predefined ACL for the 'bucket-owner-full-control' ACL. Applies only to
  /// objects.
  static const PredefinedAcl bucketOwnerFullControl =
      const PredefinedAcl._('bucketOwnerFullControl');

  /// Predefined ACL for the 'bucket-owner-read' ACL. Applies only to
  /// objects.
  static const PredefinedAcl bucketOwnerRead =
      const PredefinedAcl._('bucketOwnerRead');

  String toString() => 'PredefinedAcl($_name)';
}

/// Information on a bucket.
abstract class BucketInfo {
  /// Name of the bucket.
  String get bucketName;

  /// Entity tag for the bucket.
  String get etag;

  /// When this bucket was created.
  DateTime get created;

  /// Bucket ID.
  String get id;

  /// Acl of the bucket.
  Acl get acl;
}

/// Access to Cloud Storage
abstract class Storage {
  /// List of required OAuth2 scopes for Cloud Storage operation.
  static const List<String> SCOPES =
      const <String>[storage_api.StorageApi.DevstorageFullControlScope];

  /// Initializes access to cloud storage.
  factory Storage(http.Client client, String project) = _StorageImpl;

  /// Create a cloud storage bucket.
  ///
  /// Creates a cloud storage bucket named [bucketName].
  ///
  /// The bucket ACL can be set by passing [predefinedAcl] or [acl]. If both
  /// are passed the entries from [acl] with be followed by the expansion of
  /// [predefinedAcl].
  ///
  /// Returns a [Future] which completes when the bucket has been created.
  Future createBucket(String bucketName,
                      {PredefinedAcl predefinedAcl, Acl acl});

  /// Delete a cloud storage bucket.
  ///
  /// Deletes the cloud storage bucket named [bucketName].
  ///
  /// If the bucket is not empty the operation will fail.
  ///
  /// The returned [Future] completes when the operation is finished.
  Future deleteBucket(String bucketName);

  /// Access bucket object operations.
  ///
  /// Instantiates a `Bucket` object refering to the bucket named [bucketName].
  ///
  /// When an object is created using the resulting `Bucket` an ACL will always
  /// be set. If the object creation does not pass any explicit ACL information
  /// a default ACL will be used.
  ///
  /// If the arguments [defaultPredefinedObjectAcl] or [defaultObjectAcl] are
  /// passed they define the default ACL. If both are passed the entries from
  /// [defaultObjectAcl] with be followed by the expansion of
  /// [defaultPredefinedObjectAcl] when an object is created.
  ///
  /// Otherwise the default object ACL attached to the bucket will be used.
  ///
  /// Returns a `Bucket` instance.
  Bucket bucket(String bucketName,
                {PredefinedAcl defaultPredefinedObjectAcl,
                 Acl defaultObjectAcl});

  /// Check whether a cloud storage bucket exists.
  ///
  /// Checks whether the bucket named [bucketName] exists.
  ///
  /// Returns a [Future] which completes with `true` if the bucket exists.
  Future<bool> bucketExists(String bucketName);

  /// Get information on a bucket
  ///
  /// Provide metadata information for bucket named [bucketName].
  ///
  /// Returns a [Future] which completes with a `BuckerInfo` object.
  Future<BucketInfo> bucketInfo(String bucketName);

  /// List names of all buckets.
  ///
  /// Returns a [Stream] of bucket names.
  Stream<String> listBucketNames();

  /// Start paging through names of all buckets.
  ///
  /// The maximum number of buckets in each page is specified in [pageSize].
  ///
  /// Returns a [Future] which completes with a `Page` object holding the
  /// first page. Use the `Page` object to move to the next page of buckets.
  Future<Page<String>> pageBucketNames({int pageSize: 50});

  /// Copy an object.
  ///
  /// Copy object [src] to object [dest].
  ///
  /// The names of [src] and [dest] must be absolute.
  Future copyObject(String src, String dest);
}

/// Information on a specific object.
///
/// This class provides access to information on an object. This includes
/// both the properties which are provided by Cloud Storage (such as the
/// MD5 hash) and the properties which can be changed (such as content type).
///
///  The properties provided by Cloud Storage are direct properties on this
///  object.
///
///  The mutable properties are properties on the `metadata` property.
abstract class ObjectInfo {
  /// Name of the object.
  String get name;

  /// Length of the data.
  int get length;

  /// When this object was updated.
  DateTime get updated;

  /// Entity tag for the object.
  String get etag;

  /// MD5 hash of the object.
  List<int> get md5Hash;

  /// CRC32c checksum, as described in RFC 4960.
  int get crc32CChecksum;

  /// URL for direct download.
  Uri get downloadLink;

  /// Object generation.
  ObjectGeneration get generation;

  /// Additional metadata.
  ObjectMetadata get metadata;
}

/// Generational information on an object.
class ObjectGeneration {
  /// Object generation.
  final String objectGeneration;

  /// Metadata generation.
  final int metaGeneration;

  const ObjectGeneration(this.objectGeneration, this.metaGeneration);
}

/// Access to object metadata.
abstract class ObjectMetadata {
  factory ObjectMetadata({Acl acl, String contentType, String contentEncoding,
      String cacheControl, String contentDisposition, String contentLanguage,
      Map<String, String> custom}) = _ObjectMetadata;
  /// ACL.
  Acl get acl;

  /// `Content-Type` for this object.
  String get contentType;

  /// `Content-Encoding` for this object.
  String get contentEncoding;

  /// `Cache-Control` for this object.
  String get cacheControl;

  /// `Content-Disposition` for this object.
  String get contentDisposition;

  /// `Content-Language` for this object.
  ///
  /// The value of this field must confirm to RFC 3282.
  String get contentLanguage;

  /// Custom metadata.
  Map<String, String> get custom;

  /// Create a copy of this object with some values replaced.
  ///
  // TODO: This cannot be used to set values to null.
  ObjectMetadata replace({Acl acl, String contentType, String contentEncoding,
      String cacheControl, String contentDisposition, String contentLanguage,
      Map<String, String> custom});
}

/// Result from List objects in a bucket.
///
/// Listing operate like a directory listing, despite the object
/// namespace being flat.
///
/// See [Bucket.list] for information on how the hierarchical structure
/// is determined.
class BucketEntry {
  /// Whether this is information on an object.
  final bool isObject;

  /// Name of object or directory.
  final String name;

  BucketEntry._object(this.name) : isObject = true;

  BucketEntry._directory(this.name) : isObject = false;

  /// Whether this is a prefix.
  bool get isDirectory => !isObject;
}

/// Access to operations on a specific cloud storage buket.
abstract class Bucket {
  /// Name of this bucket.
  String get bucketName;

  /// Absolute name of an object in this bucket. This includes the gs:// prefix.
  String absoluteObjectName(String objectName);

  /// Create a new object.
  ///
  /// Create an object named [objectName] in the bucket.
  ///
  /// If an object named [objectName] already exists this object will be
  /// replaced.
  ///
  /// If the length of the data to write is known in advance this can be passed
  /// as [length]. This can help to optimize the upload process.
  ///
  /// Additional metadata on the object can be passed either through the
  /// `metadata` argument or through the specific named arguments
  /// (such as `contentType`). Values passed through the specific named
  /// arguments takes precedence over the values in `metadata`.
  ///
  /// If [contentType] is not passed the default value of
  /// `application/octet-stream` will be used.
  ///
  /// It is possible to at one of the predefined ACLs on the created object
  /// using the [predefinedAcl] argument. If the [metadata] argument contain a
  /// ACL as well, this ACL with be followed by the expansion of
  /// [predefinedAcl].
  ///
  /// Returns a `StreamSink` where the object content can be written. When
  /// The object content has been written the `StreamSink` completes with
  /// an `ObjectInfo` instance with the information on the object created.
  StreamSink<List<int>> write(String objectName,
      {int length, ObjectMetadata metadata,
       Acl acl, PredefinedAcl predefinedAcl, String contentType});

  /// Create an new object in the bucket with specified content.
  ///
  /// Writes [bytes] to the created object.
  ///
  /// See [write] for more information on the additional arguments.
  ///
  /// Returns a `Future` which completes with an `ObjectInfo` instance when
  /// the object is written.
  Future<ObjectInfo> writeBytes(String name, List<int> bytes,
      {ObjectMetadata metadata,
        Acl acl, PredefinedAcl predefinedAcl, String contentType});

  /// Read object content as byte stream.
  ///
  /// If [offset] is provided, [length] must also be provided.
  ///
  /// If [length] is provided, it must be greater than `0`.
  ///
  /// If there is a problem accessing the file, a [DetailedApiRequestError] is
  /// thrown.
  Stream<List<int>> read(String objectName, {int offset, int length});

  /// Lookup object metadata.
  ///
  // TODO: More documentation
  Future<ObjectInfo> info(String name);

  /// Delete an object.
  ///
  // TODO: More documentation
  Future delete(String name);

  /// Update object metadata.
  ///
  // TODO: More documentation
  Future updateMetadata(String objectName, ObjectMetadata metadata);

  /// List objects in the bucket.
  ///
  /// Listing operates like a directory listing, despite the object
  /// namespace being flat. The character `/` is being used to separate
  /// object names into directory components.
  ///
  /// Retrieves a list of objects and directory components starting
  /// with [prefix].
  ///
  /// Returns a [Stream] of [BucketEntry]. Each element of the stream
  /// represents either an object or a directory component.
  Stream<BucketEntry> list({String prefix});

  /// Start paging through objects in the bucket.
  ///
  /// The maximum number of objects in each page is specified in [pageSize].
  ///
  /// See [list] for more information on the other arguments.
  ///
  /// Returns a `Future` which completes with a `Page` object holding the
  /// first page. Use the `Page` object to move to the next page.
  Future<Page<BucketEntry>> page({String prefix, int pageSize: 50});
}