mojo/public/tools/mojom_fetcher/mojom_fetcher.py

#!/usr/bin/env python
# Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Tool to manage external mojom interfaces."""

import argparse
import errno
import logging
import os
import sys
import urllib2

# Local library
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                "pylib"))
# Bindings library
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                "..", "bindings", "pylib"))
# Requests library
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                "..", "..", "..", "..", "third_party",
                                "requests", "src"))

import requests

from fetcher.repository import Repository
from fetcher.dependency import Dependency
from mojom.parse.parser import Parse, ParseError


class UrlRewriterException(Exception):
  """Exception when processing URL rewrite rules."""
  pass

class UrlRewriter(object):
  """UrlRewriter rewrites URLs according to the provided mappings.

  Note that mappings are not followed transitively. If mappings contains
  {"a": "b", "b": "c"}, then UrlRewriter.rewrite("a") will return "b", not "c".
  """

  def __init__(self, mappings):
    self._mappings = mappings
    for target in self._mappings.values():
      for source in self._mappings.keys():
        if source in target or target in source:
          raise UrlRewriterException(
              "%s and %s share a common subpath" % (source, target))

  def rewrite(self, path):
    for origin, destination in self._mappings.items():
      if path.startswith(origin):
        return destination + path[len(origin):]
    return path


class MojomFetcher(object):
  def __init__(self, repository, url_rewriter):
    self._repository = repository
    self._url_rewriter = url_rewriter

  def _requests_get(self, url):
    return requests.get(url, verify=True)

  def _os_makedirs(self, dirs):
    try:
      os.makedirs(dirs)
    except OSError as e:
      # The directory may already exist, we don't care.
      if e.errno != errno.EEXIST:
        raise

  def _open(self, f, mode="r"):
    return open(f, mode)

  def _download_dependencies(self, dependencies):
    """Takes the list of mojom dependencies and download the external ones.
    Returns the number of successfully downloaded dependencies."""

    downloaded = 0
    for dep in dependencies:
      if self._maybe_download_dep(dep):
        downloaded += 1
    return downloaded

  def _maybe_download_dep(self, dep):
    if not dep.maybe_is_a_url():
      return False

    for candidate in dep.generate_candidate_urls():
      url = self._url_rewriter.rewrite(candidate)
      response = self._requests_get("https://" + url)
      if not response.ok:
        # If we get an error, it just mean that this candidate URL is not
        # correct. We must try the other ones before giving up.
        logging.debug("Error while downloading %s (%s)", candidate, url)
        continue
      # This is an external dependency
      directory = os.path.dirname(candidate)
      full_directory = os.path.join(self._repository.get_external_directory(),
                                    directory)
      try:
        self._os_makedirs(full_directory)
      except OSError as e:
        # The directory may already exist, we don't care.
        if e.errno != errno.EEXIST:
          raise
      with self._open(os.path.join(self._repository.get_external_directory(),
                             candidate), "w") as f:
        data = response.content
        try:
          Parse(data, candidate)
        except ParseError:
          logging.warn("File at %s is not a mojom", url)
          break
        f.write(data)
      return True
    return False

  def discover(self):
    """Discover missing .mojom dependencies and download them."""
    while True:
      missing_deps = self._repository.get_missing_dependencies()
      downloaded = self._download_dependencies(missing_deps)
      if downloaded == 0:
        return 0

  def get(self, dep):
    dependency = Dependency(self._repository, ".", dep)
    downloaded = self._download_dependencies([dependency])
    if downloaded != 0:
      return self.discover()
    else:
      return -1

  def update(self):
    dependencies = [Dependency(self._repository, ".", f)
                    for f in self._repository.get_external_urls()]
    # TODO(etiennej): We may want to suggest to the user to delete
    # un-downloadable dependencies.
    downloaded = self._download_dependencies(dependencies)
    if downloaded != 0:
      return self.discover()
    else:
      return -1

def _main(args):
  if args.prefix_rewrite:
    rewrite_rules = dict([x.split(':', 1) for x in args.prefix_rewrite])
  else:
    rewrite_rules = {}
  rewriter = UrlRewriter(rewrite_rules)
  repository_path = os.path.abspath(args.repository_path)
  repository = Repository(repository_path, args.external_dir)
  fetcher = MojomFetcher(repository, rewriter)
  if args.action == 'discover':
    return fetcher.discover()
  elif args.action == 'get':
    return fetcher.get(args.url)
  elif args.action == 'update':
    return fetcher.update()
  else:
    logging.error("No matching action %s", args.action[0])
    return -1

def main():
  logging.basicConfig(level=logging.ERROR)
  parser = argparse.ArgumentParser(description='Download mojom dependencies.')
  parser.add_argument('--repository-path', type=str, action='store',
                      default='.', help='The path to the client repository.')
  parser.add_argument('--external-dir', type=str, action='store',
                      default='external',
                      help='Directory for external interfaces')
  parser.add_argument('--prefix-rewrite', type=str, action='append',
                      help='If present, "origin:destination" pairs. "origin" '
                      'prefixes will be rewritten as "destination". May be '
                      'used several times. Rewrites are not transitive.')

  subparsers = parser.add_subparsers(dest='action', help='action')
  parser_get = subparsers.add_parser(
      'get', help='Get the specified URL and all its transitive dependencies')
  parser_get.add_argument('url', type=str, nargs=1,
                      help='URL to download for get action')
  subparsers.add_parser(
      'discover',
      help='Recursively discover and download new external dependencies')
  subparsers.add_parser('update', help='Update all external dependencies')

  args = parser.parse_args()
  return _main(args)


if __name__ == '__main__':
  sys.exit(main())