1 files changed, 421 insertions, 0 deletions
diff --git a/repo_to_repo.py b/repo_to_repo.py
new file mode 100755
index 00000000..3b3b9bc4
--- /dev/null
+++ b/repo_to_repo.py
@@ -0,0 +1,421 @@
+#!/usr/bin/python2
+#
+# Copyright 2010 Google Inc. All Rights Reserved.
+"""Module for transferring files between various types of repositories."""
+
+from __future__ import print_function
+
+__author__ = 'asharif@google.com (Ahmad Sharif)'
+
+import argparse
+import datetime
+import json
+import os
+import re
+import socket
+import sys
+import tempfile
+
+from automation.clients.helper import perforce
+from cros_utils import command_executer
+from cros_utils import logger
+from cros_utils import misc
+
+# pylint: disable=anomalous-backslash-in-string
+
+def GetCanonicalMappings(mappings):
+  canonical_mappings = []
+  for mapping in mappings:
+    remote_path, local_path = mapping.split()
+    if local_path.endswith('/') and not remote_path.endswith('/'):
+      local_path = os.path.join(local_path, os.path.basename(remote_path))
+    remote_path = remote_path.lstrip('/').split('/', 1)[1]
+    canonical_mappings.append(perforce.PathMapping(remote_path, local_path))
+  return canonical_mappings
+
+
+def SplitMapping(mapping):
+  parts = mapping.split()
+  assert len(parts) <= 2, 'Mapping %s invalid' % mapping
+  remote_path = parts[0]
+  if len(parts) == 2:
+    local_path = parts[1]
+  else:
+    local_path = '.'
+  return remote_path, local_path
+
+
+class Repo(object):
+  """Basic repository base class."""
+
+  def __init__(self, no_create_tmp_dir=False):
+    self.repo_type = None
+    self.address = None
+    self.mappings = None
+    self.revision = None
+    self.ignores = ['.gitignore', '.p4config', 'README.google']
+    if no_create_tmp_dir:
+      self._root_dir = None
+    else:
+      self._root_dir = tempfile.mkdtemp()
+    self._ce = command_executer.GetCommandExecuter()
+    self._logger = logger.GetLogger()
+
+  def PullSources(self):
+    """Pull all sources into an internal dir."""
+    pass
+
+  def SetupForPush(self):
+    """Setup a repository for pushing later."""
+    pass
+
+  def PushSources(self, commit_message=None, dry_run=False, message_file=None):
+    """Push to the external repo with the commit message."""
+    pass
+
+  def _RsyncExcludingRepoDirs(self, source_dir, dest_dir):
+    for f in os.listdir(source_dir):
+      if f in ['.git', '.svn', '.p4config']:
+        continue
+      dest_file = os.path.join(dest_dir, f)
+      source_file = os.path.join(source_dir, f)
+      if os.path.exists(dest_file):
+        command = 'rm -rf %s' % dest_file
+        self._ce.RunCommand(command)
+      command = 'rsync -a %s %s' % (source_file, dest_dir)
+      self._ce.RunCommand(command)
+    return 0
+
+  def MapSources(self, dest_dir):
+    """Copy sources from the internal dir to root_dir."""
+    return self._RsyncExcludingRepoDirs(self._root_dir, dest_dir)
+
+  def GetRoot(self):
+    return self._root_dir
+
+  def SetRoot(self, directory):
+    self._root_dir = directory
+
+  def CleanupRoot(self):
+    command = 'rm -rf %s' % self._root_dir
+    return self._ce.RunCommand(command)
+
+  def __str__(self):
+    return '\n'.join(str(s)
+                     for s in [self.repo_type, self.address, self.mappings])
+
+
+# Note - this type of repo is used only for "readonly", in other words, this
+# only serves as a incoming repo.
+class FileRepo(Repo):
+  """Class for file repositories."""
+
+  def __init__(self, address, ignores=None):
+    Repo.__init__(self, no_create_tmp_dir=True)
+    self.repo_type = 'file'
+    self.address = address
+    self.mappings = None
+    self.branch = None
+    self.revision = '{0} (as of "{1}")'.format(address, datetime.datetime.now())
+    self.gerrit = None
+    self._root_dir = self.address
+    if ignores:
+      self.ignores += ignores
+
+  def CleanupRoot(self):
+    """Override to prevent deletion."""
+    pass
+
+
+class P4Repo(Repo):
+  """Class for P4 repositories."""
+
+
+  def __init__(self, address, mappings, revision=None):
+    Repo.__init__(self)
+    self.repo_type = 'p4'
+    self.address = address
+    self.mappings = mappings
+    self.revision = revision
+
+  def PullSources(self):
+    client_name = socket.gethostname()
+    client_name += tempfile.mkstemp()[1].replace('/', '-')
+    mappings = self.mappings
+    p4view = perforce.View('depot2', GetCanonicalMappings(mappings))
+    p4client = perforce.CommandsFactory(self._root_dir,
+                                        p4view,
+                                        name=client_name)
+    command = p4client.SetupAndDo(p4client.Sync(self.revision))
+    ret = self._ce.RunCommand(command)
+    assert ret == 0, 'Could not setup client.'
+    command = p4client.InCheckoutDir(p4client.SaveCurrentCLNumber())
+    ret, o, _ = self._ce.RunCommandWOutput(command)
+    assert ret == 0, 'Could not get version from client.'
+    self.revision = re.search('^\d+$', o.strip(), re.MULTILINE).group(0)
+    command = p4client.InCheckoutDir(p4client.Remove())
+    ret = self._ce.RunCommand(command)
+    assert ret == 0, 'Could not delete client.'
+    return 0
+
+
+class SvnRepo(Repo):
+  """Class for svn repositories."""
+
+  def __init__(self, address, mappings):
+    Repo.__init__(self)
+    self.repo_type = 'svn'
+    self.address = address
+    self.mappings = mappings
+
+  def PullSources(self):
+    with misc.WorkingDirectory(self._root_dir):
+      for mapping in self.mappings:
+        remote_path, local_path = SplitMapping(mapping)
+        command = 'svn co %s/%s %s' % (self.address, remote_path, local_path)
+      ret = self._ce.RunCommand(command)
+      if ret:
+        return ret
+
+      self.revision = ''
+      for mapping in self.mappings:
+        remote_path, local_path = SplitMapping(mapping)
+        command = 'cd %s && svnversion -c .' % (local_path)
+        ret, o, _ = self._ce.RunCommandWOutput(command)
+        self.revision += o.strip().split(':')[-1]
+        if ret:
+          return ret
+    return 0
+
+
+class GitRepo(Repo):
+  """Class for git repositories."""
+
+  def __init__(self, address, branch, mappings=None, ignores=None, gerrit=None):
+    Repo.__init__(self)
+    self.repo_type = 'git'
+    self.address = address
+    self.branch = branch or 'master'
+    if ignores:
+      self.ignores += ignores
+    self.mappings = mappings
+    self.gerrit = gerrit
+
+  def _CloneSources(self):
+    with misc.WorkingDirectory(self._root_dir):
+      command = 'git clone %s .' % (self.address)
+      return self._ce.RunCommand(command)
+
+  def PullSources(self):
+    with misc.WorkingDirectory(self._root_dir):
+      ret = self._CloneSources()
+      if ret:
+        return ret
+
+      command = 'git checkout %s' % self.branch
+      ret = self._ce.RunCommand(command)
+      if ret:
+        return ret
+
+      command = 'git describe --always'
+      ret, o, _ = self._ce.RunCommandWOutput(command)
+      self.revision = o.strip()
+      return ret
+
+  def SetupForPush(self):
+    with misc.WorkingDirectory(self._root_dir):
+      ret = self._CloneSources()
+      logger.GetLogger().LogFatalIf(ret, 'Could not clone git repo %s.' %
+                                    self.address)
+
+      command = 'git branch -a | grep -wq %s' % self.branch
+      ret = self._ce.RunCommand(command)
+
+      if ret == 0:
+        if self.branch != 'master':
+          command = ('git branch --track %s remotes/origin/%s' %
+                     (self.branch, self.branch))
+        else:
+          command = 'pwd'
+        command += '&& git checkout %s' % self.branch
+      else:
+        command = 'git symbolic-ref HEAD refs/heads/%s' % self.branch
+      command += '&& rm -rf *'
+      ret = self._ce.RunCommand(command)
+      return ret
+
+  def CommitLocally(self, commit_message=None, message_file=None):
+    with misc.WorkingDirectory(self._root_dir):
+      command = 'pwd'
+      for ignore in self.ignores:
+        command += '&& echo \'%s\' >> .git/info/exclude' % ignore
+      command += '&& git add -Av .'
+      if message_file:
+        message_arg = '-F %s' % message_file
+      elif commit_message:
+        message_arg = '-m \'%s\'' % commit_message
+      else:
+        raise RuntimeError('No commit message given!')
+      command += '&& git commit -v %s' % message_arg
+      return self._ce.RunCommand(command)
+
+  def PushSources(self, commit_message=None, dry_run=False, message_file=None):
+    ret = self.CommitLocally(commit_message, message_file)
+    if ret:
+      return ret
+    push_args = ''
+    if dry_run:
+      push_args += ' -n '
+    with misc.WorkingDirectory(self._root_dir):
+      if self.gerrit:
+        label = 'somelabel'
+        command = 'git remote add %s %s' % (label, self.address)
+        command += ('&& git push %s %s HEAD:refs/for/master' %
+                    (push_args, label))
+      else:
+        command = 'git push -v %s origin %s:%s' % (push_args, self.branch,
+                                                   self.branch)
+      ret = self._ce.RunCommand(command)
+    return ret
+
+  def MapSources(self, root_dir):
+    if not self.mappings:
+      self._RsyncExcludingRepoDirs(self._root_dir, root_dir)
+      return
+    with misc.WorkingDirectory(self._root_dir):
+      for mapping in self.mappings:
+        remote_path, local_path = SplitMapping(mapping)
+        remote_path.rstrip('...')
+        local_path.rstrip('...')
+        full_local_path = os.path.join(root_dir, local_path)
+        ret = self._RsyncExcludingRepoDirs(remote_path, full_local_path)
+        if ret:
+          return ret
+    return 0
+
+
+class RepoReader(object):
+  """Class for reading repositories."""
+
+  def __init__(self, filename):
+    self.filename = filename
+    self.main_dict = {}
+    self.input_repos = []
+    self.output_repos = []
+
+  def ParseFile(self):
+    with open(self.filename) as f:
+      self.main_dict = json.load(f)
+      self.CreateReposFromDict(self.main_dict)
+    return [self.input_repos, self.output_repos]
+
+  def CreateReposFromDict(self, main_dict):
+    for key, repo_list in main_dict.items():
+      for repo_dict in repo_list:
+        repo = self.CreateRepoFromDict(repo_dict)
+        if key == 'input':
+          self.input_repos.append(repo)
+        elif key == 'output':
+          self.output_repos.append(repo)
+        else:
+          logger.GetLogger().LogFatal('Unknown key: %s found' % key)
+
+  def CreateRepoFromDict(self, repo_dict):
+    repo_type = repo_dict.get('type', None)
+    repo_address = repo_dict.get('address', None)
+    repo_mappings = repo_dict.get('mappings', None)
+    repo_ignores = repo_dict.get('ignores', None)
+    repo_branch = repo_dict.get('branch', None)
+    gerrit = repo_dict.get('gerrit', None)
+    revision = repo_dict.get('revision', None)
+
+    if repo_type == 'p4':
+      repo = P4Repo(repo_address, repo_mappings, revision=revision)
+    elif repo_type == 'svn':
+      repo = SvnRepo(repo_address, repo_mappings)
+    elif repo_type == 'git':
+      repo = GitRepo(repo_address,
+                     repo_branch,
+                     mappings=repo_mappings,
+                     ignores=repo_ignores,
+                     gerrit=gerrit)
+    elif repo_type == 'file':
+      repo = FileRepo(repo_address)
+    else:
+      logger.GetLogger().LogFatal('Unknown repo type: %s' % repo_type)
+    return repo
+
+
+@logger.HandleUncaughtExceptions
+def Main(argv):
+  parser = argparse.ArgumentParser()
+  parser.add_argument('-i',
+                      '--input_file',
+                      dest='input_file',
+                      help='The input file that contains repo descriptions.')
+
+  parser.add_argument('-n',
+                      '--dry_run',
+                      dest='dry_run',
+                      action='store_true',
+                      default=False,
+                      help='Do a dry run of the push.')
+
+  parser.add_argument('-F',
+                      '--message_file',
+                      dest='message_file',
+                      default=None,
+                      help=('Use contents of the log file as the commit '
+                            'message.'))
+
+  options = parser.parse_args(argv)
+  if not options.input_file:
+    parser.print_help()
+    return 1
+  rr = RepoReader(options.input_file)
+  [input_repos, output_repos] = rr.ParseFile()
+
+  # Make sure FileRepo is not used as output destination.
+  for output_repo in output_repos:
+    if output_repo.repo_type == 'file':
+      logger.GetLogger().LogFatal(
+          'FileRepo is only supported as an input repo.')
+
+  for output_repo in output_repos:
+    ret = output_repo.SetupForPush()
+    if ret:
+      return ret
+
+  input_revisions = []
+  for input_repo in input_repos:
+    ret = input_repo.PullSources()
+    if ret:
+      return ret
+    input_revisions.append(input_repo.revision)
+
+  for input_repo in input_repos:
+    for output_repo in output_repos:
+      ret = input_repo.MapSources(output_repo.GetRoot())
+      if ret:
+        return ret
+
+  commit_message = 'Synced repos to: %s' % ','.join(input_revisions)
+  for output_repo in output_repos:
+    ret = output_repo.PushSources(commit_message=commit_message,
+                                  dry_run=options.dry_run,
+                                  message_file=options.message_file)
+    if ret:
+      return ret
+
+  if not options.dry_run:
+    for output_repo in output_repos:
+      output_repo.CleanupRoot()
+    for input_repo in input_repos:
+      input_repo.CleanupRoot()
+
+  return ret
+
+
+if __name__ == '__main__':
+  retval = Main(sys.argv[1:])
+  sys.exit(retval)