Add a script to delete nightly test data that are old.

TEST=tested on tc-build.hot

Change-Id: I6b6bc298b56c6ac79cbbd66f8168bc234580c884
Reviewed-on: https://chrome-internal-review.googlesource.com/150264
Reviewed-by: Yunlian Jiang <yunlian@google.com>
Commit-Queue: Han Shen <shenhan@google.com>
Tested-by: Han Shen <shenhan@google.com>

5 files changed, 169 insertions, 20 deletions

diff --git a/auto_delete_nightly_test_data.py b/auto_delete_nightly_test_data.py
new file mode 100755
index 00000000..86f8e558
--- /dev/null
+++ b/auto_delete_nightly_test_data.py
@@ -0,0 +1,80 @@
+#!/usr/bin/python
+
+"""A crontab script to delete night test data."""
+__author__ = 'shenhan@google.com (Han Shen)'
+
+import datetime
+import optparse
+import os
+import sys
+
+from utils import command_executer
+from utils import constants
+from utils import misc
+
+DIR_BY_WEEKDAY = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')
+
+
+def CleanNumberedDir(s, dry_run=False):
+  """Deleted directories under each dated_dir."""
+  chromeos_dirs = [os.path.join(s, x) for x in os.listdir(s)
+                   if misc.IsChromeOsTree(os.path.join(s, x))]
+  ce = command_executer.GetCommandExecuter()
+  for cd in chromeos_dirs:
+    misc.DeleteChromeOsTree(cd, dry_run=dry_run)
+  ## Now delete the numbered dir
+  cmd = 'rm -fr {0}'.format(s)
+  if dry_run:
+    print cmd
+  else:
+    ce.RunCommand(cmd, return_output=True, terminated_timeout=480)
+
+
+def CleanDatedDir(dated_dir, dry_run=False):
+  # List subdirs under dir
+  subdirs = [os.path.join(dated_dir, x) for x in os.listdir(dated_dir)
+             if os.path.isdir(os.path.join(dated_dir, x))]
+  for s in subdirs:
+    CleanNumberedDir(s, dry_run)
+
+
+def ProcessArguments(argv):
+  """Process arguments."""
+  parser = optparse.OptionParser(
+      description='Automatically delete nightly test data directories.',
+      usage='auto_delete.py options')
+  parser.add_option('-d', '--dry_run', dest='dry_run',
+                    default=False, action='store_true',
+                    help='Only print command line, do not execute anything.')
+  parser.add_option('--days_to_perserve', dest='days_to_preserve', default=3,
+                    help=('Specify the number of days, test data generated '
+                          'on these days will *NOT* be deleted. '
+                          'Defaults to 3.'))
+  options, _ = parser.parse_args(argv)
+  return options
+
+
+def Main(argv):
+  """Delete nightly test data directories."""
+  options = ProcessArguments(argv)
+  # Function 'isoweekday' returns 1(Monday) - 7 (Sunday).
+  d = datetime.datetime.today().isoweekday()
+  # We go back 1 week, delete from that day till we are
+  # options.days_to_preserve away from today.
+  s = d - 7
+  e = d - options.days_to_preserve
+  for i in range(s + 1, e):
+    if i <= 0:
+      ## Wrap around if index is negative.  6 is from i + 7 - 1, because
+      ## DIR_BY_WEEKDAY starts from 0, while isoweekday is from 1-7.
+      dated_dir = DIR_BY_WEEKDAY[i+6]
+    else:
+      dated_dir = DIR_BY_WEEKDAY[i-1]
+    CleanDatedDir(os.path.join(
+        constants.CROSTC_WORKSPACE, dated_dir), options.dry_run)
+  return 0
+
+
+if __name__ == '__main__':
+  retval = Main(sys.argv)
+  sys.exit(retval)
diff --git a/build_tc.py b/build_tc.py
index 02918b68..5419b55d 100755
--- a/build_tc.py
+++ b/build_tc.py
@@ -38,7 +38,7 @@ class ToolchainPart(object):
         "etc/portage/package.mask/cross-%s" % self._ctarget)
     self._new_mask_file = None
 
-    self._chroot_source_path = os.path.join(constants.mounted_toolchain_root,
+    self._chroot_source_path = os.path.join(constants.MOUNTED_TOOLCHAIN_ROOT,
                                             self._name).lstrip("/")
     self._incremental = incremental
     self._build_env = build_env
diff --git a/dejagnu/run_dejagnu.py b/dejagnu/run_dejagnu.py
index 8943bb20..14bf058d 100755
--- a/dejagnu/run_dejagnu.py
+++ b/dejagnu/run_dejagnu.py
@@ -99,7 +99,7 @@ class DejagnuExecuter(object):
     self._chromeos_chroot = path.join(chromeos_root, 'chroot')
     if mount:
       self._gcc_source_dir_to_mount = mount
-      self._gcc_source_dir = path.join(constants.mounted_toolchain_root, 'gcc')
+      self._gcc_source_dir = path.join(constants.MOUNTED_TOOLCHAIN_ROOT, 'gcc')
     else:
       self._gcc_source_dir = None
 
diff --git a/utils/constants.py b/utils/constants.py
index fdefeb28..abf9ff11 100644
--- a/utils/constants.py
+++ b/utils/constants.py
@@ -5,6 +5,9 @@
 """Generic constants used accross modules.
 """
 
-__author__ = "shenhan@google.com (Han Shen)"
+__author__ = 'shenhan@google.com (Han Shen)'
 
-mounted_toolchain_root='/usr/local/toolchain_root'
+MOUNTED_TOOLCHAIN_ROOT = '/usr/local/toolchain_root'
+
+# Root directory for night testing run.
+CROSTC_WORKSPACE = '/usr/local/google/crostc'
diff --git a/utils/misc.py b/utils/misc.py
index 61a0feb9..9db6fd53 100644
--- a/utils/misc.py
+++ b/utils/misc.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 
-# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Copyright 2013 Google Inc. All Rights Reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
@@ -154,9 +154,9 @@ def GetBuildPackagesCommand(board, usepkg=False, debug=False):
   else:
     usepkg_flag = "--nousepkg"
   if debug:
-    withdebug_flag = '--withdebug'
+    withdebug_flag = "--withdebug"
   else:
-    withdebug_flag = '--nowithdebug'
+    withdebug_flag = "--nowithdebug"
   return ("./build_packages %s --withdev --withtest --withautotest "
           "--skip_toolchain_update %s --board=%s "
           "--accept_licenses=@CHROMEOS" %
@@ -169,6 +169,7 @@ def GetBuildImageCommand(board, dev=False):
     dev_args = "--noenable_rootfs_verification --disk_layout=2gb-rootfs"
   return "./build_image --board=%s %s test" % (board, dev_args)
 
+
 def GetSetupBoardCommand(board, gcc_version=None, binutils_version=None,
                          usepkg=None, force=None):
   """Get setup_board command."""
@@ -303,8 +304,9 @@ def IsFloat(text):
   except ValueError:
     return False
 
+
 def RemoveChromeBrowserObjectFiles(chromeos_root, board):
-  """ Remove any object files from all the posible locations """
+  """Remove any object files from all the posible locations."""
   out_dir = os.path.join(
       GetChrootPath(chromeos_root),
       "var/cache/chromeos-chrome/chrome-src/src/out_%s" % board)
@@ -318,6 +320,7 @@ def RemoveChromeBrowserObjectFiles(chromeos_root, board):
     shutil.rmtree(out_dir)
     logger.GetLogger().LogCmd("rm -rf %s" % out_dir)
 
+
 @contextmanager
 def WorkingDirectory(new_dir):
   """Get the working directory."""
@@ -332,27 +335,38 @@ def WorkingDirectory(new_dir):
     logger.GetLogger().LogCmd(msg)
   os.chdir(old_dir)
 
+
 def HasGitStagedChanges(git_dir):
   """Return True if git repository has staged changes."""
-  command = 'cd {0} && git diff --quiet --cached --exit-code HEAD'.format(
-    git_dir)
+  command = "cd {0} && git diff --quiet --cached --exit-code HEAD".format(
+      git_dir)
   return command_executer.GetCommandExecuter().RunCommand(
-    command, print_to_console=False)
+      command, print_to_console=False)
+
 
 def HasGitUnstagedChanges(git_dir):
   """Return True if git repository has un-staged changes."""
-  command = 'cd {0} && git diff --quiet --exit-code HEAD'.format(git_dir)
+  command = "cd {0} && git diff --quiet --exit-code HEAD".format(git_dir)
   return command_executer.GetCommandExecuter().RunCommand(
-    command, print_to_console=False)
+      command, print_to_console=False)
+
 
 def HasGitUntrackedChanges(git_dir):
   """Return True if git repository has un-tracked changes."""
-  command = 'cd {0} && test -z $(git ls-files --exclude-standard --others)' \
-      .format(git_dir)
+  command = ("cd {0} && test -z "
+             "$(git ls-files --exclude-standard --others)").format(git_dir)
   return command_executer.GetCommandExecuter().RunCommand(
-    command,print_to_console=False)
+      command, print_to_console=False)
+
 
 def IsGitTreeClean(git_dir):
+  """Test if git tree has no local changes.
+
+  Args:
+    git_dir: git tree directory.
+  Returns:
+    True if git dir is clean.
+  """
   if HasGitStagedChanges(git_dir):
     logger.GetLogger().LogWarning("Git tree has staged changes.")
     return False
@@ -364,15 +378,67 @@ def IsGitTreeClean(git_dir):
     return False
   return True
 
+
 def GetGitChangesAsList(git_dir, path=None, staged=False):
-  command = 'cd {0} && git diff --name-only'.format(git_dir)
+  """Get changed files as a list.
+
+  Args:
+    git_dir: git tree directory.
+    path: a relative path that is part of the tree directory, could be null.
+    staged: whether to include staged files as well.
+  Returns:
+    A list containing all the changed files.
+  """
+  command = "cd {0} && git diff --name-only".format(git_dir)
   if staged:
-    command = command + ' --cached'
+    command += " --cached"
   if path:
-    command = command + ' -- ' + path
+    command += " -- " + path
   _, out, _ = command_executer.GetCommandExecuter().RunCommand(
-    command, return_output=True, print_to_console=False)
+      command, return_output=True, print_to_console=False)
   rv = []
   for line in out.splitlines():
     rv.append(line)
   return rv
+
+
+def IsChromeOsTree(chromeos_root):
+  return (os.path.isdir(os.path.join(
+      chromeos_root, "src/third_party/chromiumos-overlay")) and
+          os.path.isdir(os.path.join(
+              chromeos_root, "manifest")))
+
+
+def DeleteChromeOsTree(chromeos_root, dry_run=False):
+  """Delete a ChromeOs tree *safely*.
+
+  Args:
+    chromeos_root: dir of the tree, could be a relative one (but be careful)
+    dry_run: only prints out the command if True
+
+  Returns:
+    True if everything is ok.
+  """
+  if not IsChromeOsTree(chromeos_root):
+    logger.GetLogger().LogWarning(
+        '"{0}" does not seem to be a valid chromeos tree, do nothing.'.format(
+            chromeos_root))
+    return False
+  cmd0 = "cd {0} && cros_sdk --delete".format(chromeos_root)
+  if dry_run:
+    print cmd0
+  else:
+    if command_executer.GetCommandExecuter().RunCommand(
+        cmd0, return_output=False, print_to_console=True) != 0:
+      return False
+
+  cmd1 = ('export CHROMEOSDIRNAME="$(dirname $(cd {0} && pwd))" && '
+          'export CHROMEOSBASENAME="$(basename $(cd {0} && pwd))" && '
+          'cd $CHROMEOSDIRNAME && sudo rm -fr $CHROMEOSBASENAME').format(
+              chromeos_root)
+  if dry_run:
+    print cmd1
+    return True
+
+  return command_executer.GetCommandExecuter().RunCommand(
+      cmd1, return_output=False, print_to_console=True) == 0