diff options
Diffstat (limited to 'auto_delete_nightly_test_data.py')
| -rwxr-xr-x | auto_delete_nightly_test_data.py | 114 |
1 files changed, 85 insertions, 29 deletions
diff --git a/auto_delete_nightly_test_data.py b/auto_delete_nightly_test_data.py index 429d0b4a..67841188 100755 --- a/auto_delete_nightly_test_data.py +++ b/auto_delete_nightly_test_data.py @@ -15,7 +15,10 @@ import argparse import datetime import os import re +import shutil +import shlex import sys +import time from cros_utils import command_executer from cros_utils import constants @@ -47,10 +50,10 @@ def CleanNumberedDir(s, dry_run=False): return False ## Now delete the numbered dir Before forcibly removing the directory, just - ## check 's' to make sure it is sane. A valid dir to be removed must be - ## '/usr/local/google/crostc/(SUN|MON|TUE...|SAT)'. - valid_dir_pattern = ( - '^' + NIGHTLY_TESTS_WORKSPACE + '/(' + '|'.join(DIR_BY_WEEKDAY) + ')') + ## check 's' to make sure it matches the expected pattern. A valid dir to be + ## removed must be '/usr/local/google/crostc/(SUN|MON|TUE...|SAT)'. + valid_dir_pattern = ('^' + NIGHTLY_TESTS_WORKSPACE + '/(' + + '|'.join(DIR_BY_WEEKDAY) + ')') if not re.search(valid_dir_pattern, s): print('Trying to delete an invalid dir "{0}" (must match "{1}"), ' 'please check.'.format(s, valid_dir_pattern)) @@ -134,30 +137,33 @@ def CleanChromeOsTmpFiles(chroot_tmp, days_to_preserve, dry_run): def CleanChromeOsImageFiles(chroot_tmp, subdir_suffix, days_to_preserve, dry_run): - rv = 0 - rv2 = 0 - ce = command_executer.GetCommandExecuter() - minutes = 1440 * days_to_preserve # Clean files that were last accessed more than the specified time. - rv2 = 0 - cmd = (r'find {0}/*{1}/* -maxdepth 1 -type d ' - r'-amin +{2} ' - r'-exec bash -c "echo rm -fr {{}}" \; ' - r'-exec bash -c "rm -fr {{}}" \;').format(chroot_tmp, subdir_suffix, - minutes) - if dry_run: - print('Going to execute:\n%s' % cmd) - else: - rv2 = ce.RunCommand(cmd, print_to_console=False) - if rv2 == 0: - print('Successfully cleaned chromeos image autotest directories from ' - '"{0}/*{1}".'.format(chroot_tmp, subdir_suffix)) - else: - print('Some image autotest directories were not removed from ' - '"{0}/*{1}".'.format(chroot_tmp, subdir_suffix)) + seconds_delta = days_to_preserve * 24 * 3600 + now = time.time() + errors = 0 - rv += rv2 - return rv + for tmp_dir in os.listdir(chroot_tmp): + # Directory under /tmp + tmp_dir = os.path.join(chroot_tmp, tmp_dir) + if tmp_dir.endswith(subdir_suffix): + # Tmp directory which ends with subdir_suffix. + for subdir in os.listdir(tmp_dir): + # Subdirectories targeted for deletion. + subdir_path = os.path.join(tmp_dir, subdir) + if now - os.path.getatime(subdir_path) > seconds_delta: + if dry_run: + print('Will run:\nshutil.rmtree({})'.format(subdir_path)) + else: + try: + shutil.rmtree(subdir_path) + print('Successfully cleaned chromeos image autotest directories ' + 'from "{}".'.format(subdir_path)) + except OSError: + print('Some image autotest directories were not removed from ' + '"{}".'.format(subdir_path)) + errors += 1 + + return errors def CleanChromeOsTmpAndImages(days_to_preserve=1, dry_run=False): @@ -175,10 +181,54 @@ def CleanChromeOsTmpAndImages(days_to_preserve=1, dry_run=False): # Clean image files in *-pfq directories rv += CleanChromeOsImageFiles(chromeos_chroot_tmp, '-pfq', days_to_preserve, dry_run) + # Clean image files in *-llvm-next-nightly directories + rv += CleanChromeOsImageFiles(chromeos_chroot_tmp, '-llvm-next-nightly', + days_to_preserve, dry_run) return rv +def CleanOldCLs(days_to_preserve='1', dry_run=False): + """Abandon old CLs created by automation tooling.""" + ce = command_executer.GetCommandExecuter() + chromeos_root = os.path.join(constants.CROSTC_WORKSPACE, 'chromeos') + # Find Old CLs. + old_cls_cmd = ('gerrit --raw search "owner:me status:open age:%sd"' % + days_to_preserve) + _, cls, _ = ce.ChrootRunCommandWOutput( + chromeos_root, old_cls_cmd, print_to_console=False) + # Convert any whitespaces to spaces. + cls = ' '.join(cls.split()) + if not cls: + return 0 + + abandon_cls_cmd = ('gerrit abandon %s' % cls) + if dry_run: + print('Going to execute: %s' % abandon_cls_cmd) + return 0 + + return ce.ChrootRunCommand( + chromeos_root, abandon_cls_cmd, print_to_console=False) + + +def CleanChromeTelemetryTmpFiles(dry_run): + rv = 0 + ce = command_executer.GetCommandExecuter() + tmp_dir = os.path.join(constants.CROSTC_WORKSPACE, 'chromeos', '.cache', + 'distfiles', 'chrome-src-internal', 'src', 'tmp') + cmd = f'rm -fr {shlex.quote(tmp_dir)}/tmp*telemetry_Crosperf' + if dry_run: + print(f'Going to execute:\n{cmd}') + else: + rv = ce.RunCommand(cmd, print_to_console=False) + if rv == 0: + print(f'Successfully cleaned chrome tree tmp directory ' f'{tmp_dir!r} .') + else: + print(f'Some directories were not removed under chrome tree ' + f'tmp directory {tmp_dir!r}.') + return rv + + def Main(argv): """Delete nightly test data directories, tmps and test images.""" options = ProcessArguments(argv) @@ -201,12 +251,18 @@ def Main(argv): os.path.join(NIGHTLY_TESTS_WORKSPACE, dated_dir), options.dry_run) else 1 - -## Finally clean temporaries, images under crostc/chromeos + ## Clean temporaries, images under crostc/chromeos rv2 = CleanChromeOsTmpAndImages( int(options.days_to_preserve), options.dry_run) - return rv + rv2 + # Clean CLs that are not updated in last 2 weeks. + rv3 = CleanOldCLs('14', options.dry_run) + + # Clean telemetry temporaries from chrome source tree inside chroot. + rv4 = CleanChromeTelemetryTmpFiles(options.dry_run) + + return rv + rv2 + rv3 + rv4 + if __name__ == '__main__': retval = Main(sys.argv[1:]) |