diff options
Diffstat (limited to 'deprecated/automation/server')
25 files changed, 0 insertions, 1425 deletions
diff --git a/deprecated/automation/server/__init__.py b/deprecated/automation/server/__init__.py deleted file mode 100644 index 8b137891..00000000 --- a/deprecated/automation/server/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/deprecated/automation/server/job_executer.py b/deprecated/automation/server/job_executer.py deleted file mode 100644 index 30b59463..00000000 --- a/deprecated/automation/server/job_executer.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2010 Google Inc. All Rights Reserved. -# - -import logging -import os.path -import threading - -from automation.common import command as cmd -from automation.common import job -from automation.common import logger -from automation.common.command_executer import LoggingCommandExecuter -from automation.common.command_executer import CommandTerminator - - -class JobExecuter(threading.Thread): - - def __init__(self, job_to_execute, machines, listeners): - threading.Thread.__init__(self) - - assert machines - - self.job = job_to_execute - self.listeners = listeners - self.machines = machines - - # Set Thread name. - self.name = '%s-%s' % (self.__class__.__name__, self.job.id) - - self._logger = logging.getLogger(self.__class__.__name__) - self._executer = LoggingCommandExecuter(self.job.dry_run) - self._terminator = CommandTerminator() - - def _RunRemotely(self, command, fail_msg, command_timeout=1 * 60 * 60): - exit_code = self._executer.RunCommand(command, - self.job.primary_machine.hostname, - self.job.primary_machine.username, - command_terminator=self._terminator, - command_timeout=command_timeout) - if exit_code: - raise job.JobFailure(fail_msg, exit_code) - - def _RunLocally(self, command, fail_msg, command_timeout=1 * 60 * 60): - exit_code = self._executer.RunCommand(command, - command_terminator=self._terminator, - command_timeout=command_timeout) - if exit_code: - raise job.JobFailure(fail_msg, exit_code) - - def Kill(self): - self._terminator.Terminate() - - def CleanUpWorkDir(self): - self._logger.debug('Cleaning up %r work directory.', self.job) - self._RunRemotely(cmd.RmTree(self.job.work_dir), 'Cleanup workdir failed.') - - def CleanUpHomeDir(self): - self._logger.debug('Cleaning up %r home directory.', self.job) - self._RunLocally(cmd.RmTree(self.job.home_dir), 'Cleanup homedir failed.') - - def _PrepareRuntimeEnvironment(self): - self._RunRemotely( - cmd.MakeDir(self.job.work_dir, self.job.logs_dir, self.job.results_dir), - 'Creating new job directory failed.') - - # The log directory is ready, so we can prepare to log command's output. - self._executer.OpenLog(os.path.join(self.job.logs_dir, - self.job.log_filename_prefix)) - - def _SatisfyFolderDependencies(self): - for dependency in self.job.folder_dependencies: - to_folder = os.path.join(self.job.work_dir, dependency.dest) - from_folder = os.path.join(dependency.job.work_dir, dependency.src) - from_machine = dependency.job.primary_machine - - if from_machine == self.job.primary_machine and dependency.read_only: - # No need to make a copy, just symlink it - self._RunRemotely( - cmd.MakeSymlink(from_folder, to_folder), - 'Failed to create symlink to required directory.') - else: - self._RunRemotely( - cmd.RemoteCopyFrom(from_machine.hostname, - from_folder, - to_folder, - username=from_machine.username), - 'Failed to copy required files.') - - def _LaunchJobCommand(self): - command = self.job.GetCommand() - - self._RunRemotely('%s; %s' % ('PS1=. TERM=linux source ~/.bashrc', - cmd.Wrapper(command, - cwd=self.job.work_dir)), - "Command failed to execute: '%s'." % command, - self.job.timeout) - - def _CopyJobResults(self): - """Copy test results back to directory.""" - self._RunLocally( - cmd.RemoteCopyFrom(self.job.primary_machine.hostname, - self.job.results_dir, - self.job.home_dir, - username=self.job.primary_machine.username), - 'Failed to copy results.') - - def run(self): - self.job.status = job.STATUS_SETUP - self.job.machines = self.machines - self._logger.debug('Executing %r on %r in directory %s.', self.job, - self.job.primary_machine.hostname, self.job.work_dir) - - try: - self.CleanUpWorkDir() - - self._PrepareRuntimeEnvironment() - - self.job.status = job.STATUS_COPYING - - self._SatisfyFolderDependencies() - - self.job.status = job.STATUS_RUNNING - - self._LaunchJobCommand() - self._CopyJobResults() - - # If we get here, the job succeeded. - self.job.status = job.STATUS_SUCCEEDED - except job.JobFailure as ex: - self._logger.error('Job failed. Exit code %s. %s', ex.exit_code, ex) - if self._terminator.IsTerminated(): - self._logger.info('%r was killed', self.job) - - self.job.status = job.STATUS_FAILED - - self._executer.CloseLog() - - for listener in self.listeners: - listener.NotifyJobComplete(self.job) diff --git a/deprecated/automation/server/job_group_manager.py b/deprecated/automation/server/job_group_manager.py deleted file mode 100644 index d66f5e07..00000000 --- a/deprecated/automation/server/job_group_manager.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2010 Google Inc. All Rights Reserved. -# - -import copy -import logging -import threading - -from automation.common import command as cmd -from automation.common import logger -from automation.common.command_executer import CommandExecuter -from automation.common import job -from automation.common import job_group -from automation.server.job_manager import IdProducerPolicy - - -class JobGroupManager(object): - - def __init__(self, job_manager): - self.all_job_groups = [] - - self.job_manager = job_manager - self.job_manager.AddListener(self) - - self._lock = threading.Lock() - self._job_group_finished = threading.Condition(self._lock) - - self._id_producer = IdProducerPolicy() - self._id_producer.Initialize(job_group.JobGroup.HOMEDIR_PREFIX, - 'job-group-(?P<id>\d+)') - - self._logger = logging.getLogger(self.__class__.__name__) - - def GetJobGroup(self, group_id): - with self._lock: - for group in self.all_job_groups: - if group.id == group_id: - return group - - return None - - def GetAllJobGroups(self): - with self._lock: - return copy.deepcopy(self.all_job_groups) - - def AddJobGroup(self, group): - with self._lock: - group.id = self._id_producer.GetNextId() - - self._logger.debug('Creating runtime environment for %r.', group) - - CommandExecuter().RunCommand(cmd.Chain( - cmd.RmTree(group.home_dir), cmd.MakeDir(group.home_dir))) - - with self._lock: - self.all_job_groups.append(group) - - for job_ in group.jobs: - self.job_manager.AddJob(job_) - - group.status = job_group.STATUS_EXECUTING - - self._logger.info('Added %r to queue.', group) - - return group.id - - def KillJobGroup(self, group): - with self._lock: - self._logger.debug('Killing all jobs that belong to %r.', group) - - for job_ in group.jobs: - self.job_manager.KillJob(job_) - - self._logger.debug('Waiting for jobs to quit.') - - # Lets block until the group is killed so we know it is completed - # when we return. - while group.status not in [job_group.STATUS_SUCCEEDED, - job_group.STATUS_FAILED]: - self._job_group_finished.wait() - - def NotifyJobComplete(self, job_): - self._logger.debug('Handling %r completion event.', job_) - - group = job_.group - - with self._lock: - # We need to perform an action only if the group hasn't already failed. - if group.status != job_group.STATUS_FAILED: - if job_.status == job.STATUS_FAILED: - # We have a failed job, abort the job group - group.status = job_group.STATUS_FAILED - if group.cleanup_on_failure: - for job_ in group.jobs: - # TODO(bjanakiraman): We should probably only kill dependent jobs - # instead of the whole job group. - self.job_manager.KillJob(job_) - self.job_manager.CleanUpJob(job_) - else: - # The job succeeded successfully -- lets check to see if we are done. - assert job_.status == job.STATUS_SUCCEEDED - finished = True - for other_job in group.jobs: - assert other_job.status != job.STATUS_FAILED - if other_job.status != job.STATUS_SUCCEEDED: - finished = False - break - - if finished and group.status != job_group.STATUS_SUCCEEDED: - # TODO(kbaclawski): Without check performed above following code - # could be called more than once. This would trigger StateMachine - # crash, because it cannot transition from STATUS_SUCCEEDED to - # STATUS_SUCCEEDED. Need to address that bug in near future. - group.status = job_group.STATUS_SUCCEEDED - if group.cleanup_on_completion: - for job_ in group.jobs: - self.job_manager.CleanUpJob(job_) - - self._job_group_finished.notifyAll() diff --git a/deprecated/automation/server/job_manager.py b/deprecated/automation/server/job_manager.py deleted file mode 100644 index 7a65b918..00000000 --- a/deprecated/automation/server/job_manager.py +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright 2010 Google Inc. All Rights Reserved. -# - -import logging -import os -import re -import threading - -from automation.common import job -from automation.common import logger -from automation.server.job_executer import JobExecuter - - -class IdProducerPolicy(object): - """Produces series of unique integer IDs. - - Example: - id_producer = IdProducerPolicy() - id_a = id_producer.GetNextId() - id_b = id_producer.GetNextId() - assert id_a != id_b - """ - - def __init__(self): - self._counter = 1 - - def Initialize(self, home_prefix, home_pattern): - """Find first available ID based on a directory listing. - - Args: - home_prefix: A directory to be traversed. - home_pattern: A regexp describing all files/directories that will be - considered. The regexp must contain exactly one match group with name - "id", which must match an integer number. - - Example: - id_producer.Initialize(JOBDIR_PREFIX, 'job-(?P<id>\d+)') - """ - harvested_ids = [] - - if os.path.isdir(home_prefix): - for filename in os.listdir(home_prefix): - path = os.path.join(home_prefix, filename) - - if os.path.isdir(path): - match = re.match(home_pattern, filename) - - if match: - harvested_ids.append(int(match.group('id'))) - - self._counter = max(harvested_ids or [0]) + 1 - - def GetNextId(self): - """Calculates another ID considered to be unique.""" - new_id = self._counter - self._counter += 1 - return new_id - - -class JobManager(threading.Thread): - - def __init__(self, machine_manager): - threading.Thread.__init__(self, name=self.__class__.__name__) - self.all_jobs = [] - self.ready_jobs = [] - self.job_executer_mapping = {} - - self.machine_manager = machine_manager - - self._lock = threading.Lock() - self._jobs_available = threading.Condition(self._lock) - self._exit_request = False - - self.listeners = [] - self.listeners.append(self) - - self._id_producer = IdProducerPolicy() - self._id_producer.Initialize(job.Job.WORKDIR_PREFIX, 'job-(?P<id>\d+)') - - self._logger = logging.getLogger(self.__class__.__name__) - - def StartJobManager(self): - self._logger.info('Starting...') - - with self._lock: - self.start() - self._jobs_available.notifyAll() - - def StopJobManager(self): - self._logger.info('Shutdown request received.') - - with self._lock: - for job_ in self.all_jobs: - self._KillJob(job_.id) - - # Signal to die - self._exit_request = True - self._jobs_available.notifyAll() - - # Wait for all job threads to finish - for executer in self.job_executer_mapping.values(): - executer.join() - - def KillJob(self, job_id): - """Kill a job by id. - - Does not block until the job is completed. - """ - with self._lock: - self._KillJob(job_id) - - def GetJob(self, job_id): - for job_ in self.all_jobs: - if job_.id == job_id: - return job_ - return None - - def _KillJob(self, job_id): - self._logger.info('Killing [Job: %d].', job_id) - - if job_id in self.job_executer_mapping: - self.job_executer_mapping[job_id].Kill() - for job_ in self.ready_jobs: - if job_.id == job_id: - self.ready_jobs.remove(job_) - break - - def AddJob(self, job_): - with self._lock: - job_.id = self._id_producer.GetNextId() - - self.all_jobs.append(job_) - # Only queue a job as ready if it has no dependencies - if job_.is_ready: - self.ready_jobs.append(job_) - - self._jobs_available.notifyAll() - - return job_.id - - def CleanUpJob(self, job_): - with self._lock: - if job_.id in self.job_executer_mapping: - self.job_executer_mapping[job_.id].CleanUpWorkDir() - del self.job_executer_mapping[job_.id] - # TODO(raymes): remove job from self.all_jobs - - def NotifyJobComplete(self, job_): - self.machine_manager.ReturnMachines(job_.machines) - - with self._lock: - self._logger.debug('Handling %r completion event.', job_) - - if job_.status == job.STATUS_SUCCEEDED: - for succ in job_.successors: - if succ.is_ready: - if succ not in self.ready_jobs: - self.ready_jobs.append(succ) - - self._jobs_available.notifyAll() - - def AddListener(self, listener): - self.listeners.append(listener) - - @logger.HandleUncaughtExceptions - def run(self): - self._logger.info('Started.') - - while not self._exit_request: - with self._lock: - # Get the next ready job, block if there are none - self._jobs_available.wait() - - while self.ready_jobs: - ready_job = self.ready_jobs.pop() - - required_machines = ready_job.machine_dependencies - for pred in ready_job.predecessors: - required_machines[0].AddPreferredMachine( - pred.primary_machine.hostname) - - machines = self.machine_manager.GetMachines(required_machines) - if not machines: - # If we can't get the necessary machines right now, simply wait - # for some jobs to complete - self.ready_jobs.insert(0, ready_job) - break - else: - # Mark as executing - executer = JobExecuter(ready_job, machines, self.listeners) - executer.start() - self.job_executer_mapping[ready_job.id] = executer - - self._logger.info('Stopped.') diff --git a/deprecated/automation/server/machine_manager.py b/deprecated/automation/server/machine_manager.py deleted file mode 100644 index b7186077..00000000 --- a/deprecated/automation/server/machine_manager.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2010 Google Inc. All Rights Reserved. - -__author__ = 'asharif@google.com (Ahmad Sharif)' - -from operator import attrgetter -import copy -import csv -import threading -import os.path - -from automation.common import machine - -DEFAULT_MACHINES_FILE = os.path.join(os.path.dirname(__file__), 'test_pool.csv') - - -class MachineManager(object): - """Container for list of machines one can run jobs on.""" - - @classmethod - def FromMachineListFile(cls, filename): - # Read the file and skip header - csv_file = csv.reader(open(filename, 'rb'), delimiter=',', quotechar='"') - csv_file.next() - - return cls([machine.Machine(hostname, label, cpu, int(cores), os, user) - for hostname, label, cpu, cores, os, user in csv_file]) - - def __init__(self, machines): - self._machine_pool = machines - self._lock = threading.RLock() - - def _GetMachine(self, mach_spec): - available_pool = [m for m in self._machine_pool if mach_spec.IsMatch(m)] - - if available_pool: - # find a machine with minimum uses - uses = attrgetter('uses') - - mach = min(available_pool, key=uses) - - if mach_spec.preferred_machines: - preferred_pool = [m - for m in available_pool - if m.hostname in mach_spec.preferred_machines] - if preferred_pool: - mach = min(preferred_pool, key=uses) - - mach.Acquire(mach_spec.lock_required) - - return mach - - def GetMachines(self, required_machines): - """Acquire machines for use by a job.""" - - with self._lock: - acquired_machines = [self._GetMachine(ms) for ms in required_machines] - - if not all(acquired_machines): - # Roll back acquires - while acquired_machines: - mach = acquired_machines.pop() - if mach: - mach.Release() - - return acquired_machines - - def GetMachineList(self): - with self._lock: - return copy.deepcopy(self._machine_pool) - - def ReturnMachines(self, machines): - with self._lock: - for m in machines: - m.Release() - - def __str__(self): - return str(self._machine_pool) diff --git a/deprecated/automation/server/machine_manager_test.py b/deprecated/automation/server/machine_manager_test.py deleted file mode 100755 index 2fa5bb4b..00000000 --- a/deprecated/automation/server/machine_manager_test.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/python2 -# -# Copyright 2010 Google Inc. All Rights Reserved. - -__author__ = 'asharif@google.com (Ahmad Sharif)' - -import unittest -from automation.common import machine -from automation.server import machine_manager - - -class MachineManagerTest(unittest.TestCase): - - def setUp(self): - self.machine_manager = machine_manager.MachineManager() - - def testPrint(self): - print self.machine_manager - - def testGetLinuxBox(self): - mach_spec_list = [machine.MachineSpecification(os='linux')] - machines = self.machine_manager.GetMachines(mach_spec_list) - self.assertTrue(machines) - - def testGetChromeOSBox(self): - mach_spec_list = [machine.MachineSpecification(os='chromeos')] - machines = self.machine_manager.GetMachines(mach_spec_list) - self.assertTrue(machines) - - -if __name__ == '__main__': - unittest.main() diff --git a/deprecated/automation/server/monitor/__init__.py b/deprecated/automation/server/monitor/__init__.py deleted file mode 100644 index 8b137891..00000000 --- a/deprecated/automation/server/monitor/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/deprecated/automation/server/monitor/dashboard.py b/deprecated/automation/server/monitor/dashboard.py deleted file mode 100644 index f6befed8..00000000 --- a/deprecated/automation/server/monitor/dashboard.py +++ /dev/null @@ -1,259 +0,0 @@ -# Copyright 2011 Google Inc. All Rights Reserved. -# - -__author__ = 'kbaclawski@google.com (Krystian Baclawski)' - -from collections import namedtuple -import glob -import gzip -import os.path -import pickle -import time -import xmlrpclib - -from django import forms -from django.http import HttpResponseRedirect -from django.shortcuts import render_to_response -from django.template import Context -from django.views import static - -Link = namedtuple('Link', 'href name') - - -def GetServerConnection(): - return xmlrpclib.Server('http://localhost:8000') - - -def MakeDefaultContext(*args): - context = Context({'links': [ - Link('/job-group', 'Job Groups'), Link('/machine', 'Machines') - ]}) - - for arg in args: - context.update(arg) - - return context - - -class JobInfo(object): - - def __init__(self, job_id): - self._job = pickle.loads(GetServerConnection().GetJob(job_id)) - - def GetAttributes(self): - job = self._job - - group = [Link('/job-group/%d' % job.group.id, job.group.label)] - - predecessors = [Link('/job/%d' % pred.id, pred.label) - for pred in job.predecessors] - - successors = [Link('/job/%d' % succ.id, succ.label) - for succ in job.successors] - - machines = [Link('/machine/%s' % mach.hostname, mach.hostname) - for mach in job.machines] - - logs = [Link('/job/%d/log' % job.id, 'Log')] - - commands = enumerate(job.PrettyFormatCommand().split('\n'), start=1) - - return {'text': [('Label', job.label), ('Directory', job.work_dir)], - 'link': [('Group', group), ('Predecessors', predecessors), - ('Successors', successors), ('Machines', machines), - ('Logs', logs)], - 'code': [('Command', commands)]} - - def GetTimeline(self): - return [{'started': evlog.GetTimeStartedFormatted(), - 'state_from': evlog.event.from_, - 'state_to': evlog.event.to_, - 'elapsed': evlog.GetTimeElapsedRounded()} - for evlog in self._job.timeline.GetTransitionEventHistory()] - - def GetLog(self): - log_path = os.path.join(self._job.logs_dir, - '%s.gz' % self._job.log_filename_prefix) - - try: - log = gzip.open(log_path, 'r') - except IOError: - content = [] - else: - # There's a good chance that file is not closed yet, so EOF handling - # function and CRC calculation will fail, thus we need to monkey patch the - # _read_eof method. - log._read_eof = lambda: None - - def SplitLine(line): - prefix, msg = line.split(': ', 1) - datetime, stream = prefix.rsplit(' ', 1) - - return datetime, stream, msg - - content = map(SplitLine, log.readlines()) - finally: - log.close() - - return content - - -class JobGroupInfo(object): - - def __init__(self, job_group_id): - self._job_group = pickle.loads(GetServerConnection().GetJobGroup( - job_group_id)) - - def GetAttributes(self): - group = self._job_group - - home_dir = [Link('/job-group/%d/files/' % group.id, group.home_dir)] - - return {'text': [('Label', group.label), - ('Time submitted', time.ctime(group.time_submitted)), - ('State', group.status), - ('Cleanup on completion', group.cleanup_on_completion), - ('Cleanup on failure', group.cleanup_on_failure)], - 'link': [('Directory', home_dir)]} - - def _GetJobStatus(self, job): - status_map = {'SUCCEEDED': 'success', 'FAILED': 'failure'} - return status_map.get(str(job.status), None) - - def GetJobList(self): - return [{'id': job.id, - 'label': job.label, - 'state': job.status, - 'status': self._GetJobStatus(job), - 'elapsed': job.timeline.GetTotalTime()} - for job in self._job_group.jobs] - - def GetHomeDirectory(self): - return self._job_group.home_dir - - def GetReportList(self): - job_dir_pattern = os.path.join(self._job_group.home_dir, 'job-*') - - filenames = [] - - for job_dir in glob.glob(job_dir_pattern): - filename = os.path.join(job_dir, 'report.html') - - if os.access(filename, os.F_OK): - filenames.append(filename) - - reports = [] - - for filename in sorted(filenames, key=lambda f: os.stat(f).st_ctime): - try: - with open(filename, 'r') as report: - reports.append(report.read()) - except IOError: - pass - - return reports - - -class JobGroupListInfo(object): - - def __init__(self): - self._all_job_groups = pickle.loads(GetServerConnection().GetAllJobGroups()) - - def _GetJobGroupState(self, group): - return str(group.status) - - def _GetJobGroupStatus(self, group): - status_map = {'SUCCEEDED': 'success', 'FAILED': 'failure'} - return status_map.get(self._GetJobGroupState(group), None) - - def GetList(self): - return [{'id': group.id, - 'label': group.label, - 'submitted': time.ctime(group.time_submitted), - 'state': self._GetJobGroupState(group), - 'status': self._GetJobGroupStatus(group)} - for group in self._all_job_groups] - - def GetLabelList(self): - return sorted(set(group.label for group in self._all_job_groups)) - - -def JobPageHandler(request, job_id): - job = JobInfo(int(job_id)) - - ctx = MakeDefaultContext({ - 'job_id': job_id, - 'attributes': job.GetAttributes(), - 'timeline': job.GetTimeline() - }) - - return render_to_response('job.html', ctx) - - -def LogPageHandler(request, job_id): - job = JobInfo(int(job_id)) - - ctx = MakeDefaultContext({'job_id': job_id, 'log_lines': job.GetLog()}) - - return render_to_response('job_log.html', ctx) - - -def JobGroupPageHandler(request, job_group_id): - group = JobGroupInfo(int(job_group_id)) - - ctx = MakeDefaultContext({ - 'group_id': job_group_id, - 'attributes': group.GetAttributes(), - 'job_list': group.GetJobList(), - 'reports': group.GetReportList() - }) - - return render_to_response('job_group.html', ctx) - - -def JobGroupFilesPageHandler(request, job_group_id, path): - group = JobGroupInfo(int(job_group_id)) - - return static.serve(request, - path, - document_root=group.GetHomeDirectory(), - show_indexes=True) - - -class FilterJobGroupsForm(forms.Form): - label = forms.ChoiceField(label='Filter by label:', required=False) - - -def JobGroupListPageHandler(request): - groups = JobGroupListInfo() - group_list = groups.GetList() - - field = FilterJobGroupsForm.base_fields['label'] - field.choices = [('*', '--- no filtering ---')] - field.choices.extend([(label, label) for label in groups.GetLabelList()]) - - if request.method == 'POST': - form = FilterJobGroupsForm(request.POST) - - if form.is_valid(): - label = form.cleaned_data['label'] - - if label != '*': - group_list = [group for group in group_list if group['label'] == label] - else: - form = FilterJobGroupsForm({'initial': '*'}) - - ctx = MakeDefaultContext({'filter': form, 'groups': group_list}) - - return render_to_response('job_group_list.html', ctx) - - -def MachineListPageHandler(request): - machine_list = pickle.loads(GetServerConnection().GetMachineList()) - - return render_to_response('machine_list.html', - MakeDefaultContext({'machines': machine_list})) - - -def DefaultPageHandler(request): - return HttpResponseRedirect('/job-group') diff --git a/deprecated/automation/server/monitor/manage.py b/deprecated/automation/server/monitor/manage.py deleted file mode 100755 index 59f6e216..00000000 --- a/deprecated/automation/server/monitor/manage.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/python2 -# -# Copyright 2011 Google Inc. All Rights Reserved. -# - -__author__ = 'kbaclawski@google.com (Krystian Baclawski)' - -from django.core.management import execute_manager - -try: - import settings # Assumed to be in the same directory. -except ImportError: - import sys - - sys.stderr.write('Error: Can\'t find settings.py file in the directory ' - 'containing %r.' % __file__) - sys.exit(1) - -if __name__ == '__main__': - execute_manager(settings) diff --git a/deprecated/automation/server/monitor/settings.py b/deprecated/automation/server/monitor/settings.py deleted file mode 100644 index 8cd20e35..00000000 --- a/deprecated/automation/server/monitor/settings.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2011 Google Inc. All Rights Reserved. -# -# Django settings for monitor project. -# -# For explanation look here: http://docs.djangoproject.com/en/dev/ref/settings -# - -__author__ = 'kbaclawski@google.com (Krystian Baclawski)' - -import os.path -import sys - -# Path to the root of application. It's a custom setting, not related to Django. -ROOT_PATH = os.path.dirname(os.path.realpath(sys.argv[0])) - -# Print useful information during runtime if possible. -DEBUG = True -TEMPLATE_DEBUG = DEBUG - -# Sqlite3 database configuration, though we don't use it right now. -DATABASE_ENGINE = 'sqlite3' -DATABASE_NAME = os.path.join(ROOT_PATH, 'monitor.db') - -# Local time zone for this installation. -TIME_ZONE = 'America/Los_Angeles' - -# Language code for this installation. -LANGUAGE_CODE = 'en-us' - -# If you set this to False, Django will make some optimizations so as not -# to load the internationalization machinery. -USE_I18N = True - -# Absolute path to the directory that holds media. -MEDIA_ROOT = os.path.join(ROOT_PATH, 'static') + '/' - -# URL that handles the media served from MEDIA_ROOT. Make sure to use a -# trailing slash if there is a path component (optional in other cases). -MEDIA_URL = '/static/' - -# Used to provide a seed in secret-key hashing algorithms. Make this unique, -# and don't share it with anybody. -SECRET_KEY = '13p5p_4q91*8@yo+tvvt#2k&6#d_&e_zvxdpdil53k419i5sop' - -# A string representing the full Python import path to your root URLconf. -ROOT_URLCONF = 'monitor.urls' - -# List of locations of the template source files, in search order. -TEMPLATE_DIRS = (os.path.join(ROOT_PATH, 'templates'),) diff --git a/deprecated/automation/server/monitor/start.sh b/deprecated/automation/server/monitor/start.sh deleted file mode 100755 index 4fc53bef..00000000 --- a/deprecated/automation/server/monitor/start.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# -# Copyright 2011 Google Inc. All Rights Reserved. -# Author: kbaclawski@google.com (Krystian Baclawski) -# - -./manage.py runserver "$HOSTNAME":8080 diff --git a/deprecated/automation/server/monitor/static/style.css b/deprecated/automation/server/monitor/static/style.css deleted file mode 100644 index b571b059..00000000 --- a/deprecated/automation/server/monitor/static/style.css +++ /dev/null @@ -1,101 +0,0 @@ -* { font-family: sans-serif; } - -.left { text-align: left; } -.right { text-align: right; } - -.code { font-family: monospace; text-align: left; } -.line1 { background-color: Gainsboro; } -.line2 { background-color: WhiteSmoke; } - -.title { margin-bottom: 0.25em; } - -.success { background-color: LightGreen; } -.failure { background-color: LightPink; } - -pre.code { margin: 0px; } - -div.header p.title { - border: 1px solid black; - font-size: 32px; - font-style: bold; - background-color: LightBlue; - text-align: center; - margin: 0px; - padding: 10px; - font-weight: bold; -} - -div.links { - background-color: Azure; - margin-top: 2px; - padding: 8px 4px 8px 4px; - border: solid 1px; -} - -div.content { - margin-top: 2px; - padding: 8px; - border: solid 1px; -} - -div.content p.title { - font-size: 28px; - text-align: left; - margin: 0px; - margin-bottom: 8px; - padding: 12px; - font-weight: bold; -} - -table { border-collapse: collapse; } -td, th { text-align: center; } - -table.list td, th { padding: 3px 8px 2px 8px; border:1px solid black; } -table.list td { font-family: monospace; } -table.list th { background-color: LightGray; } - -table.attributes td { text-align: left; } -table.attributes > tbody > tr > td:first-child { font-family: sans-serif; } - -table.raw { border-style: none; } -table.raw td { - padding: 0em 0.5em 0em 0.5em; - border-style: none; - vertical-align: top; - text-align: right; - font-family: monospace; -} -table.raw > tbody > tr > td:first-child { border-left: 0px; } -table.raw > tbody > tr > td { border-left: 1px solid; } - -a.button { - background-color: PeachPuff; - text-decoration: underline; - text-align: center; - color: Black; - padding: 4px; - border: solid 1px; -} - -a.small { - padding: 2px 4px 2px 4px; - font-size: small; - border-color: Gray; - background-color: PapayaWhip; -} - -a.button:hover { background-color: LightYellow; } -a.button:active { background-color: Yellow; } - -a.column { - border-style: none; - display: block; - margin: -3px -8px -2px -8px; -} - -div.warning { - background-color: MistyRose; - border: 1px solid Crimson; - padding: 0.5em; - font-size: x-large; -} diff --git a/deprecated/automation/server/monitor/templates/base.html b/deprecated/automation/server/monitor/templates/base.html deleted file mode 100644 index 95ffc222..00000000 --- a/deprecated/automation/server/monitor/templates/base.html +++ /dev/null @@ -1,30 +0,0 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" -"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> - <head> - <link rel="stylesheet" href="/static/style.css" /> - <title>{% block title %}Automation Dashboard{% endblock %}</title> - </head> - - <body> - <div class="header"> - {% block header %} - <p class="title">Automation Dashboard</p> - {% endblock %} - </div> - - <div class="links"> - <span>Subpages:</span> - {% block links %} - {% for link in links %} - <a class="button" href="{{ link.href }}">{{ link.name }}</a> - {% endfor %} - {% endblock %} - </div> - - <div class="content"> - {% block content %} - {% endblock %} - </div> - </body> -</html> diff --git a/deprecated/automation/server/monitor/templates/job.html b/deprecated/automation/server/monitor/templates/job.html deleted file mode 100644 index 90acd969..00000000 --- a/deprecated/automation/server/monitor/templates/job.html +++ /dev/null @@ -1,29 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -<h1 class="title">Job {{ job_id }}</h1> - -<h2 class="title">General information</h2> -{% include "snippet_attribute_table.html" %} - -<h2 class="title">Timeline of status events</h2> -<table class="list"> - <tbody> - <tr> - <th>Started</th> - <th>From State</th> - <th>To State</th> - <th>Elapsed</th> - </tr> - {% for entry in timeline %} - <tr> - <td>{{ entry.started }}</td> - <td>{{ entry.state_from }}</td> - <td>{{ entry.state_to }}</td> - <td>{{ entry.elapsed }}</td> - </tr> - {% endfor %} - </tbody> -</table> - -{% endblock %} diff --git a/deprecated/automation/server/monitor/templates/job_group.html b/deprecated/automation/server/monitor/templates/job_group.html deleted file mode 100644 index b6ed8ea8..00000000 --- a/deprecated/automation/server/monitor/templates/job_group.html +++ /dev/null @@ -1,46 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -<h1 class="title">Job Group {{ group_id }}</h1> - -<h2 class="title">General information</h2> -{% include "snippet_attribute_table.html" %} - -<h2 class="title">Job Listing</h2> -<table class="list"> - <tbody> - <tr> - <th>Job ID</th> - <th>Label</th> - <th>Turnaround Time</th> - <th>State</th> - </tr> - {% for job in job_list %} - <tr> - <td> - <a class="button column" href="/job/{{ job.id }}">{{ job.id }}</a> - </td> - <td>{{ job.label }}</td> - <td>{{ job.elapsed }}</td> - {% if job.status %} - <td class="{{ job.status }}">{{ job.state }}</td> - {% else %} - <td>{{ job.state }}</td> - {% endif %} - </tr> - {% endfor %} - </tbody> -</table> - -<h2 class="title">Report</h2> -{% if reports %} -{% autoescape off %} -{% for report in reports %} -{{ report }} -{% endfor %} -{% endautoescape %} -{% else %} -<div class="warning">No reports found!</div> -{% endif %} - -{% endblock %} diff --git a/deprecated/automation/server/monitor/templates/job_group_list.html b/deprecated/automation/server/monitor/templates/job_group_list.html deleted file mode 100644 index b82fa730..00000000 --- a/deprecated/automation/server/monitor/templates/job_group_list.html +++ /dev/null @@ -1,35 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -<p class="title">Job Groups</p> - -<form action="/job-group" method="post"> -{{ filter.as_p }} -<p><input type="submit" value="Filter!" /></p> -</form> - -<table class="list"> - <tbody> - <tr> - <th>Group ID</th> - <th>Label</th> - <th>Time Submitted</th> - <th>Status</th> - </tr> - {% for group in groups %} - <tr> - <td> - <a class="button column" href="/job-group/{{ group.id }}">{{ group.id }}</a> - </td> - <td>{{ group.label }}</td> - <td>{{ group.submitted }}</td> - {% if group.status %} - <td class="{{ group.status }}">{{ group.state }}</td> - {% else %} - <td>{{ group.state }}</td> - {% endif %} - </tr> - {% endfor %} - </tbody> -</table> -{% endblock %} diff --git a/deprecated/automation/server/monitor/templates/job_log.html b/deprecated/automation/server/monitor/templates/job_log.html deleted file mode 100644 index 937b21b0..00000000 --- a/deprecated/automation/server/monitor/templates/job_log.html +++ /dev/null @@ -1,20 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -<h1 class="title">Job {{ job_id }}</h1> - -<h2 class="title">Command output:</h2> - -<table class="raw"> -<tbody> -{% for datetime, stream, line in log_lines %} -<tr class="{% cycle 'line1' 'line2' %}"> - <td>{{ datetime }}</td> - <td>{{ stream }}</td> - <td><pre class="code">{{ line|wordwrap:80 }}</pre></td> -</tr> -{% endfor %} -</tbody> -</table> - -{% endblock %} diff --git a/deprecated/automation/server/monitor/templates/machine_list.html b/deprecated/automation/server/monitor/templates/machine_list.html deleted file mode 100644 index f81422d3..00000000 --- a/deprecated/automation/server/monitor/templates/machine_list.html +++ /dev/null @@ -1,39 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -<p class="title">Machines</p> - -<table class="list"> -<tbody> -<tr> - <th>Hostname</th> - <th>Label</th> - <th>CPU</th> - <th>Cores</th> - <th>Operating System</th> - <th>Jobs Running</th> - <th>Locked</th> -</tr> -{% for machine in machines %} -<tr> - <td> - <a class="button column" href="/machine/{{ machine.hostname }}"> - {{ machine.hostname }} - </a> - </td> - <td>{{ machine.label }}</td> - <td>{{ machine.cpu }}</td> - <td>{{ machine.cores }}</td> - <td>{{ machine.os }}</td> - <td>{{ machine.uses }}</td> - {% if machine.locked %} - <td class="failure">Yes</td> - {% else %} - <td class="success">No</td> - {% endif %} -</tr> -{% endfor %} -</tbody> -</table> - -{% endblock %} diff --git a/deprecated/automation/server/monitor/templates/snippet_attribute_table.html b/deprecated/automation/server/monitor/templates/snippet_attribute_table.html deleted file mode 100644 index 24bacc17..00000000 --- a/deprecated/automation/server/monitor/templates/snippet_attribute_table.html +++ /dev/null @@ -1,36 +0,0 @@ -<table class="list attributes"> - <tbody> - <tr> - <th>Attribute</th> - <th>Value</th> - </tr> - {% for name, value in attributes.text %} - <tr> - <td>{{ name }}</td> - <td>{{ value }}</td> - </tr> - {% endfor %} - - {% for name, links in attributes.link %} - <tr> - <td>{{ name }}</td> - <td> - {% if links %} - {% for link in links %} - <a class="button small" href="{{ link.href }}">{{ link.name }}</a> - {% endfor %} - {% else %} - None - {% endif %} - </td> - </tr> - {% endfor %} - - {% for name, code in attributes.code %} - <tr> - <td>{{ name }}</td> - <td>{% include "snippet_code.html" %}</td> - </tr> - {% endfor %} - </tbody> -</table> diff --git a/deprecated/automation/server/monitor/templates/snippet_code.html b/deprecated/automation/server/monitor/templates/snippet_code.html deleted file mode 100644 index 281754d6..00000000 --- a/deprecated/automation/server/monitor/templates/snippet_code.html +++ /dev/null @@ -1,10 +0,0 @@ -<table class="raw"> -<tbody> -{% for num, line in code %} -<tr class="{% cycle 'line1' 'line2' %}"> - <td>{{ num }}</td> - <td><pre class="code">{{ line|wordwrap:120 }}</pre></td> -</tr> -{% endfor %} -</tbody> -</table> diff --git a/deprecated/automation/server/monitor/templates/snippet_links.html b/deprecated/automation/server/monitor/templates/snippet_links.html deleted file mode 100644 index f19fa6e5..00000000 --- a/deprecated/automation/server/monitor/templates/snippet_links.html +++ /dev/null @@ -1,7 +0,0 @@ -{% if param %} -{% for link in param %} -<a class="button small" href="{{ link.href }}">{{ link.name }}</a> -{% endfor %} -{% else %} -None -{% endif %} diff --git a/deprecated/automation/server/monitor/urls.py b/deprecated/automation/server/monitor/urls.py deleted file mode 100644 index 1a6b2485..00000000 --- a/deprecated/automation/server/monitor/urls.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2011 Google Inc. All Rights Reserved. -# - -__author__ = 'kbaclawski@google.com (Krystian Baclawski)' - -from django.conf import settings -from django.conf.urls.defaults import patterns - -urlpatterns = patterns( - 'dashboard', (r'^job-group$', 'JobGroupListPageHandler'), - (r'^machine$', 'MachineListPageHandler'), - (r'^job/(?P<job_id>\d+)/log$', 'LogPageHandler'), - (r'^job/(?P<job_id>\d+)$', 'JobPageHandler'), ( - r'^job-group/(?P<job_group_id>\d+)/files/(?P<path>.*)$', - 'JobGroupFilesPageHandler'), - (r'^job-group/(?P<job_group_id>\d+)$', 'JobGroupPageHandler'), - (r'^$', 'DefaultPageHandler')) - -urlpatterns += patterns('', - (r'^static/(?P<path>.*)$', 'django.views.static.serve', - {'document_root': settings.MEDIA_ROOT})) diff --git a/deprecated/automation/server/server.py b/deprecated/automation/server/server.py deleted file mode 100755 index c8f22521..00000000 --- a/deprecated/automation/server/server.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/python2 -# -# Copyright 2010 Google Inc. All Rights Reserved. - -import logging -import optparse -import pickle -import signal -from SimpleXMLRPCServer import SimpleXMLRPCServer -import sys - -from automation.common import logger -from automation.common.command_executer import CommandExecuter -from automation.server import machine_manager -from automation.server.job_group_manager import JobGroupManager -from automation.server.job_manager import JobManager - - -class Server(object): - """Plays a role of external interface accessible over XMLRPC.""" - - def __init__(self, machines_file=None, dry_run=False): - """Default constructor. - - Args: - machines_file: Path to file storing a list of machines. - dry_run: If True, the server only simulates command execution. - """ - CommandExecuter.Configure(dry_run) - - self.job_manager = JobManager( - machine_manager.MachineManager.FromMachineListFile( - machines_file or machine_manager.DEFAULT_MACHINES_FILE)) - - self.job_group_manager = JobGroupManager(self.job_manager) - - self._logger = logging.getLogger(self.__class__.__name__) - - def ExecuteJobGroup(self, job_group, dry_run=False): - job_group = pickle.loads(job_group) - self._logger.info('Received ExecuteJobGroup(%r, dry_run=%s) request.', - job_group, dry_run) - - for job in job_group.jobs: - job.dry_run = dry_run - return self.job_group_manager.AddJobGroup(job_group) - - def GetAllJobGroups(self): - self._logger.info('Received GetAllJobGroups() request.') - return pickle.dumps(self.job_group_manager.GetAllJobGroups()) - - def KillJobGroup(self, job_group_id): - self._logger.info('Received KillJobGroup(%d) request.', job_group_id) - self.job_group_manager.KillJobGroup(pickle.loads(job_group_id)) - - def GetJobGroup(self, job_group_id): - self._logger.info('Received GetJobGroup(%d) request.', job_group_id) - - return pickle.dumps(self.job_group_manager.GetJobGroup(job_group_id)) - - def GetJob(self, job_id): - self._logger.info('Received GetJob(%d) request.', job_id) - - return pickle.dumps(self.job_manager.GetJob(job_id)) - - def GetMachineList(self): - self._logger.info('Received GetMachineList() request.') - - return pickle.dumps(self.job_manager.machine_manager.GetMachineList()) - - def StartServer(self): - self.job_manager.StartJobManager() - - def StopServer(self): - self.job_manager.StopJobManager() - self.job_manager.join() - - -def GetServerOptions(): - """Get server's settings from command line options.""" - parser = optparse.OptionParser() - parser.add_option('-m', - '--machines-file', - dest='machines_file', - help='The location of the file ' - 'containing the machines database', - default=machine_manager.DEFAULT_MACHINES_FILE) - parser.add_option('-n', - '--dry-run', - dest='dry_run', - help='Start the server in dry-run mode, where jobs will ' - 'not actually be executed.', - action='store_true', - default=False) - return parser.parse_args()[0] - - -def Main(): - logger.SetUpRootLogger(filename='server.log', level=logging.DEBUG) - - options = GetServerOptions() - server = Server(options.machines_file, options.dry_run) - server.StartServer() - - def _HandleKeyboardInterrupt(*_): - server.StopServer() - sys.exit(1) - - signal.signal(signal.SIGINT, _HandleKeyboardInterrupt) - - try: - xmlserver = SimpleXMLRPCServer( - ('localhost', 8000), - allow_none=True, - logRequests=False) - xmlserver.register_instance(server) - xmlserver.serve_forever() - except Exception as ex: - logging.error(ex) - server.StopServer() - sys.exit(1) - - -if __name__ == '__main__': - Main() diff --git a/deprecated/automation/server/server_test.py b/deprecated/automation/server/server_test.py deleted file mode 100755 index 131ebb3b..00000000 --- a/deprecated/automation/server/server_test.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/python2 -# -# Copyright 2010 Google Inc. All Rights Reserved. -"""Machine manager unittest. - -MachineManagerTest tests MachineManager. -""" - -__author__ = 'asharif@google.com (Ahmad Sharif)' - -import server -import unittest - - -class ServerTest(unittest.TestCase): - - def setUp(self): - pass - - def testGetAllJobs(self): - s = server.Server() - print s.GetAllJobs() - - -if __name__ == '__main__': - unittest.main() diff --git a/deprecated/automation/server/test_pool.csv b/deprecated/automation/server/test_pool.csv deleted file mode 100644 index b0700c9b..00000000 --- a/deprecated/automation/server/test_pool.csv +++ /dev/null @@ -1,4 +0,0 @@ -hostname,label,cpu,cores,os,username -"chrotomation.mtv.corp.google.com","pc-workstation","core2duo",8,"linux","mobiletc-prebuild" -"chromeos-test1.mtv.corp.google.com","cr48","atom",1,"chromeos","chromeos" -"chromeos-test2.mtv.corp.google.com","cr48","atom",1,"chromeos","chromeos" |