diff options
Diffstat (limited to 'automation/server')
25 files changed, 1425 insertions, 0 deletions
diff --git a/automation/server/__init__.py b/automation/server/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/automation/server/__init__.py @@ -0,0 +1 @@ + diff --git a/automation/server/job_executer.py b/automation/server/job_executer.py new file mode 100644 index 00000000..30b59463 --- /dev/null +++ b/automation/server/job_executer.py @@ -0,0 +1,138 @@ +# Copyright 2010 Google Inc. All Rights Reserved. +# + +import logging +import os.path +import threading + +from automation.common import command as cmd +from automation.common import job +from automation.common import logger +from automation.common.command_executer import LoggingCommandExecuter +from automation.common.command_executer import CommandTerminator + + +class JobExecuter(threading.Thread): + + def __init__(self, job_to_execute, machines, listeners): + threading.Thread.__init__(self) + + assert machines + + self.job = job_to_execute + self.listeners = listeners + self.machines = machines + + # Set Thread name. + self.name = '%s-%s' % (self.__class__.__name__, self.job.id) + + self._logger = logging.getLogger(self.__class__.__name__) + self._executer = LoggingCommandExecuter(self.job.dry_run) + self._terminator = CommandTerminator() + + def _RunRemotely(self, command, fail_msg, command_timeout=1 * 60 * 60): + exit_code = self._executer.RunCommand(command, + self.job.primary_machine.hostname, + self.job.primary_machine.username, + command_terminator=self._terminator, + command_timeout=command_timeout) + if exit_code: + raise job.JobFailure(fail_msg, exit_code) + + def _RunLocally(self, command, fail_msg, command_timeout=1 * 60 * 60): + exit_code = self._executer.RunCommand(command, + command_terminator=self._terminator, + command_timeout=command_timeout) + if exit_code: + raise job.JobFailure(fail_msg, exit_code) + + def Kill(self): + self._terminator.Terminate() + + def CleanUpWorkDir(self): + self._logger.debug('Cleaning up %r work directory.', self.job) + self._RunRemotely(cmd.RmTree(self.job.work_dir), 'Cleanup workdir failed.') + + def CleanUpHomeDir(self): + self._logger.debug('Cleaning up %r home directory.', self.job) + self._RunLocally(cmd.RmTree(self.job.home_dir), 'Cleanup homedir failed.') + + def _PrepareRuntimeEnvironment(self): + self._RunRemotely( + cmd.MakeDir(self.job.work_dir, self.job.logs_dir, self.job.results_dir), + 'Creating new job directory failed.') + + # The log directory is ready, so we can prepare to log command's output. + self._executer.OpenLog(os.path.join(self.job.logs_dir, + self.job.log_filename_prefix)) + + def _SatisfyFolderDependencies(self): + for dependency in self.job.folder_dependencies: + to_folder = os.path.join(self.job.work_dir, dependency.dest) + from_folder = os.path.join(dependency.job.work_dir, dependency.src) + from_machine = dependency.job.primary_machine + + if from_machine == self.job.primary_machine and dependency.read_only: + # No need to make a copy, just symlink it + self._RunRemotely( + cmd.MakeSymlink(from_folder, to_folder), + 'Failed to create symlink to required directory.') + else: + self._RunRemotely( + cmd.RemoteCopyFrom(from_machine.hostname, + from_folder, + to_folder, + username=from_machine.username), + 'Failed to copy required files.') + + def _LaunchJobCommand(self): + command = self.job.GetCommand() + + self._RunRemotely('%s; %s' % ('PS1=. TERM=linux source ~/.bashrc', + cmd.Wrapper(command, + cwd=self.job.work_dir)), + "Command failed to execute: '%s'." % command, + self.job.timeout) + + def _CopyJobResults(self): + """Copy test results back to directory.""" + self._RunLocally( + cmd.RemoteCopyFrom(self.job.primary_machine.hostname, + self.job.results_dir, + self.job.home_dir, + username=self.job.primary_machine.username), + 'Failed to copy results.') + + def run(self): + self.job.status = job.STATUS_SETUP + self.job.machines = self.machines + self._logger.debug('Executing %r on %r in directory %s.', self.job, + self.job.primary_machine.hostname, self.job.work_dir) + + try: + self.CleanUpWorkDir() + + self._PrepareRuntimeEnvironment() + + self.job.status = job.STATUS_COPYING + + self._SatisfyFolderDependencies() + + self.job.status = job.STATUS_RUNNING + + self._LaunchJobCommand() + self._CopyJobResults() + + # If we get here, the job succeeded. + self.job.status = job.STATUS_SUCCEEDED + except job.JobFailure as ex: + self._logger.error('Job failed. Exit code %s. %s', ex.exit_code, ex) + if self._terminator.IsTerminated(): + self._logger.info('%r was killed', self.job) + + self.job.status = job.STATUS_FAILED + + self._executer.CloseLog() + + for listener in self.listeners: + listener.NotifyJobComplete(self.job) diff --git a/automation/server/job_group_manager.py b/automation/server/job_group_manager.py new file mode 100644 index 00000000..d66f5e07 --- /dev/null +++ b/automation/server/job_group_manager.py @@ -0,0 +1,118 @@ +# Copyright 2010 Google Inc. All Rights Reserved. +# + +import copy +import logging +import threading + +from automation.common import command as cmd +from automation.common import logger +from automation.common.command_executer import CommandExecuter +from automation.common import job +from automation.common import job_group +from automation.server.job_manager import IdProducerPolicy + + +class JobGroupManager(object): + + def __init__(self, job_manager): + self.all_job_groups = [] + + self.job_manager = job_manager + self.job_manager.AddListener(self) + + self._lock = threading.Lock() + self._job_group_finished = threading.Condition(self._lock) + + self._id_producer = IdProducerPolicy() + self._id_producer.Initialize(job_group.JobGroup.HOMEDIR_PREFIX, + 'job-group-(?P<id>\d+)') + + self._logger = logging.getLogger(self.__class__.__name__) + + def GetJobGroup(self, group_id): + with self._lock: + for group in self.all_job_groups: + if group.id == group_id: + return group + + return None + + def GetAllJobGroups(self): + with self._lock: + return copy.deepcopy(self.all_job_groups) + + def AddJobGroup(self, group): + with self._lock: + group.id = self._id_producer.GetNextId() + + self._logger.debug('Creating runtime environment for %r.', group) + + CommandExecuter().RunCommand(cmd.Chain( + cmd.RmTree(group.home_dir), cmd.MakeDir(group.home_dir))) + + with self._lock: + self.all_job_groups.append(group) + + for job_ in group.jobs: + self.job_manager.AddJob(job_) + + group.status = job_group.STATUS_EXECUTING + + self._logger.info('Added %r to queue.', group) + + return group.id + + def KillJobGroup(self, group): + with self._lock: + self._logger.debug('Killing all jobs that belong to %r.', group) + + for job_ in group.jobs: + self.job_manager.KillJob(job_) + + self._logger.debug('Waiting for jobs to quit.') + + # Lets block until the group is killed so we know it is completed + # when we return. + while group.status not in [job_group.STATUS_SUCCEEDED, + job_group.STATUS_FAILED]: + self._job_group_finished.wait() + + def NotifyJobComplete(self, job_): + self._logger.debug('Handling %r completion event.', job_) + + group = job_.group + + with self._lock: + # We need to perform an action only if the group hasn't already failed. + if group.status != job_group.STATUS_FAILED: + if job_.status == job.STATUS_FAILED: + # We have a failed job, abort the job group + group.status = job_group.STATUS_FAILED + if group.cleanup_on_failure: + for job_ in group.jobs: + # TODO(bjanakiraman): We should probably only kill dependent jobs + # instead of the whole job group. + self.job_manager.KillJob(job_) + self.job_manager.CleanUpJob(job_) + else: + # The job succeeded successfully -- lets check to see if we are done. + assert job_.status == job.STATUS_SUCCEEDED + finished = True + for other_job in group.jobs: + assert other_job.status != job.STATUS_FAILED + if other_job.status != job.STATUS_SUCCEEDED: + finished = False + break + + if finished and group.status != job_group.STATUS_SUCCEEDED: + # TODO(kbaclawski): Without check performed above following code + # could be called more than once. This would trigger StateMachine + # crash, because it cannot transition from STATUS_SUCCEEDED to + # STATUS_SUCCEEDED. Need to address that bug in near future. + group.status = job_group.STATUS_SUCCEEDED + if group.cleanup_on_completion: + for job_ in group.jobs: + self.job_manager.CleanUpJob(job_) + + self._job_group_finished.notifyAll() diff --git a/automation/server/job_manager.py b/automation/server/job_manager.py new file mode 100644 index 00000000..7a65b918 --- /dev/null +++ b/automation/server/job_manager.py @@ -0,0 +1,194 @@ +# Copyright 2010 Google Inc. All Rights Reserved. +# + +import logging +import os +import re +import threading + +from automation.common import job +from automation.common import logger +from automation.server.job_executer import JobExecuter + + +class IdProducerPolicy(object): + """Produces series of unique integer IDs. + + Example: + id_producer = IdProducerPolicy() + id_a = id_producer.GetNextId() + id_b = id_producer.GetNextId() + assert id_a != id_b + """ + + def __init__(self): + self._counter = 1 + + def Initialize(self, home_prefix, home_pattern): + """Find first available ID based on a directory listing. + + Args: + home_prefix: A directory to be traversed. + home_pattern: A regexp describing all files/directories that will be + considered. The regexp must contain exactly one match group with name + "id", which must match an integer number. + + Example: + id_producer.Initialize(JOBDIR_PREFIX, 'job-(?P<id>\d+)') + """ + harvested_ids = [] + + if os.path.isdir(home_prefix): + for filename in os.listdir(home_prefix): + path = os.path.join(home_prefix, filename) + + if os.path.isdir(path): + match = re.match(home_pattern, filename) + + if match: + harvested_ids.append(int(match.group('id'))) + + self._counter = max(harvested_ids or [0]) + 1 + + def GetNextId(self): + """Calculates another ID considered to be unique.""" + new_id = self._counter + self._counter += 1 + return new_id + + +class JobManager(threading.Thread): + + def __init__(self, machine_manager): + threading.Thread.__init__(self, name=self.__class__.__name__) + self.all_jobs = [] + self.ready_jobs = [] + self.job_executer_mapping = {} + + self.machine_manager = machine_manager + + self._lock = threading.Lock() + self._jobs_available = threading.Condition(self._lock) + self._exit_request = False + + self.listeners = [] + self.listeners.append(self) + + self._id_producer = IdProducerPolicy() + self._id_producer.Initialize(job.Job.WORKDIR_PREFIX, 'job-(?P<id>\d+)') + + self._logger = logging.getLogger(self.__class__.__name__) + + def StartJobManager(self): + self._logger.info('Starting...') + + with self._lock: + self.start() + self._jobs_available.notifyAll() + + def StopJobManager(self): + self._logger.info('Shutdown request received.') + + with self._lock: + for job_ in self.all_jobs: + self._KillJob(job_.id) + + # Signal to die + self._exit_request = True + self._jobs_available.notifyAll() + + # Wait for all job threads to finish + for executer in self.job_executer_mapping.values(): + executer.join() + + def KillJob(self, job_id): + """Kill a job by id. + + Does not block until the job is completed. + """ + with self._lock: + self._KillJob(job_id) + + def GetJob(self, job_id): + for job_ in self.all_jobs: + if job_.id == job_id: + return job_ + return None + + def _KillJob(self, job_id): + self._logger.info('Killing [Job: %d].', job_id) + + if job_id in self.job_executer_mapping: + self.job_executer_mapping[job_id].Kill() + for job_ in self.ready_jobs: + if job_.id == job_id: + self.ready_jobs.remove(job_) + break + + def AddJob(self, job_): + with self._lock: + job_.id = self._id_producer.GetNextId() + + self.all_jobs.append(job_) + # Only queue a job as ready if it has no dependencies + if job_.is_ready: + self.ready_jobs.append(job_) + + self._jobs_available.notifyAll() + + return job_.id + + def CleanUpJob(self, job_): + with self._lock: + if job_.id in self.job_executer_mapping: + self.job_executer_mapping[job_.id].CleanUpWorkDir() + del self.job_executer_mapping[job_.id] + # TODO(raymes): remove job from self.all_jobs + + def NotifyJobComplete(self, job_): + self.machine_manager.ReturnMachines(job_.machines) + + with self._lock: + self._logger.debug('Handling %r completion event.', job_) + + if job_.status == job.STATUS_SUCCEEDED: + for succ in job_.successors: + if succ.is_ready: + if succ not in self.ready_jobs: + self.ready_jobs.append(succ) + + self._jobs_available.notifyAll() + + def AddListener(self, listener): + self.listeners.append(listener) + + @logger.HandleUncaughtExceptions + def run(self): + self._logger.info('Started.') + + while not self._exit_request: + with self._lock: + # Get the next ready job, block if there are none + self._jobs_available.wait() + + while self.ready_jobs: + ready_job = self.ready_jobs.pop() + + required_machines = ready_job.machine_dependencies + for pred in ready_job.predecessors: + required_machines[0].AddPreferredMachine( + pred.primary_machine.hostname) + + machines = self.machine_manager.GetMachines(required_machines) + if not machines: + # If we can't get the necessary machines right now, simply wait + # for some jobs to complete + self.ready_jobs.insert(0, ready_job) + break + else: + # Mark as executing + executer = JobExecuter(ready_job, machines, self.listeners) + executer.start() + self.job_executer_mapping[ready_job.id] = executer + + self._logger.info('Stopped.') diff --git a/automation/server/machine_manager.py b/automation/server/machine_manager.py new file mode 100644 index 00000000..b7186077 --- /dev/null +++ b/automation/server/machine_manager.py @@ -0,0 +1,77 @@ +# Copyright 2010 Google Inc. All Rights Reserved. + +__author__ = 'asharif@google.com (Ahmad Sharif)' + +from operator import attrgetter +import copy +import csv +import threading +import os.path + +from automation.common import machine + +DEFAULT_MACHINES_FILE = os.path.join(os.path.dirname(__file__), 'test_pool.csv') + + +class MachineManager(object): + """Container for list of machines one can run jobs on.""" + + @classmethod + def FromMachineListFile(cls, filename): + # Read the file and skip header + csv_file = csv.reader(open(filename, 'rb'), delimiter=',', quotechar='"') + csv_file.next() + + return cls([machine.Machine(hostname, label, cpu, int(cores), os, user) + for hostname, label, cpu, cores, os, user in csv_file]) + + def __init__(self, machines): + self._machine_pool = machines + self._lock = threading.RLock() + + def _GetMachine(self, mach_spec): + available_pool = [m for m in self._machine_pool if mach_spec.IsMatch(m)] + + if available_pool: + # find a machine with minimum uses + uses = attrgetter('uses') + + mach = min(available_pool, key=uses) + + if mach_spec.preferred_machines: + preferred_pool = [m + for m in available_pool + if m.hostname in mach_spec.preferred_machines] + if preferred_pool: + mach = min(preferred_pool, key=uses) + + mach.Acquire(mach_spec.lock_required) + + return mach + + def GetMachines(self, required_machines): + """Acquire machines for use by a job.""" + + with self._lock: + acquired_machines = [self._GetMachine(ms) for ms in required_machines] + + if not all(acquired_machines): + # Roll back acquires + while acquired_machines: + mach = acquired_machines.pop() + if mach: + mach.Release() + + return acquired_machines + + def GetMachineList(self): + with self._lock: + return copy.deepcopy(self._machine_pool) + + def ReturnMachines(self, machines): + with self._lock: + for m in machines: + m.Release() + + def __str__(self): + return str(self._machine_pool) diff --git a/automation/server/machine_manager_test.py b/automation/server/machine_manager_test.py new file mode 100755 index 00000000..67fdcc2b --- /dev/null +++ b/automation/server/machine_manager_test.py @@ -0,0 +1,32 @@ +#!/usr/bin/python +# +# Copyright 2010 Google Inc. All Rights Reserved. + +__author__ = 'asharif@google.com (Ahmad Sharif)' + +import unittest +from automation.common import machine +from automation.server import machine_manager + + +class MachineManagerTest(unittest.TestCase): + + def setUp(self): + self.machine_manager = machine_manager.MachineManager() + + def testPrint(self): + print self.machine_manager + + def testGetLinuxBox(self): + mach_spec_list = [machine.MachineSpecification(os='linux')] + machines = self.machine_manager.GetMachines(mach_spec_list) + self.assertTrue(machines) + + def testGetChromeOSBox(self): + mach_spec_list = [machine.MachineSpecification(os='chromeos')] + machines = self.machine_manager.GetMachines(mach_spec_list) + self.assertTrue(machines) + + +if __name__ == '__main__': + unittest.main() diff --git a/automation/server/monitor/__init__.py b/automation/server/monitor/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/automation/server/monitor/__init__.py @@ -0,0 +1 @@ + diff --git a/automation/server/monitor/dashboard.py b/automation/server/monitor/dashboard.py new file mode 100644 index 00000000..f6befed8 --- /dev/null +++ b/automation/server/monitor/dashboard.py @@ -0,0 +1,259 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +# + +__author__ = 'kbaclawski@google.com (Krystian Baclawski)' + +from collections import namedtuple +import glob +import gzip +import os.path +import pickle +import time +import xmlrpclib + +from django import forms +from django.http import HttpResponseRedirect +from django.shortcuts import render_to_response +from django.template import Context +from django.views import static + +Link = namedtuple('Link', 'href name') + + +def GetServerConnection(): + return xmlrpclib.Server('http://localhost:8000') + + +def MakeDefaultContext(*args): + context = Context({'links': [ + Link('/job-group', 'Job Groups'), Link('/machine', 'Machines') + ]}) + + for arg in args: + context.update(arg) + + return context + + +class JobInfo(object): + + def __init__(self, job_id): + self._job = pickle.loads(GetServerConnection().GetJob(job_id)) + + def GetAttributes(self): + job = self._job + + group = [Link('/job-group/%d' % job.group.id, job.group.label)] + + predecessors = [Link('/job/%d' % pred.id, pred.label) + for pred in job.predecessors] + + successors = [Link('/job/%d' % succ.id, succ.label) + for succ in job.successors] + + machines = [Link('/machine/%s' % mach.hostname, mach.hostname) + for mach in job.machines] + + logs = [Link('/job/%d/log' % job.id, 'Log')] + + commands = enumerate(job.PrettyFormatCommand().split('\n'), start=1) + + return {'text': [('Label', job.label), ('Directory', job.work_dir)], + 'link': [('Group', group), ('Predecessors', predecessors), + ('Successors', successors), ('Machines', machines), + ('Logs', logs)], + 'code': [('Command', commands)]} + + def GetTimeline(self): + return [{'started': evlog.GetTimeStartedFormatted(), + 'state_from': evlog.event.from_, + 'state_to': evlog.event.to_, + 'elapsed': evlog.GetTimeElapsedRounded()} + for evlog in self._job.timeline.GetTransitionEventHistory()] + + def GetLog(self): + log_path = os.path.join(self._job.logs_dir, + '%s.gz' % self._job.log_filename_prefix) + + try: + log = gzip.open(log_path, 'r') + except IOError: + content = [] + else: + # There's a good chance that file is not closed yet, so EOF handling + # function and CRC calculation will fail, thus we need to monkey patch the + # _read_eof method. + log._read_eof = lambda: None + + def SplitLine(line): + prefix, msg = line.split(': ', 1) + datetime, stream = prefix.rsplit(' ', 1) + + return datetime, stream, msg + + content = map(SplitLine, log.readlines()) + finally: + log.close() + + return content + + +class JobGroupInfo(object): + + def __init__(self, job_group_id): + self._job_group = pickle.loads(GetServerConnection().GetJobGroup( + job_group_id)) + + def GetAttributes(self): + group = self._job_group + + home_dir = [Link('/job-group/%d/files/' % group.id, group.home_dir)] + + return {'text': [('Label', group.label), + ('Time submitted', time.ctime(group.time_submitted)), + ('State', group.status), + ('Cleanup on completion', group.cleanup_on_completion), + ('Cleanup on failure', group.cleanup_on_failure)], + 'link': [('Directory', home_dir)]} + + def _GetJobStatus(self, job): + status_map = {'SUCCEEDED': 'success', 'FAILED': 'failure'} + return status_map.get(str(job.status), None) + + def GetJobList(self): + return [{'id': job.id, + 'label': job.label, + 'state': job.status, + 'status': self._GetJobStatus(job), + 'elapsed': job.timeline.GetTotalTime()} + for job in self._job_group.jobs] + + def GetHomeDirectory(self): + return self._job_group.home_dir + + def GetReportList(self): + job_dir_pattern = os.path.join(self._job_group.home_dir, 'job-*') + + filenames = [] + + for job_dir in glob.glob(job_dir_pattern): + filename = os.path.join(job_dir, 'report.html') + + if os.access(filename, os.F_OK): + filenames.append(filename) + + reports = [] + + for filename in sorted(filenames, key=lambda f: os.stat(f).st_ctime): + try: + with open(filename, 'r') as report: + reports.append(report.read()) + except IOError: + pass + + return reports + + +class JobGroupListInfo(object): + + def __init__(self): + self._all_job_groups = pickle.loads(GetServerConnection().GetAllJobGroups()) + + def _GetJobGroupState(self, group): + return str(group.status) + + def _GetJobGroupStatus(self, group): + status_map = {'SUCCEEDED': 'success', 'FAILED': 'failure'} + return status_map.get(self._GetJobGroupState(group), None) + + def GetList(self): + return [{'id': group.id, + 'label': group.label, + 'submitted': time.ctime(group.time_submitted), + 'state': self._GetJobGroupState(group), + 'status': self._GetJobGroupStatus(group)} + for group in self._all_job_groups] + + def GetLabelList(self): + return sorted(set(group.label for group in self._all_job_groups)) + + +def JobPageHandler(request, job_id): + job = JobInfo(int(job_id)) + + ctx = MakeDefaultContext({ + 'job_id': job_id, + 'attributes': job.GetAttributes(), + 'timeline': job.GetTimeline() + }) + + return render_to_response('job.html', ctx) + + +def LogPageHandler(request, job_id): + job = JobInfo(int(job_id)) + + ctx = MakeDefaultContext({'job_id': job_id, 'log_lines': job.GetLog()}) + + return render_to_response('job_log.html', ctx) + + +def JobGroupPageHandler(request, job_group_id): + group = JobGroupInfo(int(job_group_id)) + + ctx = MakeDefaultContext({ + 'group_id': job_group_id, + 'attributes': group.GetAttributes(), + 'job_list': group.GetJobList(), + 'reports': group.GetReportList() + }) + + return render_to_response('job_group.html', ctx) + + +def JobGroupFilesPageHandler(request, job_group_id, path): + group = JobGroupInfo(int(job_group_id)) + + return static.serve(request, + path, + document_root=group.GetHomeDirectory(), + show_indexes=True) + + +class FilterJobGroupsForm(forms.Form): + label = forms.ChoiceField(label='Filter by label:', required=False) + + +def JobGroupListPageHandler(request): + groups = JobGroupListInfo() + group_list = groups.GetList() + + field = FilterJobGroupsForm.base_fields['label'] + field.choices = [('*', '--- no filtering ---')] + field.choices.extend([(label, label) for label in groups.GetLabelList()]) + + if request.method == 'POST': + form = FilterJobGroupsForm(request.POST) + + if form.is_valid(): + label = form.cleaned_data['label'] + + if label != '*': + group_list = [group for group in group_list if group['label'] == label] + else: + form = FilterJobGroupsForm({'initial': '*'}) + + ctx = MakeDefaultContext({'filter': form, 'groups': group_list}) + + return render_to_response('job_group_list.html', ctx) + + +def MachineListPageHandler(request): + machine_list = pickle.loads(GetServerConnection().GetMachineList()) + + return render_to_response('machine_list.html', + MakeDefaultContext({'machines': machine_list})) + + +def DefaultPageHandler(request): + return HttpResponseRedirect('/job-group') diff --git a/automation/server/monitor/manage.py b/automation/server/monitor/manage.py new file mode 100755 index 00000000..57deb5c2 --- /dev/null +++ b/automation/server/monitor/manage.py @@ -0,0 +1,20 @@ +#!/usr/bin/python +# +# Copyright 2011 Google Inc. All Rights Reserved. +# + +__author__ = 'kbaclawski@google.com (Krystian Baclawski)' + +from django.core.management import execute_manager + +try: + import settings # Assumed to be in the same directory. +except ImportError: + import sys + + sys.stderr.write('Error: Can\'t find settings.py file in the directory ' + 'containing %r.' % __file__) + sys.exit(1) + +if __name__ == '__main__': + execute_manager(settings) diff --git a/automation/server/monitor/settings.py b/automation/server/monitor/settings.py new file mode 100644 index 00000000..8cd20e35 --- /dev/null +++ b/automation/server/monitor/settings.py @@ -0,0 +1,49 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +# +# Django settings for monitor project. +# +# For explanation look here: http://docs.djangoproject.com/en/dev/ref/settings +# + +__author__ = 'kbaclawski@google.com (Krystian Baclawski)' + +import os.path +import sys + +# Path to the root of application. It's a custom setting, not related to Django. +ROOT_PATH = os.path.dirname(os.path.realpath(sys.argv[0])) + +# Print useful information during runtime if possible. +DEBUG = True +TEMPLATE_DEBUG = DEBUG + +# Sqlite3 database configuration, though we don't use it right now. +DATABASE_ENGINE = 'sqlite3' +DATABASE_NAME = os.path.join(ROOT_PATH, 'monitor.db') + +# Local time zone for this installation. +TIME_ZONE = 'America/Los_Angeles' + +# Language code for this installation. +LANGUAGE_CODE = 'en-us' + +# If you set this to False, Django will make some optimizations so as not +# to load the internationalization machinery. +USE_I18N = True + +# Absolute path to the directory that holds media. +MEDIA_ROOT = os.path.join(ROOT_PATH, 'static') + '/' + +# URL that handles the media served from MEDIA_ROOT. Make sure to use a +# trailing slash if there is a path component (optional in other cases). +MEDIA_URL = '/static/' + +# Used to provide a seed in secret-key hashing algorithms. Make this unique, +# and don't share it with anybody. +SECRET_KEY = '13p5p_4q91*8@yo+tvvt#2k&6#d_&e_zvxdpdil53k419i5sop' + +# A string representing the full Python import path to your root URLconf. +ROOT_URLCONF = 'monitor.urls' + +# List of locations of the template source files, in search order. +TEMPLATE_DIRS = (os.path.join(ROOT_PATH, 'templates'),) diff --git a/automation/server/monitor/start.sh b/automation/server/monitor/start.sh new file mode 100755 index 00000000..4fc53bef --- /dev/null +++ b/automation/server/monitor/start.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# +# Copyright 2011 Google Inc. All Rights Reserved. +# Author: kbaclawski@google.com (Krystian Baclawski) +# + +./manage.py runserver "$HOSTNAME":8080 diff --git a/automation/server/monitor/static/style.css b/automation/server/monitor/static/style.css new file mode 100644 index 00000000..b571b059 --- /dev/null +++ b/automation/server/monitor/static/style.css @@ -0,0 +1,101 @@ +* { font-family: sans-serif; } + +.left { text-align: left; } +.right { text-align: right; } + +.code { font-family: monospace; text-align: left; } +.line1 { background-color: Gainsboro; } +.line2 { background-color: WhiteSmoke; } + +.title { margin-bottom: 0.25em; } + +.success { background-color: LightGreen; } +.failure { background-color: LightPink; } + +pre.code { margin: 0px; } + +div.header p.title { + border: 1px solid black; + font-size: 32px; + font-style: bold; + background-color: LightBlue; + text-align: center; + margin: 0px; + padding: 10px; + font-weight: bold; +} + +div.links { + background-color: Azure; + margin-top: 2px; + padding: 8px 4px 8px 4px; + border: solid 1px; +} + +div.content { + margin-top: 2px; + padding: 8px; + border: solid 1px; +} + +div.content p.title { + font-size: 28px; + text-align: left; + margin: 0px; + margin-bottom: 8px; + padding: 12px; + font-weight: bold; +} + +table { border-collapse: collapse; } +td, th { text-align: center; } + +table.list td, th { padding: 3px 8px 2px 8px; border:1px solid black; } +table.list td { font-family: monospace; } +table.list th { background-color: LightGray; } + +table.attributes td { text-align: left; } +table.attributes > tbody > tr > td:first-child { font-family: sans-serif; } + +table.raw { border-style: none; } +table.raw td { + padding: 0em 0.5em 0em 0.5em; + border-style: none; + vertical-align: top; + text-align: right; + font-family: monospace; +} +table.raw > tbody > tr > td:first-child { border-left: 0px; } +table.raw > tbody > tr > td { border-left: 1px solid; } + +a.button { + background-color: PeachPuff; + text-decoration: underline; + text-align: center; + color: Black; + padding: 4px; + border: solid 1px; +} + +a.small { + padding: 2px 4px 2px 4px; + font-size: small; + border-color: Gray; + background-color: PapayaWhip; +} + +a.button:hover { background-color: LightYellow; } +a.button:active { background-color: Yellow; } + +a.column { + border-style: none; + display: block; + margin: -3px -8px -2px -8px; +} + +div.warning { + background-color: MistyRose; + border: 1px solid Crimson; + padding: 0.5em; + font-size: x-large; +} diff --git a/automation/server/monitor/templates/base.html b/automation/server/monitor/templates/base.html new file mode 100644 index 00000000..95ffc222 --- /dev/null +++ b/automation/server/monitor/templates/base.html @@ -0,0 +1,30 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" +"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> + <head> + <link rel="stylesheet" href="/static/style.css" /> + <title>{% block title %}Automation Dashboard{% endblock %}</title> + </head> + + <body> + <div class="header"> + {% block header %} + <p class="title">Automation Dashboard</p> + {% endblock %} + </div> + + <div class="links"> + <span>Subpages:</span> + {% block links %} + {% for link in links %} + <a class="button" href="{{ link.href }}">{{ link.name }}</a> + {% endfor %} + {% endblock %} + </div> + + <div class="content"> + {% block content %} + {% endblock %} + </div> + </body> +</html> diff --git a/automation/server/monitor/templates/job.html b/automation/server/monitor/templates/job.html new file mode 100644 index 00000000..90acd969 --- /dev/null +++ b/automation/server/monitor/templates/job.html @@ -0,0 +1,29 @@ +{% extends "base.html" %} + +{% block content %} +<h1 class="title">Job {{ job_id }}</h1> + +<h2 class="title">General information</h2> +{% include "snippet_attribute_table.html" %} + +<h2 class="title">Timeline of status events</h2> +<table class="list"> + <tbody> + <tr> + <th>Started</th> + <th>From State</th> + <th>To State</th> + <th>Elapsed</th> + </tr> + {% for entry in timeline %} + <tr> + <td>{{ entry.started }}</td> + <td>{{ entry.state_from }}</td> + <td>{{ entry.state_to }}</td> + <td>{{ entry.elapsed }}</td> + </tr> + {% endfor %} + </tbody> +</table> + +{% endblock %} diff --git a/automation/server/monitor/templates/job_group.html b/automation/server/monitor/templates/job_group.html new file mode 100644 index 00000000..b6ed8ea8 --- /dev/null +++ b/automation/server/monitor/templates/job_group.html @@ -0,0 +1,46 @@ +{% extends "base.html" %} + +{% block content %} +<h1 class="title">Job Group {{ group_id }}</h1> + +<h2 class="title">General information</h2> +{% include "snippet_attribute_table.html" %} + +<h2 class="title">Job Listing</h2> +<table class="list"> + <tbody> + <tr> + <th>Job ID</th> + <th>Label</th> + <th>Turnaround Time</th> + <th>State</th> + </tr> + {% for job in job_list %} + <tr> + <td> + <a class="button column" href="/job/{{ job.id }}">{{ job.id }}</a> + </td> + <td>{{ job.label }}</td> + <td>{{ job.elapsed }}</td> + {% if job.status %} + <td class="{{ job.status }}">{{ job.state }}</td> + {% else %} + <td>{{ job.state }}</td> + {% endif %} + </tr> + {% endfor %} + </tbody> +</table> + +<h2 class="title">Report</h2> +{% if reports %} +{% autoescape off %} +{% for report in reports %} +{{ report }} +{% endfor %} +{% endautoescape %} +{% else %} +<div class="warning">No reports found!</div> +{% endif %} + +{% endblock %} diff --git a/automation/server/monitor/templates/job_group_list.html b/automation/server/monitor/templates/job_group_list.html new file mode 100644 index 00000000..b82fa730 --- /dev/null +++ b/automation/server/monitor/templates/job_group_list.html @@ -0,0 +1,35 @@ +{% extends "base.html" %} + +{% block content %} +<p class="title">Job Groups</p> + +<form action="/job-group" method="post"> +{{ filter.as_p }} +<p><input type="submit" value="Filter!" /></p> +</form> + +<table class="list"> + <tbody> + <tr> + <th>Group ID</th> + <th>Label</th> + <th>Time Submitted</th> + <th>Status</th> + </tr> + {% for group in groups %} + <tr> + <td> + <a class="button column" href="/job-group/{{ group.id }}">{{ group.id }}</a> + </td> + <td>{{ group.label }}</td> + <td>{{ group.submitted }}</td> + {% if group.status %} + <td class="{{ group.status }}">{{ group.state }}</td> + {% else %} + <td>{{ group.state }}</td> + {% endif %} + </tr> + {% endfor %} + </tbody> +</table> +{% endblock %} diff --git a/automation/server/monitor/templates/job_log.html b/automation/server/monitor/templates/job_log.html new file mode 100644 index 00000000..937b21b0 --- /dev/null +++ b/automation/server/monitor/templates/job_log.html @@ -0,0 +1,20 @@ +{% extends "base.html" %} + +{% block content %} +<h1 class="title">Job {{ job_id }}</h1> + +<h2 class="title">Command output:</h2> + +<table class="raw"> +<tbody> +{% for datetime, stream, line in log_lines %} +<tr class="{% cycle 'line1' 'line2' %}"> + <td>{{ datetime }}</td> + <td>{{ stream }}</td> + <td><pre class="code">{{ line|wordwrap:80 }}</pre></td> +</tr> +{% endfor %} +</tbody> +</table> + +{% endblock %} diff --git a/automation/server/monitor/templates/machine_list.html b/automation/server/monitor/templates/machine_list.html new file mode 100644 index 00000000..f81422d3 --- /dev/null +++ b/automation/server/monitor/templates/machine_list.html @@ -0,0 +1,39 @@ +{% extends "base.html" %} + +{% block content %} +<p class="title">Machines</p> + +<table class="list"> +<tbody> +<tr> + <th>Hostname</th> + <th>Label</th> + <th>CPU</th> + <th>Cores</th> + <th>Operating System</th> + <th>Jobs Running</th> + <th>Locked</th> +</tr> +{% for machine in machines %} +<tr> + <td> + <a class="button column" href="/machine/{{ machine.hostname }}"> + {{ machine.hostname }} + </a> + </td> + <td>{{ machine.label }}</td> + <td>{{ machine.cpu }}</td> + <td>{{ machine.cores }}</td> + <td>{{ machine.os }}</td> + <td>{{ machine.uses }}</td> + {% if machine.locked %} + <td class="failure">Yes</td> + {% else %} + <td class="success">No</td> + {% endif %} +</tr> +{% endfor %} +</tbody> +</table> + +{% endblock %} diff --git a/automation/server/monitor/templates/snippet_attribute_table.html b/automation/server/monitor/templates/snippet_attribute_table.html new file mode 100644 index 00000000..24bacc17 --- /dev/null +++ b/automation/server/monitor/templates/snippet_attribute_table.html @@ -0,0 +1,36 @@ +<table class="list attributes"> + <tbody> + <tr> + <th>Attribute</th> + <th>Value</th> + </tr> + {% for name, value in attributes.text %} + <tr> + <td>{{ name }}</td> + <td>{{ value }}</td> + </tr> + {% endfor %} + + {% for name, links in attributes.link %} + <tr> + <td>{{ name }}</td> + <td> + {% if links %} + {% for link in links %} + <a class="button small" href="{{ link.href }}">{{ link.name }}</a> + {% endfor %} + {% else %} + None + {% endif %} + </td> + </tr> + {% endfor %} + + {% for name, code in attributes.code %} + <tr> + <td>{{ name }}</td> + <td>{% include "snippet_code.html" %}</td> + </tr> + {% endfor %} + </tbody> +</table> diff --git a/automation/server/monitor/templates/snippet_code.html b/automation/server/monitor/templates/snippet_code.html new file mode 100644 index 00000000..281754d6 --- /dev/null +++ b/automation/server/monitor/templates/snippet_code.html @@ -0,0 +1,10 @@ +<table class="raw"> +<tbody> +{% for num, line in code %} +<tr class="{% cycle 'line1' 'line2' %}"> + <td>{{ num }}</td> + <td><pre class="code">{{ line|wordwrap:120 }}</pre></td> +</tr> +{% endfor %} +</tbody> +</table> diff --git a/automation/server/monitor/templates/snippet_links.html b/automation/server/monitor/templates/snippet_links.html new file mode 100644 index 00000000..f19fa6e5 --- /dev/null +++ b/automation/server/monitor/templates/snippet_links.html @@ -0,0 +1,7 @@ +{% if param %} +{% for link in param %} +<a class="button small" href="{{ link.href }}">{{ link.name }}</a> +{% endfor %} +{% else %} +None +{% endif %} diff --git a/automation/server/monitor/urls.py b/automation/server/monitor/urls.py new file mode 100644 index 00000000..1a6b2485 --- /dev/null +++ b/automation/server/monitor/urls.py @@ -0,0 +1,21 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +# + +__author__ = 'kbaclawski@google.com (Krystian Baclawski)' + +from django.conf import settings +from django.conf.urls.defaults import patterns + +urlpatterns = patterns( + 'dashboard', (r'^job-group$', 'JobGroupListPageHandler'), + (r'^machine$', 'MachineListPageHandler'), + (r'^job/(?P<job_id>\d+)/log$', 'LogPageHandler'), + (r'^job/(?P<job_id>\d+)$', 'JobPageHandler'), ( + r'^job-group/(?P<job_group_id>\d+)/files/(?P<path>.*)$', + 'JobGroupFilesPageHandler'), + (r'^job-group/(?P<job_group_id>\d+)$', 'JobGroupPageHandler'), + (r'^$', 'DefaultPageHandler')) + +urlpatterns += patterns('', + (r'^static/(?P<path>.*)$', 'django.views.static.serve', + {'document_root': settings.MEDIA_ROOT})) diff --git a/automation/server/server.py b/automation/server/server.py new file mode 100755 index 00000000..f02a1d0f --- /dev/null +++ b/automation/server/server.py @@ -0,0 +1,125 @@ +#!/usr/bin/python +# +# Copyright 2010 Google Inc. All Rights Reserved. + +import logging +import optparse +import pickle +import signal +from SimpleXMLRPCServer import SimpleXMLRPCServer +import sys + +from automation.common import logger +from automation.common.command_executer import CommandExecuter +from automation.server import machine_manager +from automation.server.job_group_manager import JobGroupManager +from automation.server.job_manager import JobManager + + +class Server(object): + """Plays a role of external interface accessible over XMLRPC.""" + + def __init__(self, machines_file=None, dry_run=False): + """Default constructor. + + Args: + machines_file: Path to file storing a list of machines. + dry_run: If True, the server only simulates command execution. + """ + CommandExecuter.Configure(dry_run) + + self.job_manager = JobManager( + machine_manager.MachineManager.FromMachineListFile( + machines_file or machine_manager.DEFAULT_MACHINES_FILE)) + + self.job_group_manager = JobGroupManager(self.job_manager) + + self._logger = logging.getLogger(self.__class__.__name__) + + def ExecuteJobGroup(self, job_group, dry_run=False): + job_group = pickle.loads(job_group) + self._logger.info('Received ExecuteJobGroup(%r, dry_run=%s) request.', + job_group, dry_run) + + for job in job_group.jobs: + job.dry_run = dry_run + return self.job_group_manager.AddJobGroup(job_group) + + def GetAllJobGroups(self): + self._logger.info('Received GetAllJobGroups() request.') + return pickle.dumps(self.job_group_manager.GetAllJobGroups()) + + def KillJobGroup(self, job_group_id): + self._logger.info('Received KillJobGroup(%d) request.', job_group_id) + self.job_group_manager.KillJobGroup(pickle.loads(job_group_id)) + + def GetJobGroup(self, job_group_id): + self._logger.info('Received GetJobGroup(%d) request.', job_group_id) + + return pickle.dumps(self.job_group_manager.GetJobGroup(job_group_id)) + + def GetJob(self, job_id): + self._logger.info('Received GetJob(%d) request.', job_id) + + return pickle.dumps(self.job_manager.GetJob(job_id)) + + def GetMachineList(self): + self._logger.info('Received GetMachineList() request.') + + return pickle.dumps(self.job_manager.machine_manager.GetMachineList()) + + def StartServer(self): + self.job_manager.StartJobManager() + + def StopServer(self): + self.job_manager.StopJobManager() + self.job_manager.join() + + +def GetServerOptions(): + """Get server's settings from command line options.""" + parser = optparse.OptionParser() + parser.add_option('-m', + '--machines-file', + dest='machines_file', + help='The location of the file ' + 'containing the machines database', + default=machine_manager.DEFAULT_MACHINES_FILE) + parser.add_option('-n', + '--dry-run', + dest='dry_run', + help='Start the server in dry-run mode, where jobs will ' + 'not actually be executed.', + action='store_true', + default=False) + return parser.parse_args()[0] + + +def Main(): + logger.SetUpRootLogger(filename='server.log', level=logging.DEBUG) + + options = GetServerOptions() + server = Server(options.machines_file, options.dry_run) + server.StartServer() + + def _HandleKeyboardInterrupt(*_): + server.StopServer() + sys.exit(1) + + signal.signal(signal.SIGINT, _HandleKeyboardInterrupt) + + try: + xmlserver = SimpleXMLRPCServer( + ('localhost', 8000), + allow_none=True, + logRequests=False) + xmlserver.register_instance(server) + xmlserver.serve_forever() + except Exception as ex: + logging.error(ex) + server.StopServer() + sys.exit(1) + + +if __name__ == '__main__': + Main() diff --git a/automation/server/server_test.py b/automation/server/server_test.py new file mode 100755 index 00000000..bcf1b9f5 --- /dev/null +++ b/automation/server/server_test.py @@ -0,0 +1,26 @@ +#!/usr/bin/python +# +# Copyright 2010 Google Inc. All Rights Reserved. +"""Machine manager unittest. + +MachineManagerTest tests MachineManager. +""" + +__author__ = 'asharif@google.com (Ahmad Sharif)' + +import server +import unittest + + +class ServerTest(unittest.TestCase): + + def setUp(self): + pass + + def testGetAllJobs(self): + s = server.Server() + print s.GetAllJobs() + + +if __name__ == '__main__': + unittest.main() diff --git a/automation/server/test_pool.csv b/automation/server/test_pool.csv new file mode 100644 index 00000000..b0700c9b --- /dev/null +++ b/automation/server/test_pool.csv @@ -0,0 +1,4 @@ +hostname,label,cpu,cores,os,username +"chrotomation.mtv.corp.google.com","pc-workstation","core2duo",8,"linux","mobiletc-prebuild" +"chromeos-test1.mtv.corp.google.com","cr48","atom",1,"chromeos","chromeos" +"chromeos-test2.mtv.corp.google.com","cr48","atom",1,"chromeos","chromeos" |