diff options
Diffstat (limited to 'deprecated/automation/server/monitor')
16 files changed, 710 insertions, 0 deletions
diff --git a/deprecated/automation/server/monitor/__init__.py b/deprecated/automation/server/monitor/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/deprecated/automation/server/monitor/__init__.py @@ -0,0 +1 @@ + diff --git a/deprecated/automation/server/monitor/dashboard.py b/deprecated/automation/server/monitor/dashboard.py new file mode 100644 index 00000000..f6befed8 --- /dev/null +++ b/deprecated/automation/server/monitor/dashboard.py @@ -0,0 +1,259 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +# + +__author__ = 'kbaclawski@google.com (Krystian Baclawski)' + +from collections import namedtuple +import glob +import gzip +import os.path +import pickle +import time +import xmlrpclib + +from django import forms +from django.http import HttpResponseRedirect +from django.shortcuts import render_to_response +from django.template import Context +from django.views import static + +Link = namedtuple('Link', 'href name') + + +def GetServerConnection(): + return xmlrpclib.Server('http://localhost:8000') + + +def MakeDefaultContext(*args): + context = Context({'links': [ + Link('/job-group', 'Job Groups'), Link('/machine', 'Machines') + ]}) + + for arg in args: + context.update(arg) + + return context + + +class JobInfo(object): + + def __init__(self, job_id): + self._job = pickle.loads(GetServerConnection().GetJob(job_id)) + + def GetAttributes(self): + job = self._job + + group = [Link('/job-group/%d' % job.group.id, job.group.label)] + + predecessors = [Link('/job/%d' % pred.id, pred.label) + for pred in job.predecessors] + + successors = [Link('/job/%d' % succ.id, succ.label) + for succ in job.successors] + + machines = [Link('/machine/%s' % mach.hostname, mach.hostname) + for mach in job.machines] + + logs = [Link('/job/%d/log' % job.id, 'Log')] + + commands = enumerate(job.PrettyFormatCommand().split('\n'), start=1) + + return {'text': [('Label', job.label), ('Directory', job.work_dir)], + 'link': [('Group', group), ('Predecessors', predecessors), + ('Successors', successors), ('Machines', machines), + ('Logs', logs)], + 'code': [('Command', commands)]} + + def GetTimeline(self): + return [{'started': evlog.GetTimeStartedFormatted(), + 'state_from': evlog.event.from_, + 'state_to': evlog.event.to_, + 'elapsed': evlog.GetTimeElapsedRounded()} + for evlog in self._job.timeline.GetTransitionEventHistory()] + + def GetLog(self): + log_path = os.path.join(self._job.logs_dir, + '%s.gz' % self._job.log_filename_prefix) + + try: + log = gzip.open(log_path, 'r') + except IOError: + content = [] + else: + # There's a good chance that file is not closed yet, so EOF handling + # function and CRC calculation will fail, thus we need to monkey patch the + # _read_eof method. + log._read_eof = lambda: None + + def SplitLine(line): + prefix, msg = line.split(': ', 1) + datetime, stream = prefix.rsplit(' ', 1) + + return datetime, stream, msg + + content = map(SplitLine, log.readlines()) + finally: + log.close() + + return content + + +class JobGroupInfo(object): + + def __init__(self, job_group_id): + self._job_group = pickle.loads(GetServerConnection().GetJobGroup( + job_group_id)) + + def GetAttributes(self): + group = self._job_group + + home_dir = [Link('/job-group/%d/files/' % group.id, group.home_dir)] + + return {'text': [('Label', group.label), + ('Time submitted', time.ctime(group.time_submitted)), + ('State', group.status), + ('Cleanup on completion', group.cleanup_on_completion), + ('Cleanup on failure', group.cleanup_on_failure)], + 'link': [('Directory', home_dir)]} + + def _GetJobStatus(self, job): + status_map = {'SUCCEEDED': 'success', 'FAILED': 'failure'} + return status_map.get(str(job.status), None) + + def GetJobList(self): + return [{'id': job.id, + 'label': job.label, + 'state': job.status, + 'status': self._GetJobStatus(job), + 'elapsed': job.timeline.GetTotalTime()} + for job in self._job_group.jobs] + + def GetHomeDirectory(self): + return self._job_group.home_dir + + def GetReportList(self): + job_dir_pattern = os.path.join(self._job_group.home_dir, 'job-*') + + filenames = [] + + for job_dir in glob.glob(job_dir_pattern): + filename = os.path.join(job_dir, 'report.html') + + if os.access(filename, os.F_OK): + filenames.append(filename) + + reports = [] + + for filename in sorted(filenames, key=lambda f: os.stat(f).st_ctime): + try: + with open(filename, 'r') as report: + reports.append(report.read()) + except IOError: + pass + + return reports + + +class JobGroupListInfo(object): + + def __init__(self): + self._all_job_groups = pickle.loads(GetServerConnection().GetAllJobGroups()) + + def _GetJobGroupState(self, group): + return str(group.status) + + def _GetJobGroupStatus(self, group): + status_map = {'SUCCEEDED': 'success', 'FAILED': 'failure'} + return status_map.get(self._GetJobGroupState(group), None) + + def GetList(self): + return [{'id': group.id, + 'label': group.label, + 'submitted': time.ctime(group.time_submitted), + 'state': self._GetJobGroupState(group), + 'status': self._GetJobGroupStatus(group)} + for group in self._all_job_groups] + + def GetLabelList(self): + return sorted(set(group.label for group in self._all_job_groups)) + + +def JobPageHandler(request, job_id): + job = JobInfo(int(job_id)) + + ctx = MakeDefaultContext({ + 'job_id': job_id, + 'attributes': job.GetAttributes(), + 'timeline': job.GetTimeline() + }) + + return render_to_response('job.html', ctx) + + +def LogPageHandler(request, job_id): + job = JobInfo(int(job_id)) + + ctx = MakeDefaultContext({'job_id': job_id, 'log_lines': job.GetLog()}) + + return render_to_response('job_log.html', ctx) + + +def JobGroupPageHandler(request, job_group_id): + group = JobGroupInfo(int(job_group_id)) + + ctx = MakeDefaultContext({ + 'group_id': job_group_id, + 'attributes': group.GetAttributes(), + 'job_list': group.GetJobList(), + 'reports': group.GetReportList() + }) + + return render_to_response('job_group.html', ctx) + + +def JobGroupFilesPageHandler(request, job_group_id, path): + group = JobGroupInfo(int(job_group_id)) + + return static.serve(request, + path, + document_root=group.GetHomeDirectory(), + show_indexes=True) + + +class FilterJobGroupsForm(forms.Form): + label = forms.ChoiceField(label='Filter by label:', required=False) + + +def JobGroupListPageHandler(request): + groups = JobGroupListInfo() + group_list = groups.GetList() + + field = FilterJobGroupsForm.base_fields['label'] + field.choices = [('*', '--- no filtering ---')] + field.choices.extend([(label, label) for label in groups.GetLabelList()]) + + if request.method == 'POST': + form = FilterJobGroupsForm(request.POST) + + if form.is_valid(): + label = form.cleaned_data['label'] + + if label != '*': + group_list = [group for group in group_list if group['label'] == label] + else: + form = FilterJobGroupsForm({'initial': '*'}) + + ctx = MakeDefaultContext({'filter': form, 'groups': group_list}) + + return render_to_response('job_group_list.html', ctx) + + +def MachineListPageHandler(request): + machine_list = pickle.loads(GetServerConnection().GetMachineList()) + + return render_to_response('machine_list.html', + MakeDefaultContext({'machines': machine_list})) + + +def DefaultPageHandler(request): + return HttpResponseRedirect('/job-group') diff --git a/deprecated/automation/server/monitor/manage.py b/deprecated/automation/server/monitor/manage.py new file mode 100755 index 00000000..59f6e216 --- /dev/null +++ b/deprecated/automation/server/monitor/manage.py @@ -0,0 +1,20 @@ +#!/usr/bin/python2 +# +# Copyright 2011 Google Inc. All Rights Reserved. +# + +__author__ = 'kbaclawski@google.com (Krystian Baclawski)' + +from django.core.management import execute_manager + +try: + import settings # Assumed to be in the same directory. +except ImportError: + import sys + + sys.stderr.write('Error: Can\'t find settings.py file in the directory ' + 'containing %r.' % __file__) + sys.exit(1) + +if __name__ == '__main__': + execute_manager(settings) diff --git a/deprecated/automation/server/monitor/settings.py b/deprecated/automation/server/monitor/settings.py new file mode 100644 index 00000000..8cd20e35 --- /dev/null +++ b/deprecated/automation/server/monitor/settings.py @@ -0,0 +1,49 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +# +# Django settings for monitor project. +# +# For explanation look here: http://docs.djangoproject.com/en/dev/ref/settings +# + +__author__ = 'kbaclawski@google.com (Krystian Baclawski)' + +import os.path +import sys + +# Path to the root of application. It's a custom setting, not related to Django. +ROOT_PATH = os.path.dirname(os.path.realpath(sys.argv[0])) + +# Print useful information during runtime if possible. +DEBUG = True +TEMPLATE_DEBUG = DEBUG + +# Sqlite3 database configuration, though we don't use it right now. +DATABASE_ENGINE = 'sqlite3' +DATABASE_NAME = os.path.join(ROOT_PATH, 'monitor.db') + +# Local time zone for this installation. +TIME_ZONE = 'America/Los_Angeles' + +# Language code for this installation. +LANGUAGE_CODE = 'en-us' + +# If you set this to False, Django will make some optimizations so as not +# to load the internationalization machinery. +USE_I18N = True + +# Absolute path to the directory that holds media. +MEDIA_ROOT = os.path.join(ROOT_PATH, 'static') + '/' + +# URL that handles the media served from MEDIA_ROOT. Make sure to use a +# trailing slash if there is a path component (optional in other cases). +MEDIA_URL = '/static/' + +# Used to provide a seed in secret-key hashing algorithms. Make this unique, +# and don't share it with anybody. +SECRET_KEY = '13p5p_4q91*8@yo+tvvt#2k&6#d_&e_zvxdpdil53k419i5sop' + +# A string representing the full Python import path to your root URLconf. +ROOT_URLCONF = 'monitor.urls' + +# List of locations of the template source files, in search order. +TEMPLATE_DIRS = (os.path.join(ROOT_PATH, 'templates'),) diff --git a/deprecated/automation/server/monitor/start.sh b/deprecated/automation/server/monitor/start.sh new file mode 100755 index 00000000..4fc53bef --- /dev/null +++ b/deprecated/automation/server/monitor/start.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# +# Copyright 2011 Google Inc. All Rights Reserved. +# Author: kbaclawski@google.com (Krystian Baclawski) +# + +./manage.py runserver "$HOSTNAME":8080 diff --git a/deprecated/automation/server/monitor/static/style.css b/deprecated/automation/server/monitor/static/style.css new file mode 100644 index 00000000..b571b059 --- /dev/null +++ b/deprecated/automation/server/monitor/static/style.css @@ -0,0 +1,101 @@ +* { font-family: sans-serif; } + +.left { text-align: left; } +.right { text-align: right; } + +.code { font-family: monospace; text-align: left; } +.line1 { background-color: Gainsboro; } +.line2 { background-color: WhiteSmoke; } + +.title { margin-bottom: 0.25em; } + +.success { background-color: LightGreen; } +.failure { background-color: LightPink; } + +pre.code { margin: 0px; } + +div.header p.title { + border: 1px solid black; + font-size: 32px; + font-style: bold; + background-color: LightBlue; + text-align: center; + margin: 0px; + padding: 10px; + font-weight: bold; +} + +div.links { + background-color: Azure; + margin-top: 2px; + padding: 8px 4px 8px 4px; + border: solid 1px; +} + +div.content { + margin-top: 2px; + padding: 8px; + border: solid 1px; +} + +div.content p.title { + font-size: 28px; + text-align: left; + margin: 0px; + margin-bottom: 8px; + padding: 12px; + font-weight: bold; +} + +table { border-collapse: collapse; } +td, th { text-align: center; } + +table.list td, th { padding: 3px 8px 2px 8px; border:1px solid black; } +table.list td { font-family: monospace; } +table.list th { background-color: LightGray; } + +table.attributes td { text-align: left; } +table.attributes > tbody > tr > td:first-child { font-family: sans-serif; } + +table.raw { border-style: none; } +table.raw td { + padding: 0em 0.5em 0em 0.5em; + border-style: none; + vertical-align: top; + text-align: right; + font-family: monospace; +} +table.raw > tbody > tr > td:first-child { border-left: 0px; } +table.raw > tbody > tr > td { border-left: 1px solid; } + +a.button { + background-color: PeachPuff; + text-decoration: underline; + text-align: center; + color: Black; + padding: 4px; + border: solid 1px; +} + +a.small { + padding: 2px 4px 2px 4px; + font-size: small; + border-color: Gray; + background-color: PapayaWhip; +} + +a.button:hover { background-color: LightYellow; } +a.button:active { background-color: Yellow; } + +a.column { + border-style: none; + display: block; + margin: -3px -8px -2px -8px; +} + +div.warning { + background-color: MistyRose; + border: 1px solid Crimson; + padding: 0.5em; + font-size: x-large; +} diff --git a/deprecated/automation/server/monitor/templates/base.html b/deprecated/automation/server/monitor/templates/base.html new file mode 100644 index 00000000..95ffc222 --- /dev/null +++ b/deprecated/automation/server/monitor/templates/base.html @@ -0,0 +1,30 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" +"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> + <head> + <link rel="stylesheet" href="/static/style.css" /> + <title>{% block title %}Automation Dashboard{% endblock %}</title> + </head> + + <body> + <div class="header"> + {% block header %} + <p class="title">Automation Dashboard</p> + {% endblock %} + </div> + + <div class="links"> + <span>Subpages:</span> + {% block links %} + {% for link in links %} + <a class="button" href="{{ link.href }}">{{ link.name }}</a> + {% endfor %} + {% endblock %} + </div> + + <div class="content"> + {% block content %} + {% endblock %} + </div> + </body> +</html> diff --git a/deprecated/automation/server/monitor/templates/job.html b/deprecated/automation/server/monitor/templates/job.html new file mode 100644 index 00000000..90acd969 --- /dev/null +++ b/deprecated/automation/server/monitor/templates/job.html @@ -0,0 +1,29 @@ +{% extends "base.html" %} + +{% block content %} +<h1 class="title">Job {{ job_id }}</h1> + +<h2 class="title">General information</h2> +{% include "snippet_attribute_table.html" %} + +<h2 class="title">Timeline of status events</h2> +<table class="list"> + <tbody> + <tr> + <th>Started</th> + <th>From State</th> + <th>To State</th> + <th>Elapsed</th> + </tr> + {% for entry in timeline %} + <tr> + <td>{{ entry.started }}</td> + <td>{{ entry.state_from }}</td> + <td>{{ entry.state_to }}</td> + <td>{{ entry.elapsed }}</td> + </tr> + {% endfor %} + </tbody> +</table> + +{% endblock %} diff --git a/deprecated/automation/server/monitor/templates/job_group.html b/deprecated/automation/server/monitor/templates/job_group.html new file mode 100644 index 00000000..b6ed8ea8 --- /dev/null +++ b/deprecated/automation/server/monitor/templates/job_group.html @@ -0,0 +1,46 @@ +{% extends "base.html" %} + +{% block content %} +<h1 class="title">Job Group {{ group_id }}</h1> + +<h2 class="title">General information</h2> +{% include "snippet_attribute_table.html" %} + +<h2 class="title">Job Listing</h2> +<table class="list"> + <tbody> + <tr> + <th>Job ID</th> + <th>Label</th> + <th>Turnaround Time</th> + <th>State</th> + </tr> + {% for job in job_list %} + <tr> + <td> + <a class="button column" href="/job/{{ job.id }}">{{ job.id }}</a> + </td> + <td>{{ job.label }}</td> + <td>{{ job.elapsed }}</td> + {% if job.status %} + <td class="{{ job.status }}">{{ job.state }}</td> + {% else %} + <td>{{ job.state }}</td> + {% endif %} + </tr> + {% endfor %} + </tbody> +</table> + +<h2 class="title">Report</h2> +{% if reports %} +{% autoescape off %} +{% for report in reports %} +{{ report }} +{% endfor %} +{% endautoescape %} +{% else %} +<div class="warning">No reports found!</div> +{% endif %} + +{% endblock %} diff --git a/deprecated/automation/server/monitor/templates/job_group_list.html b/deprecated/automation/server/monitor/templates/job_group_list.html new file mode 100644 index 00000000..b82fa730 --- /dev/null +++ b/deprecated/automation/server/monitor/templates/job_group_list.html @@ -0,0 +1,35 @@ +{% extends "base.html" %} + +{% block content %} +<p class="title">Job Groups</p> + +<form action="/job-group" method="post"> +{{ filter.as_p }} +<p><input type="submit" value="Filter!" /></p> +</form> + +<table class="list"> + <tbody> + <tr> + <th>Group ID</th> + <th>Label</th> + <th>Time Submitted</th> + <th>Status</th> + </tr> + {% for group in groups %} + <tr> + <td> + <a class="button column" href="/job-group/{{ group.id }}">{{ group.id }}</a> + </td> + <td>{{ group.label }}</td> + <td>{{ group.submitted }}</td> + {% if group.status %} + <td class="{{ group.status }}">{{ group.state }}</td> + {% else %} + <td>{{ group.state }}</td> + {% endif %} + </tr> + {% endfor %} + </tbody> +</table> +{% endblock %} diff --git a/deprecated/automation/server/monitor/templates/job_log.html b/deprecated/automation/server/monitor/templates/job_log.html new file mode 100644 index 00000000..937b21b0 --- /dev/null +++ b/deprecated/automation/server/monitor/templates/job_log.html @@ -0,0 +1,20 @@ +{% extends "base.html" %} + +{% block content %} +<h1 class="title">Job {{ job_id }}</h1> + +<h2 class="title">Command output:</h2> + +<table class="raw"> +<tbody> +{% for datetime, stream, line in log_lines %} +<tr class="{% cycle 'line1' 'line2' %}"> + <td>{{ datetime }}</td> + <td>{{ stream }}</td> + <td><pre class="code">{{ line|wordwrap:80 }}</pre></td> +</tr> +{% endfor %} +</tbody> +</table> + +{% endblock %} diff --git a/deprecated/automation/server/monitor/templates/machine_list.html b/deprecated/automation/server/monitor/templates/machine_list.html new file mode 100644 index 00000000..f81422d3 --- /dev/null +++ b/deprecated/automation/server/monitor/templates/machine_list.html @@ -0,0 +1,39 @@ +{% extends "base.html" %} + +{% block content %} +<p class="title">Machines</p> + +<table class="list"> +<tbody> +<tr> + <th>Hostname</th> + <th>Label</th> + <th>CPU</th> + <th>Cores</th> + <th>Operating System</th> + <th>Jobs Running</th> + <th>Locked</th> +</tr> +{% for machine in machines %} +<tr> + <td> + <a class="button column" href="/machine/{{ machine.hostname }}"> + {{ machine.hostname }} + </a> + </td> + <td>{{ machine.label }}</td> + <td>{{ machine.cpu }}</td> + <td>{{ machine.cores }}</td> + <td>{{ machine.os }}</td> + <td>{{ machine.uses }}</td> + {% if machine.locked %} + <td class="failure">Yes</td> + {% else %} + <td class="success">No</td> + {% endif %} +</tr> +{% endfor %} +</tbody> +</table> + +{% endblock %} diff --git a/deprecated/automation/server/monitor/templates/snippet_attribute_table.html b/deprecated/automation/server/monitor/templates/snippet_attribute_table.html new file mode 100644 index 00000000..24bacc17 --- /dev/null +++ b/deprecated/automation/server/monitor/templates/snippet_attribute_table.html @@ -0,0 +1,36 @@ +<table class="list attributes"> + <tbody> + <tr> + <th>Attribute</th> + <th>Value</th> + </tr> + {% for name, value in attributes.text %} + <tr> + <td>{{ name }}</td> + <td>{{ value }}</td> + </tr> + {% endfor %} + + {% for name, links in attributes.link %} + <tr> + <td>{{ name }}</td> + <td> + {% if links %} + {% for link in links %} + <a class="button small" href="{{ link.href }}">{{ link.name }}</a> + {% endfor %} + {% else %} + None + {% endif %} + </td> + </tr> + {% endfor %} + + {% for name, code in attributes.code %} + <tr> + <td>{{ name }}</td> + <td>{% include "snippet_code.html" %}</td> + </tr> + {% endfor %} + </tbody> +</table> diff --git a/deprecated/automation/server/monitor/templates/snippet_code.html b/deprecated/automation/server/monitor/templates/snippet_code.html new file mode 100644 index 00000000..281754d6 --- /dev/null +++ b/deprecated/automation/server/monitor/templates/snippet_code.html @@ -0,0 +1,10 @@ +<table class="raw"> +<tbody> +{% for num, line in code %} +<tr class="{% cycle 'line1' 'line2' %}"> + <td>{{ num }}</td> + <td><pre class="code">{{ line|wordwrap:120 }}</pre></td> +</tr> +{% endfor %} +</tbody> +</table> diff --git a/deprecated/automation/server/monitor/templates/snippet_links.html b/deprecated/automation/server/monitor/templates/snippet_links.html new file mode 100644 index 00000000..f19fa6e5 --- /dev/null +++ b/deprecated/automation/server/monitor/templates/snippet_links.html @@ -0,0 +1,7 @@ +{% if param %} +{% for link in param %} +<a class="button small" href="{{ link.href }}">{{ link.name }}</a> +{% endfor %} +{% else %} +None +{% endif %} diff --git a/deprecated/automation/server/monitor/urls.py b/deprecated/automation/server/monitor/urls.py new file mode 100644 index 00000000..1a6b2485 --- /dev/null +++ b/deprecated/automation/server/monitor/urls.py @@ -0,0 +1,21 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +# + +__author__ = 'kbaclawski@google.com (Krystian Baclawski)' + +from django.conf import settings +from django.conf.urls.defaults import patterns + +urlpatterns = patterns( + 'dashboard', (r'^job-group$', 'JobGroupListPageHandler'), + (r'^machine$', 'MachineListPageHandler'), + (r'^job/(?P<job_id>\d+)/log$', 'LogPageHandler'), + (r'^job/(?P<job_id>\d+)$', 'JobPageHandler'), ( + r'^job-group/(?P<job_group_id>\d+)/files/(?P<path>.*)$', + 'JobGroupFilesPageHandler'), + (r'^job-group/(?P<job_group_id>\d+)$', 'JobGroupPageHandler'), + (r'^$', 'DefaultPageHandler')) + +urlpatterns += patterns('', + (r'^static/(?P<path>.*)$', 'django.views.static.serve', + {'document_root': settings.MEDIA_ROOT})) |