diff options
author | Javi Merino <javi.merino@arm.com> | 2015-08-10 15:59:10 +0100 |
---|---|---|
committer | Javi Merino <javi.merino@arm.com> | 2015-08-13 18:59:58 +0100 |
commit | 435457c8af9d69383ba45e0bd7da022d967a8dea (patch) | |
tree | c591b2b6494bf95fbe25006503f4c0cf9870cf6e /trappy/plotter | |
parent | dea8e9d314e5b0f9213c5f3ecf87ef4369537082 (diff) | |
download | trappy-435457c8af9d69383ba45e0bd7da022d967a8dea.tar.gz |
trappy: rename to trappy
Change-Id: I7e0e34c9f5565e34629683bb29ab25cf5e737088
Diffstat (limited to 'trappy/plotter')
-rw-r--r-- | trappy/plotter/AbstractDataPlotter.py | 54 | ||||
-rw-r--r-- | trappy/plotter/AttrConf.py | 131 | ||||
-rw-r--r-- | trappy/plotter/ColorMap.py | 36 | ||||
-rw-r--r-- | trappy/plotter/Constraint.py | 346 | ||||
-rw-r--r-- | trappy/plotter/EventPlot.py | 179 | ||||
-rw-r--r-- | trappy/plotter/ILinePlot.py | 163 | ||||
-rw-r--r-- | trappy/plotter/ILinePlotGen.py | 204 | ||||
-rw-r--r-- | trappy/plotter/LinePlot.py | 273 | ||||
-rw-r--r-- | trappy/plotter/PlotLayout.py | 113 | ||||
-rw-r--r-- | trappy/plotter/Utils.py | 158 | ||||
-rw-r--r-- | trappy/plotter/__init__.py | 55 | ||||
-rw-r--r-- | trappy/plotter/css/EventPlot.css | 72 | ||||
-rw-r--r-- | trappy/plotter/js/EventPlot.js | 530 | ||||
-rw-r--r-- | trappy/plotter/js/ILinePlot.js | 199 |
14 files changed, 2513 insertions, 0 deletions
diff --git a/trappy/plotter/AbstractDataPlotter.py b/trappy/plotter/AbstractDataPlotter.py new file mode 100644 index 0000000..49d3b46 --- /dev/null +++ b/trappy/plotter/AbstractDataPlotter.py @@ -0,0 +1,54 @@ +# Copyright 2015-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This is the template class that all Plotters inherit""" +from abc import abstractmethod, ABCMeta +from pandas import DataFrame +from trappy.plotter.Utils import listify +from functools import reduce +# pylint: disable=R0921 +# pylint: disable=R0903 + + +class AbstractDataPlotter(object): + + __metaclass__ = ABCMeta + + """This is an Abstract Data Plotting Class defining an interface + for the various Plotting Classes""" + + @abstractmethod + def view(self): + """View the graph""" + raise NotImplementedError("Method Not Implemented") + + @abstractmethod + def savefig(self, path): + """Save the image as a file""" + raise NotImplementedError("Method Not Implemented") + + def _check_data(self): + """Internal function to check the received data""" + + data = listify(self.runs) + + if len(data): + mask = map(lambda x: isinstance(x, DataFrame), data) + data_frame = reduce(lambda x, y: x and y, mask) + if not data_frame and not self.templates: + raise ValueError( + "Cannot understand data. Accepted DataFormats are pandas.DataFrame and trappy.Run (with templates)") + else: + raise ValueError("Empty Data received") diff --git a/trappy/plotter/AttrConf.py b/trappy/plotter/AttrConf.py new file mode 100644 index 0000000..7116112 --- /dev/null +++ b/trappy/plotter/AttrConf.py @@ -0,0 +1,131 @@ +# Copyright 2015-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""These are the default plotting Attributes""" +WIDTH = 7 +LENGTH = 7 +PER_LINE = 2 +CONCAT = False +PIVOT = "__TRAPPY_PIVOT_DEFAULT" +PIVOT_VAL = "__TRAPPY_DEFAULT_PIVOT_VAL" +DUPLICATE_VALUE_MAX_DELTA = 0.000001 +XLIM = None +YLIM = None +FILL = False +ALPHA = 0.75 + +MPL_STYLE = { + 'axes.axisbelow': True, + 'axes.color_cycle': ['#348ABD', + '#7A68A6', + '#A60628', + '#467821', + '#CF4457', + '#188487', + '#E24A33'], + 'axes.edgecolor': '#bcbcbc', + 'axes.facecolor': '#eeeeee', + 'axes.grid': True, + 'axes.labelcolor': '#555555', + 'axes.labelsize': 'large', + 'axes.linewidth': 1.0, + 'axes.titlesize': 'x-large', + 'figure.edgecolor': 'white', + 'figure.facecolor': 'white', + 'figure.figsize': (6.0, 4.0), + 'figure.subplot.hspace': 0.5, + 'font.size': 10, + 'interactive': True, + 'keymap.all_axes': ['a'], + 'keymap.back': ['left', 'c', 'backspace'], + 'keymap.forward': ['right', 'v'], + 'keymap.fullscreen': ['f'], + 'keymap.grid': ['g'], + 'keymap.home': ['h', 'r', 'home'], + 'keymap.pan': ['p'], + 'keymap.save': ['s'], + 'keymap.xscale': ['L', 'k'], + 'keymap.yscale': ['l'], + 'keymap.zoom': ['o'], + 'legend.fancybox': True, + 'lines.antialiased': True, + 'lines.linewidth': 1.0, + 'patch.antialiased': True, + 'patch.edgecolor': '#EEEEEE', + 'patch.facecolor': '#348ABD', + 'patch.linewidth': 0.5, + 'toolbar': 'toolbar2', + 'xtick.color': '#555555', + 'xtick.direction': 'in', + 'xtick.major.pad': 6.0, + 'xtick.major.size': 0.0, + 'xtick.minor.pad': 6.0, + 'xtick.minor.size': 0.0, + 'ytick.color': '#555555', + 'ytick.direction': 'in', + 'ytick.major.pad': 6.0, + 'ytick.major.size': 0.0, + 'ytick.minor.pad': 6.0, + 'ytick.minor.size': 0.0 +} +ARGS_TO_FORWARD = [ + "marker", + "markersize", + "markevery", + "linestyle", + "linewidth", + "drawstyle"] + +HTML_WIDTH = 900 +HTML_HEIGHT = 400 +# Sync Graph zoom by default in +# ILinePlot graph groups +DEFAULT_SYNC_ZOOM = False +EVENT_PLOT_STRIDE = False + +IPLOT_RESOURCES = { + "ILinePlot": [ + "http://cdnjs.cloudflare.com/ajax/libs/dygraph/1.1.1/dygraph-combined.js", + "js/ILinePlot.js", + "http://dygraphs.com/extras/synchronizer.js"], + "EventPlot": [ + "http://d3js.org/d3.v3.min.js", + "http://labratrevenge.com/d3-tip/javascripts/d3.tip.v0.6.3.js", + "js/EventPlot.js"]} + +try: + import IPython + import os + ip = IPython.get_ipython() + if not ip: + PLOTTER_IPYTHON = False + else: + PLOTTER_IPYTHON = True + PLOTTER_IPYTHON_PROFILE_DIR = ip.config.ProfileDir["location"] + PLOTTER_STATIC_DATA_DIR = os.path.join( + PLOTTER_IPYTHON_PROFILE_DIR, + "static", "plotter_data") + PLOTTER_SCRIPTS_DIR = "plotter_scripts" + PLOTTER_SCRIPTS_PATH = os.path.join( + PLOTTER_IPYTHON_PROFILE_DIR, + "static", + PLOTTER_SCRIPTS_DIR) + + if not os.path.isdir(PLOTTER_STATIC_DATA_DIR): + os.mkdir(PLOTTER_STATIC_DATA_DIR) + if not os.path.isdir(PLOTTER_SCRIPTS_PATH): + os.mkdir(PLOTTER_SCRIPTS_PATH) +except: + PLOTTER_IPYTHON = False diff --git a/trappy/plotter/ColorMap.py b/trappy/plotter/ColorMap.py new file mode 100644 index 0000000..7468d65 --- /dev/null +++ b/trappy/plotter/ColorMap.py @@ -0,0 +1,36 @@ +# Copyright 2015-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Defines a generic indexable ColorMap Class""" +import matplotlib.colors as clrs +import matplotlib.cm as cmx + + +class ColorMap(object): + + """The Color Map Class to return a gradient method""" + + def __init__(self, num_colors): + self.color_norm = clrs.Normalize(vmin=0, vmax=num_colors) + self.scalar_map = cmx.ScalarMappable(norm=self.color_norm, cmap='hsv') + self.num_colors = num_colors + + def cmap(self, index): + """Return the color at index""" + return self.scalar_map.to_rgba(index) + + def cmap_inv(self, index): + """Return the inverse color""" + return self.cmap(self.num_colors - index) diff --git a/trappy/plotter/Constraint.py b/trappy/plotter/Constraint.py new file mode 100644 index 0000000..c2a8b51 --- /dev/null +++ b/trappy/plotter/Constraint.py @@ -0,0 +1,346 @@ +# Copyright 2015-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This module provides the Constraint class for handling +filters and pivots in a modular fashion. This enable easy +constrain application + +What is a Constraint? +1. It is collection of data based on two rules: + a. A Pivot + b. A Set of Filters + +For Example: + for a dataframe + + Time CPU Latency + 1 x <val> + 2 y <val> + 3 z <val> + 4 a <val> + +The resultant data will be for each unique pivot value with the filters applied + +result["x"] = pd.Series.filtered() +result["y"] = pd.Series.filtered() +result["z"] = pd.Series.filtered() +result["a"] = pd.Series.filtered() + +""" +# pylint: disable=R0913 +from trappy.plotter.Utils import decolonize, listify, normalize_list +from trappy.plotter import AttrConf + + +class Constraint(object): + + """The constructor takes a filter and a pivot object, + The apply method takes a trappy Run object and a column + and applies the constraint on input object + """ + + def __init__( + self, trappy_run, pivot, column, template, run_index, filters): + self._trappy_run = trappy_run + self._filters = filters + self._pivot = pivot + self._column = column + self._template = template + self._dup_resolved = False + self._data = self.populate_data_frame() + + try: + self.result = self._apply() + except ValueError: + if not self._dup_resolved: + self._handle_duplicate_index() + try: + self.result = self._apply() + except: + raise ValueError("Unable to handle duplicates") + + self.run_index = run_index + + def _apply(self): + """This method applies the filter on the resultant data + on the input column. + Do we need pivot_val? + """ + data = self._data + result = {} + + try: + values = data[self._column] + except KeyError: + return result + + if self._pivot == AttrConf.PIVOT: + criterion = values.map(lambda x: True) + for key in self._filters.keys(): + if key in data.columns: + criterion = criterion & data[key].map( + lambda x: x in self._filters[key]) + values = values[criterion] + result[AttrConf.PIVOT_VAL] = values + return result + + pivot_vals = self.pivot_vals(data) + + for pivot_val in pivot_vals: + criterion = values.map(lambda x: True) + + for key in self._filters.keys(): + if key != self._pivot and key in data.columns: + criterion = criterion & data[key].map( + lambda x: x in self._filters[key]) + values = values[criterion] + + val_series = values[data[self._pivot] == pivot_val] + if len(val_series) != 0: + result[pivot_val] = val_series + + return result + + def _handle_duplicate_index(self): + """Handle duplicate values in index""" + data = self._data + self._dup_resolved = True + index = data.index + new_index = index.values + + dups = index.get_duplicates() + for dup in dups: + # Leave one of the values intact + dup_index_left = index.searchsorted(dup, side="left") + dup_index_right = index.searchsorted(dup, side="right") - 1 + num_dups = dup_index_right - dup_index_left + 1 + delta = (index[dup_index_right + 1] - dup) / num_dups + + if delta > AttrConf.DUPLICATE_VALUE_MAX_DELTA: + delta = AttrConf.DUPLICATE_VALUE_MAX_DELTA + + # Add a delta to the others + dup_index_left += 1 + while dup_index_left <= dup_index_right: + new_index[dup_index_left] += delta + delta += delta + dup_index_left += 1 + self._data = self._data.reindex(new_index) + + def _uses_trappy_run(self): + if not self._template: + return False + else: + return True + + def populate_data_frame(self): + """Return the data frame""" + if not self._uses_trappy_run(): + return self._trappy_run + + data_container = getattr( + self._trappy_run, + decolonize(self._template.name)) + return data_container.data_frame + + def pivot_vals(self, data): + """This method returns the unique pivot values for the + Constraint's pivot and the column + """ + if self._pivot == AttrConf.PIVOT: + return AttrConf.PIVOT_VAL + + if self._pivot not in data.columns: + return [] + + pivot_vals = set(data[self._pivot]) + if self._pivot in self._filters: + pivot_vals = pivot_vals & set(self._filters[self._pivot]) + + return list(pivot_vals) + + def __str__(self): + + name = self.get_data_name() + + if not self._uses_trappy_run(): + return name + ":" + self._column + + return name + ":" + \ + self._template.name + ":" + self._column + + + def get_data_name(self): + """Get name for the data Member""" + if self._uses_trappy_run(): + if self._trappy_run.name != "": + return self._trappy_run.name + else: + return "Run {}".format(self.run_index) + else: + return "DataFrame {}".format(self.run_index) + +class ConstraintManager(object): + + """A class responsible for converting inputs + to constraints and also ensuring sanity + """ + + def __init__(self, runs, columns, templates, pivot, filters, + zip_constraints=True): + + self._ip_vec = [] + self._ip_vec.append(listify(runs)) + self._ip_vec.append(listify(columns)) + self._ip_vec.append(listify(templates)) + + self._lens = map(len, self._ip_vec) + self._max_len = max(self._lens) + self._pivot = pivot + self._filters = filters + self._constraints = [] + + self._run_expanded = False + self._expand() + if zip_constraints: + self._populate_zip_constraints() + else: + self._populate_constraints() + + def _expand(self): + """This is really important. We need to + meet the following criteria for constraint + expansion: + + Len[runs] == Len[columns] == Len[templates] + OR + Permute( + Len[runs] = 1 + Len[columns] = 1 + Len[templates] != 1 + } + + + Permute( + Len[runs] = 1 + Len[columns] != 1 + Len[templates] != 1 + ) + + """ + min_len = min(self._lens) + max_pos_comp = [ + i for i, + j in enumerate( + self._lens) if j != self._max_len] + + if self._max_len == 1 and min_len != 1: + raise RuntimeError("Essential Arg Missing") + + if self._max_len > 1: + + # Are they all equal? + if len(set(self._lens)) == 1: + return + + if min_len > 1: + raise RuntimeError("Cannot Expand a list of Constraints") + + for val in max_pos_comp: + if val == 0: + self._run_expanded = True + self._ip_vec[val] = normalize_list(self._max_len, + self._ip_vec[val]) + + def _populate_constraints(self): + """Populate the constraints creating one for each column in each run + + In a multirun, multicolumn scenario, create constraints for + all the columns in each of the runs. _populate_constraints() + creates one constraint for the first run and first column, the + next for the second run and second column,... This function + creates a constraint for every combination of runs and columns + possible. + """ + + for run_idx, run in enumerate(self._ip_vec[0]): + for col in self._ip_vec[1]: + template = self._ip_vec[2][run_idx] + constraint = Constraint(run, self._pivot, col, template, + run_idx, self._filters) + self._constraints.append(constraint) + + def get_column_index(self, constraint): + return self._ip_vec[1].index(constraint._column) + + def _populate_zip_constraints(self): + """Populate the expanded constraints + + In a multirun, multicolumn scenario, create constraints for + the first run and the first column, second run and second + column,... that is, as if you run zip(runs, columns) + + """ + + for idx in range(self._max_len): + if self._run_expanded: + run_idx = 0 + else: + run_idx = idx + + run = self._ip_vec[0][idx] + col = self._ip_vec[1][idx] + template = self._ip_vec[2][idx] + self._constraints.append( + Constraint( + run, + self._pivot, + col, + template, + run_idx, + self._filters)) + + def generate_pivots(self, permute=False): + """Return a union of the pivot values""" + pivot_vals = [] + for constraint in self._constraints: + pivot_vals += constraint.result.keys() + + p_list = list(set(pivot_vals)) + runs = range(self._lens[0]) + + try: + sorted_plist = sorted(p_list, key=int) + except ValueError, TypeError: + try: + sorted_plist = sorted(p_list, key=lambda x: int(x, 16)) + except ValueError, TypeError: + sorted_plist = sorted(p_list) + + if permute: + pivot_gen = ((run_idx, pivot) for run_idx in runs for pivot in sorted_plist) + return pivot_gen, len(sorted_plist) * self._lens[0] + else: + return sorted_plist, len(sorted_plist) + + def constraint_labels(self): + """Get the Str representation of the constraints""" + return map(str, self._constraints) + + def __len__(self): + return len(self._constraints) + + def __iter__(self): + return iter(self._constraints) diff --git a/trappy/plotter/EventPlot.py b/trappy/plotter/EventPlot.py new file mode 100644 index 0000000..4943ff4 --- /dev/null +++ b/trappy/plotter/EventPlot.py @@ -0,0 +1,179 @@ +# Copyright 2015-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +The EventPlot is used to represent Events with two characteristics: + + * A name, which determines the colour on the plot + * A lane, which determines the lane in which the event occurred + +In the case of a cpu residency plot, the term lane can be equated to +a CPU and the name attribute can be the PID of the task +""" + +from trappy.plotter import AttrConf +import uuid +import json +import os +from IPython.display import display, HTML +from trappy.plotter.AbstractDataPlotter import AbstractDataPlotter + +if not AttrConf.PLOTTER_IPYTHON: + raise ImportError("Ipython Environment not Found") + +# pylint: disable=R0201 +# pylint: disable=R0921 +# Initialize Resources +from trappy.plotter import Utils +Utils.iplot_install("EventPlot") + + +class EventPlot(AbstractDataPlotter): + + """EventPlot Class that extends + AbstractDataPlotter""" + + def __init__( + self, + data, + keys, + lane_prefix, + num_lanes, + domain, + summary=True, + stride=False): + """ + Args: + data: Data of the format: + { "<name1>" : [ + [event_start, event_end, lane], + . + . + [event_start, event_end, lane], + ], + . + . + . + + "<nameN>" : [ + [event_start, event_end, lane], + . + . + [event_start, event_end, lane], + ], + } + keys: List of unique names in the data dictionary + lane_prefix: A string prefix to be used to name each lane + num_lanes: Total number of expected lanes + domain: Domain of the event data + stride: Stride can be used if the trace is very large. + It results in sampled rendering + """ + + self._fig_name = self._generate_fig_name + self._html = [] + self._fig_name = self._generate_fig_name() + avgFunc = lambda x: sum([(evt[1] - evt[0]) for evt in x]) / len(x) + avg = {k: avgFunc(v) for k, v in data.iteritems()} + graph = {} + graph["data"] = data + graph["lanes"] = self._get_lanes(lane_prefix, num_lanes) + graph["xDomain"] = domain + graph["keys"] = sorted(avg, key=lambda x: avg[x], reverse=True) + graph["showSummary"] = summary + graph["stride"] = AttrConf.EVENT_PLOT_STRIDE + + json_file = os.path.join( + AttrConf.PLOTTER_STATIC_DATA_DIR, + self._fig_name + + ".json") + + with open(json_file, "w") as json_fh: + json.dump(graph, json_fh) + + # Initialize the HTML, CSS and JS Components + self._add_css() + self._init_html() + + def view(self): + """Views the Graph Object""" + display(HTML(self.html())) + + def savefig(self, path): + """Save the plot in the provided path""" + + raise NotImplementedError( + "Save is not currently implemented for EventPlot") + + def _get_lanes(self, lane_prefix, num_lanes): + """Populate the lanes for the plot""" + + lanes = [] + for idx in range(num_lanes): + lanes.append({"id": idx, "label": "{}{}".format(lane_prefix, idx)}) + return lanes + + def _generate_fig_name(self): + """Generate a unqiue name for the figure""" + + fig_name = "fig_" + uuid.uuid4().hex + return fig_name + + def _init_html(self): + """Initialize HTML for the plot""" + div_js = """ + <script> + var req = require.config( { + + paths: { + + "EventPlot": "/static/plotter_scripts/EventPlot/EventPlot", + "d3-tip": "/static/plotter_scripts/EventPlot/d3.tip.v0.6.3", + "d3": "/static/plotter_scripts/EventPlot/d3.v3.min" + }, + shim: { + "d3-tip": ["d3"], + "EventPlot": { + + "deps": ["d3-tip", "d3" ], + "exports": "EventPlot" + } + } + }); + req(["require", "EventPlot"], function() { + EventPlot.generate('""" + self._fig_name + """'); + }); + </script> + """ + + self._html.append( + '<div id="{}" class="eventplot">{}</div>'.format(self._fig_name, + div_js)) + + def _add_css(self): + """Append the CSS to the HTML code generated""" + + base_dir = os.path.dirname(os.path.realpath(__file__)) + css_file = os.path.join(base_dir, "css/EventPlot.css") + css_fh = open(css_file, 'r') + self._html.append("<style>") + self._html += css_fh.readlines() + self._html.append("</style>") + css_fh.close() + + def html(self): + """Return a Raw HTML string for the plot""" + + return "\n".join(self._html) diff --git a/trappy/plotter/ILinePlot.py b/trappy/plotter/ILinePlot.py new file mode 100644 index 0000000..ab0bcdf --- /dev/null +++ b/trappy/plotter/ILinePlot.py @@ -0,0 +1,163 @@ +# Copyright 2015-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This module contains the class for plotting and +customizing Line Plots with a pandas dataframe input +""" + +import matplotlib.pyplot as plt +from trappy.plotter import AttrConf +from trappy.plotter.Constraint import ConstraintManager +from trappy.plotter.ILinePlotGen import ILinePlotGen +from trappy.plotter.AbstractDataPlotter import AbstractDataPlotter +from trappy.plotter.ColorMap import ColorMap +import pandas as pd + +if not AttrConf.PLOTTER_IPYTHON: + raise ImportError("Ipython Environment not Found") + +class ILinePlot(AbstractDataPlotter): + + """The plots are plotted by default against the dataframe index + The column="col_name" specifies the name of the column to + be plotted + + filters = + { + "pid": [ 3338 ], + "cpu": [0, 2, 4], + } + + The above filters will filter the column to be plotted as per the + specified criteria. + + per_line input is used to control the number of graphs + in each graph subplot row + concat, Draws all the graphs on a single plot + permute, draws one plot for each of the runs specified + """ + + def __init__(self, runs, templates=None, **kwargs): + # Default keys, each can be overridden in kwargs + self._attr = {} + self.runs = runs + self.templates = templates + self.set_defaults() + self._layout = None + + self._check_data() + for key in kwargs: + self._attr[key] = kwargs[key] + + if "column" not in self._attr: + raise RuntimeError("Value Column not specified") + + if self._attr["drawstyle"] and self._attr["drawstyle"].startswith("steps"): + self._attr["step_plot"] = True + + zip_constraints = not self._attr["permute"] + + self.c_mgr = ConstraintManager(runs, self._attr["column"], templates, + self._attr["pivot"], + self._attr["filters"], zip_constraints) + + super(ILinePlot, self).__init__() + + def savefig(self, *args, **kwargs): + raise NotImplementedError("Not Available for ILinePlot") + + def view(self, test=False): + """Displays the graph""" + + if self._attr["concat"]: + self._plot_concat() + else: + self._plot(self._attr["permute"]) + + def set_defaults(self): + """Sets the default attrs""" + self._attr["per_line"] = AttrConf.PER_LINE + self._attr["concat"] = AttrConf.CONCAT + self._attr["filters"] = {} + self._attr["pivot"] = AttrConf.PIVOT + self._attr["permute"] = False + self._attr["drawstyle"] = None + self._attr["step_plot"] = False + self._attr["fill"] = AttrConf.FILL + + def _plot(self, permute): + """Internal Method called to draw the plot""" + pivot_vals, len_pivots = self.c_mgr.generate_pivots(permute) + + self._layout = ILinePlotGen(self._attr["per_line"], + len_pivots, + **self._attr) + plot_index = 0 + for p_val in pivot_vals: + data_frame = pd.Series() + for constraint in self.c_mgr: + + if permute: + run_idx, pivot = p_val + if constraint.run_index != run_idx: + continue + title = constraint.get_data_name() + ":" + legend = constraint._column + else: + pivot = p_val + title = "" + legend = str(constraint) + + result = constraint.result + if pivot in result: + data_frame[legend] = result[pivot] + + if pivot == AttrConf.PIVOT_VAL: + title += ",".join(self._attr["column"]) + else: + title += "{0}: {1}".format(self._attr["pivot"], pivot) + + self._layout.add_plot(plot_index, data_frame, title) + plot_index += 1 + + self._layout.finish() + + def _plot_concat(self): + """Plot all lines on a single figure""" + + pivot_vals, _ = self.c_mgr.generate_pivots() + plot_index = 0 + + self._layout = ILinePlotGen(self._attr["per_line"], len(self.c_mgr), + **self._attr) + + for constraint in self.c_mgr: + result = constraint.result + title = str(constraint) + data_frame = pd.Series() + + for pivot in pivot_vals: + if pivot in result: + if pivot == AttrConf.PIVOT_VAL: + key = ",".join(self._attr["column"]) + else: + key = "{0}: {1}".format(self._attr["pivot"], pivot) + + data_frame[key] = result[pivot] + + self._layout.add_plot(plot_index, data_frame, title) + plot_index += 1 + + self._layout.finish() diff --git a/trappy/plotter/ILinePlotGen.py b/trappy/plotter/ILinePlotGen.py new file mode 100644 index 0000000..ae889f6 --- /dev/null +++ b/trappy/plotter/ILinePlotGen.py @@ -0,0 +1,204 @@ +# Copyright 2015-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This module is reponsible for creating a layout +of plots as a 2D axes and handling corener cases +and deleting empty plots +""" + +from trappy.plotter import AttrConf +import uuid +import json +import os +from IPython.display import display, HTML + + +if not AttrConf.PLOTTER_IPYTHON: + raise ImportError("No Ipython Environment found") + +# Install resources +from trappy.plotter import Utils +Utils.iplot_install("ILinePlot") + + +class ILinePlotGen(object): + + """Cols is the number of columns to draw + rows are calculated as 1D - 2D transformation + the same transformation is used to index the + axes array + """ + + def _add_graph_cell(self, fig_name): + """Add a HTML table cell to hold the plot""" + + width = int(self._attr["width"] / self._cols) + div_js = """ + <script> + var ilp_req = require.config( { + + paths: { + "dygraph-sync": "/static/plotter_scripts/ILinePlot/synchronizer", + "dygraph": "/static/plotter_scripts/ILinePlot/dygraph-combined", + "ILinePlot": "/static/plotter_scripts/ILinePlot/ILinePlot", + }, + + shim: { + "dygraph-sync": ["dygraph"], + "ILinePlot": { + + "deps": ["dygraph-sync", "dygraph" ], + "exports": "ILinePlot" + } + } + }); + ilp_req(["require", "ILinePlot"], function() { + ILinePlot.generate('""" + fig_name + """'); + }); + </script> + """ + + cell = '<td style="border-style: hidden;"><div class="ilineplot" id="{0}" style="width: \ +{1}px; height: {2}px;">{3}</div></td>'.format(fig_name, + width, + self._attr["height"], div_js) + + self._html.append(cell) + + def _add_legend_cell(self, fig_name): + """Add HTML table cell for the legend""" + + width = int(self._attr["width"] / self._cols) + legend_div_name = fig_name + "_legend" + cell = '<td style="border-style: hidden;"><div style="text-align:right; \ +width: {0}px; height: auto;"; id="{1}"></div></td>'.format(width, + legend_div_name) + + self._html.append(cell) + + def _begin_row(self): + """Add the opening tag for HTML row""" + + self._html.append("<tr>") + + def _end_row(self): + """Add the closing tag for the HTML row""" + + self._html.append("</tr>") + + def _generate_fig_name(self): + """Generate a unique figure name""" + + fig_name = "fig_" + uuid.uuid4().hex + self._fig_map[self._fig_index] = fig_name + self._fig_index += 1 + return fig_name + + def _init_html(self): + """Initialize HTML code for the plots""" + + width = self._attr["width"] + table = '<table style="width: {0}px; border-style: hidden;">'.format( + width) + self._html.append(table) + + for _ in range(self._rows): + self._begin_row() + + legend_figs = [] + for _ in range(self._cols): + fig_name = self._generate_fig_name() + legend_figs.append(fig_name) + self._add_graph_cell(fig_name) + + self._end_row() + self._begin_row() + + for l_fig in legend_figs: + self._add_legend_cell(l_fig) + + self._end_row() + + def __init__(self, cols, num_plots, **kwargs): + """ + Args: + cols (int): Number of plots in a single line + num_plots (int): Total Number of Plots + """ + + self._cols = cols + self._attr = kwargs + self._html = [] + self.num_plots = num_plots + self._fig_map = {} + self._fig_index = 0 + + self._single_plot = False + if self.num_plots == 0: + raise RuntimeError("No plots for the given constraints") + + if self.num_plots < self._cols: + self._cols = self.num_plots + self._rows = (self.num_plots / self._cols) + + if self.num_plots % self._cols != 0: + self._rows += 1 + + self._attr["width"] = AttrConf.HTML_WIDTH + self._attr["height"] = AttrConf.HTML_HEIGHT + self._init_html() + + def add_plot(self, plot_num, data_frame, title=""): + """Add a plot to for a corresponding index""" + + fig_name = self._fig_map[plot_num] + fig_params = {} + fig_params["data"] = json.loads(data_frame.to_json()) + fig_params["name"] = fig_name + fig_params["rangesel"] = False + fig_params["logscale"] = False + fig_params["title"] = title + fig_params["step_plot"] = self._attr["step_plot"] + fig_params["fill_graph"] = self._attr["fill"] + + if "group" in self._attr: + fig_params["syncGroup"] = self._attr["group"] + if "sync_zoom" in self._attr: + fig_params["syncZoom"] = self._attr["sync_zoom"] + else: + fig_params["syncZoom"] = AttrConf.DEFAULT_SYNC_ZOOM + + if "ylim" in self._attr: + fig_params["valueRange"] = self._attr["ylim"] + + json_file = os.path.join(AttrConf.PLOTTER_STATIC_DATA_DIR, fig_name + ".json") + fh = open(json_file, "w") + json.dump(fig_params, fh) + fh.close() + + def finish(self): + """Called when the Plotting is finished""" + + figs = [] + + for fig_idx in self._fig_map.keys(): + figs.append(self._fig_map[fig_idx]) + + display(HTML(self.html())) + + def html(self): + """Return the raw HTML text""" + + return "\n".join(self._html) diff --git a/trappy/plotter/LinePlot.py b/trappy/plotter/LinePlot.py new file mode 100644 index 0000000..35f3096 --- /dev/null +++ b/trappy/plotter/LinePlot.py @@ -0,0 +1,273 @@ +# Copyright 2015-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This module contains the class for plotting and +customizing Line Plots with a pandas dataframe input +""" + +import matplotlib.pyplot as plt +from trappy.plotter import AttrConf +from trappy.plotter.Constraint import ConstraintManager +from trappy.plotter.PlotLayout import PlotLayout +from trappy.plotter.AbstractDataPlotter import AbstractDataPlotter +from trappy.plotter.ColorMap import ColorMap + + +class LinePlot(AbstractDataPlotter): + + """The plots are plotted by default against the dataframe index + The column="col_name" specifies the name of the column to + be plotted + + filters = + { + "pid": [ 3338 ], + "cpu": [0, 2, 4], + } + + The above filters will filter the column to be plotted as per the + specified criteria. + + per_line input is used to control the number of graphs + in each graph subplot row + concat, Draws all the graphs on a single plot + permute, draws one plot for each of the runs specified + """ + + def __init__(self, runs, templates=None, **kwargs): + # Default keys, each can be overridden in kwargs + self._attr = {} + self.runs = runs + self.templates = templates + self.set_defaults() + self._fig = None + self._layout = None + + self._check_data() + + for key in kwargs: + if key in AttrConf.ARGS_TO_FORWARD: + self._attr["args_to_forward"][key] = kwargs[key] + else: + self._attr[key] = kwargs[key] + + if "column" not in self._attr: + raise RuntimeError("Value Column not specified") + + zip_constraints = not self._attr["permute"] + self.c_mgr = ConstraintManager( + runs, + self._attr["column"], + templates, + self._attr["pivot"], + self._attr["filters"], zip_constraints) + super(LinePlot, self).__init__() + + def savefig(self, *args, **kwargs): + if self._fig == None: + self.view() + self._fig.savefig(*args, **kwargs) + + def view(self, test=False): + """Displays the graph""" + + if test: + self._attr["style"] = True + AttrConf.MPL_STYLE["interactive"] = False + + if self._attr["concat"]: + if self._attr["style"]: + with plt.rc_context(AttrConf.MPL_STYLE): + self._plot_concat() + else: + self._plot_concat() + else: + if self._attr["style"]: + with plt.rc_context(AttrConf.MPL_STYLE): + self._plot(self._attr["permute"]) + else: + self._plot(self._attr["permute"]) + + def set_defaults(self): + """Sets the default attrs""" + self._attr["width"] = AttrConf.WIDTH + self._attr["length"] = AttrConf.LENGTH + self._attr["per_line"] = AttrConf.PER_LINE + self._attr["concat"] = AttrConf.CONCAT + self._attr["fill"] = AttrConf.FILL + self._attr["filters"] = {} + self._attr["style"] = True + self._attr["permute"] = False + self._attr["pivot"] = AttrConf.PIVOT + self._attr["xlim"] = AttrConf.XLIM + self._attr["ylim"] = AttrConf.XLIM + self._attr["args_to_forward"] = {} + + def _plot(self, permute): + """Internal Method called to draw the plot""" + pivot_vals, len_pivots = self.c_mgr.generate_pivots(permute) + + # Create a 2D Layout + self._layout = PlotLayout( + self._attr["per_line"], + len_pivots, + width=self._attr["width"], + length=self._attr["length"]) + + self._fig = self._layout.get_fig() + legend_str = [] + plot_index = 0 + + if permute: + legend = [None] * self.c_mgr._max_len + cmap = ColorMap(self.c_mgr._max_len) + else: + legend = [None] * len(self.c_mgr) + cmap = ColorMap(len(self.c_mgr)) + + for p_val in pivot_vals: + l_index = 0 + for constraint in self.c_mgr: + if permute: + run_idx, pivot = p_val + if constraint.run_index != run_idx: + continue + legend_str.append(constraint._column) + l_index = self.c_mgr.get_column_index(constraint) + title = constraint.get_data_name() + ":" + else: + pivot = p_val + legend_str.append(str(constraint)) + title = "" + + result = constraint.result + if pivot in result: + axis = self._layout.get_axis(plot_index) + line_2d_list = axis.plot( + result[pivot].index, + result[pivot].values, + color=cmap.cmap(l_index), + **self._attr["args_to_forward"]) + + if self._attr["fill"]: + drawstyle = line_2d_list[0].get_drawstyle() + # This has been fixed in upstream matplotlib + if drawstyle.startswith("steps"): + raise UserWarning("matplotlib does not support fill for step plots") + + xdat, ydat = line_2d_list[0].get_data(orig=False) + axis.fill_between(xdat, + axis.get_ylim()[0], + ydat, + facecolor=cmap.cmap(l_index), + alpha=AttrConf.ALPHA) + + legend[l_index] = line_2d_list[0] + if self._attr["xlim"] != None: + axis.set_xlim(self._attr["xlim"]) + if self._attr["ylim"] != None: + axis.set_ylim(self._attr["ylim"]) + + else: + axis = self._layout.get_axis(plot_index) + axis.plot([], [], **self._attr["args_to_forward"]) + + l_index += 1 + + if pivot == AttrConf.PIVOT_VAL: + title += ",".join(self._attr["column"]) + else: + title += "{0}: {1}".format(self._attr["pivot"], pivot) + + axis.set_title(title) + plot_index += 1 + + for l_idx, legend_line in enumerate(legend): + if not legend_line: + del legend[l_idx] + del legend_str[l_idx] + self._fig.legend(legend, legend_str) + self._layout.finish(len_pivots) + + def _plot_concat(self): + """Plot all lines on a single figure""" + + pivot_vals, len_pivots = self.c_mgr.generate_pivots() + cmap = ColorMap(len_pivots) + + self._layout = PlotLayout(self._attr["per_line"], len(self.c_mgr), + width=self._attr["width"], + length=self._attr["length"]) + + self._fig = self._layout.get_fig() + legend = [None] * len_pivots + legend_str = [""] * len_pivots + plot_index = 0 + + for constraint in self.c_mgr: + result = constraint.result + title = str(constraint) + result = constraint.result + pivot_index = 0 + for pivot in pivot_vals: + + if pivot in result: + axis = self._layout.get_axis(plot_index) + line_2d_list = axis.plot( + result[pivot].index, + result[pivot].values, + color=cmap.cmap(pivot_index), + **self._attr["args_to_forward"]) + + if self._attr["xlim"] != None: + axis.set_xlim(self._attr["xlim"]) + if self._attr["ylim"] != None: + axis.set_ylim(self._attr["ylim"]) + legend[pivot_index] = line_2d_list[0] + + if self._attr["fill"]: + drawstyle = line_2d_list[0].get_drawstyle() + if drawstyle.startswith("steps"): + # This has been fixed in upstream matplotlib + raise UserWarning("matplotlib does not support fill for step plots") + + xdat, ydat = line_2d_list[0].get_data(orig=False) + axis.fill_between(xdat, + axis.get_ylim()[0], + ydat, + facecolor=cmap.cmap(pivot_index), + alpha=AttrConf.ALPHA) + + if pivot == AttrConf.PIVOT_VAL: + legend_str[pivot_index] = self._attr["column"] + else: + legend_str[pivot_index] = "{0}: {1}".format(self._attr["pivot"], pivot) + + else: + axis = self._layout.get_axis(plot_index) + axis.plot( + [], + [], + color=cmap.cmap(pivot_index), + **self._attr["args_to_forward"]) + pivot_index += 1 + plot_index += 1 + + self._fig.legend(legend, legend_str) + plot_index = 0 + for constraint in self.c_mgr: + self._layout.get_axis(plot_index).set_title(str(constraint)) + plot_index += 1 + self._layout.finish(len(self.c_mgr)) diff --git a/trappy/plotter/PlotLayout.py b/trappy/plotter/PlotLayout.py new file mode 100644 index 0000000..c9d5e2e --- /dev/null +++ b/trappy/plotter/PlotLayout.py @@ -0,0 +1,113 @@ +# Copyright 2015-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This module is reponsible for creating a layout +of plots as a 2D axes and handling corener cases +and deleting empty plots +""" + +import matplotlib.pyplot as plt +from trappy.plotter import AttrConf + + +class PlotLayout(object): + + """Cols is the number of columns to draw + rows are calculated as 1D - 2D transformation + the same transformation is used to index the + axes array + """ + + def __init__(self, cols, num_plots, **kwargs): + + self.cols = cols + self._attr = {} + self.num_plots = num_plots + self._single_plot = False + if self.num_plots == 0: + raise RuntimeError("No plots for the given constraints") + + if self.num_plots < self.cols: + self.cols = self.num_plots + self.rows = (self.num_plots / self.cols) + # Avoid Extra Allocation (shows up in savefig!) + if self.num_plots % self.cols != 0: + self.rows += 1 + + self.usecol = False + self.userow = False + self._set_defaults() + + for key in kwargs: + self._attr[key] = kwargs[key] + + # Scale the plots if there is a single plot and + # Set boolean variables + if num_plots == 1: + self._single_plot = True + self._scale_plot() + elif self.rows == 1: + self.usecol = True + elif self.cols == 1: + self.userow = True + self._scale_plot() + + self._attr["figure"], self._attr["axes"] = plt.subplots( + self.rows, self.cols, figsize=( + self._attr["width"] * self.cols, + self._attr["length"] * self.rows)) + + def _scale_plot(self): + """Scale the graph in one + plot per line use case""" + + self._attr["width"] = int(self._attr["width"] * 2.5) + self._attr["length"] = int(self._attr["length"] * 1.25) + + def _set_defaults(self): + """set the default attrs""" + self._attr["width"] = AttrConf.WIDTH + self._attr["length"] = AttrConf.LENGTH + + def get_2d(self, linear_val): + """Convert Linear to 2D coordinates""" + if self.usecol: + return linear_val % self.cols + + if self.userow: + return linear_val % self.rows + + val_x = linear_val % self.cols + val_y = linear_val / self.cols + return val_y, val_x + + def finish(self, plot_index): + """Delete the empty cells""" + while plot_index < (self.rows * self.cols): + self._attr["figure"].delaxes( + self._attr["axes"][ + self.get_2d(plot_index)]) + plot_index += 1 + + def get_axis(self, plot_index): + """Get the axes for the plots""" + if self._single_plot: + return self._attr["axes"] + else: + return self._attr["axes"][self.get_2d(plot_index)] + + def get_fig(self): + """Return the matplotlib figure object""" + return self._attr["figure"] diff --git a/trappy/plotter/Utils.py b/trappy/plotter/Utils.py new file mode 100644 index 0000000..01a8bcc --- /dev/null +++ b/trappy/plotter/Utils.py @@ -0,0 +1,158 @@ +# Copyright 2015-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Utils module has generic utils that will be used across +objects +""" +import urllib +import os +import shutil +from trappy.plotter import AttrConf +import collections + + +def listify(to_select): + """Utitlity function to handle both single and + list inputs + """ + + if not isinstance(to_select, list): + to_select = [to_select] + + return to_select + + +def normalize_list(val, lst): + """Normalize a unitary list""" + + if len(lst) != 1: + raise RuntimeError("Cannot Normalize a non-unitary list") + + return lst * val + + +def decolonize(val): + """Remove the colon at the end of the word + This will be used by the unique word of + template class to sanitize attr accesses + """ + + return val.strip(":") + + +def install_http_resource(url, to_path): + """Install a HTTP Resource (eg. javascript) to + a destination on the disk + + Args: + url (str): HTTP URL + to_path (str): Destintation path on the disk + """ + urllib.urlretrieve(url, filename=to_path) + + +def install_local_resource(from_path, to_path): + """Move a local resource to the desired + a destination. + + Args: + from_path (str): Path relative to this file + to_path (str): Destintation path on the disk + """ + base_dir = os.path.dirname(__file__) + from_path = os.path.join(base_dir, from_path) + shutil.copy(from_path, to_path) + + +def install_resource(from_path, to_path): + """Install a resource to a location on the disk + + Args: + from_path (str): URL or relative path + to_path (str): Destintation path on the disk + """ + + if from_path.startswith("http"): + if not os.path.isfile(to_path): + install_http_resource(from_path, to_path) + else: + install_local_resource(from_path, to_path) + + +def iplot_install(module_name): + """Install the resources for the module to the Ipython + profile directory + + Args: + module_name (str): Name of the module + + Returns: + A list than can be consumed by requirejs or + any relative resource dependency resolver + """ + + resources = AttrConf.IPLOT_RESOURCES[module_name] + for resource in resources: + resource_name = os.path.basename(resource) + resource_dest_dir = os.path.join( + AttrConf.PLOTTER_SCRIPTS_PATH, + module_name) + + # Ensure if the directory exists + if not os.path.isdir(resource_dest_dir): + os.mkdir(resource_dest_dir) + resource_dest_path = os.path.join(resource_dest_dir, resource_name) + install_resource(resource, resource_dest_path) + +def get_trace_event_data(run): + """ + Args: + trappy.Run: A trappy.Run object + + Returns: + A list of objects that can be + consumed by EventPlot to plot task + residency like kernelshark + """ + + data = collections.defaultdict(list) + pmap = {} + + data_frame = run.sched_switch.data_frame + start_idx = data_frame.index.values[0] + end_idx = data_frame.index.values[-1] + + procs = {} + + for index, row in data_frame.iterrows(): + prev_pid = row["prev_pid"] + next_pid = row["next_pid"] + next_comm = row["next_comm"] + + if prev_pid in pmap.keys(): + name = pmap[prev_pid] + data[name][-1][1] = index + del pmap[prev_pid] + + if next_pid in pmap.keys(): + raise ValueError("Malformed data for PID: {}".format(next_pid)) + + if next_pid != 0 and not next_comm.startswith("migration"): + name = "{}-{}".format(next_comm, next_pid) + data[name].append([index, end_idx, row["__cpu"]]) + pmap[next_pid] = name + procs[name] = 1 + + return data, procs.keys(), [start_idx, end_idx] diff --git a/trappy/plotter/__init__.py b/trappy/plotter/__init__.py new file mode 100644 index 0000000..84232f2 --- /dev/null +++ b/trappy/plotter/__init__.py @@ -0,0 +1,55 @@ +# Copyright 2015-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Init Module for the Plotter Code""" + + +import pandas as pd +import LinePlot +import AttrConf +try: + import trappy.plotter.EventPlot +except ImportError: + pass +import Utils +import trappy + + +def register_forwarding_arg(arg_name): + """Allows the user to register args to + be forwarded to matplotlib + """ + if arg_name not in AttrConf.ARGS_TO_FORWARD: + AttrConf.ARGS_TO_FORWARD.append(arg_name) + +def unregister_forwarding_arg(arg_name): + """Unregisters arg_name from being passed to + plotter matplotlib calls + """ + try: + AttrConf.ARGS_TO_FORWARD.remove(arg_name) + except ValueError: + pass + +def plot_trace(trace_dir): + """Creates a kernelshark like plot of the trace file""" + + if not AttrConf.PLOTTER_IPYTHON: + raise RuntimeError("plot_trace needs ipython environment") + + run = trappy.Run(trace_dir) + data, procs, domain = Utils.get_trace_event_data(run) + trace_graph = EventPlot.EventPlot(data, procs, "CPU: ", int(run._cpus), domain) + trace_graph.view() diff --git a/trappy/plotter/css/EventPlot.css b/trappy/plotter/css/EventPlot.css new file mode 100644 index 0000000..b61943d --- /dev/null +++ b/trappy/plotter/css/EventPlot.css @@ -0,0 +1,72 @@ +/* + * Copyright 2015-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.d3-tip { + line-height: 1; + padding: 12px; + background: rgba(0, 0, 0, 0.6); + color: #fff; + border-radius: 2px; + position: absolute !important; + z-index: 99999; +} + +.d3-tip:after { + box-sizing: border-box; + pointer-events: none; + display: inline; + font-size: 10px; + width: 100%; + line-height: 1; + color: rgba(0, 0, 0, 0.6); + content: "\25BC"; + position: absolute !important; + z-index: 99999; + text-align: center; +} + +.d3-tip.n:after { + margin: -1px 0 0 0; + top: 100%; + left: 0; +} + +.chart { + shape-rendering: crispEdges; +} + +.mini text { + font: 9px sans-serif; +} + +.main text { + font: 12px sans-serif; +} + +.axis line, .axis path { + stroke: black; +} + +.miniItem { + stroke-width: 8; +} + +.brush .extent { + + stroke: #000; + fill-opacity: .125; + shape-rendering: crispEdges; +} diff --git a/trappy/plotter/js/EventPlot.js b/trappy/plotter/js/EventPlot.js new file mode 100644 index 0000000..ae41b6c --- /dev/null +++ b/trappy/plotter/js/EventPlot.js @@ -0,0 +1,530 @@ +/* + * Copyright 2015-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var EventPlot = (function () { + + /* EventPlot receives data that is hashed by the keys + * and each element in the data is sorted by start time. + * Since events on each lane are mutually exclusive, they + * they are also sorted by the end time. We use this information + * and binary search on the input data for filtering events + * This maintains filtering complexity to O[KLogN] + */ + + var search_data = function (data, key, value, left, right) { + + var mid; + + while (left < right) { + + mid = Math.floor((left + right) / 2) + if (data[mid][key] > value) + right = mid; + else + left = mid + 1; + } + return left; + } + + var generate = function (div_name) { + + var margin, brush, x, ext, yMain, chart, main, + mainAxis, + itemRects, items, colourAxis, tip, lanes; + + var json_file = "/static/plotter_data/" + div_name + + ".json" + + $.getJSON(json_file, function (d) { + + items = d.data; + lanes = d.lanes; + var names = d.keys; + var showSummary = d.showSummary; + + margin = { + top: 20, + right: 15, + bottom: 15, + left: 70 + }, width = 960 - margin.left - margin.right, + + mainHeight = 300 - margin.top - margin.bottom; + + x = d3.scale.linear() + .domain(d.xDomain) + .range([0, width]); + + var zoomScale = d3.scale.linear() + .domain(d.xDomain) + .range([0, width]); + + var xMin = x.domain()[0]; + var xMax = x.domain()[1]; + + + //Colour Ordinal scale. Uses Category20 Colors + colours = d3.scale.category20(); + colourAxis = d3.scale.ordinal() + .range(colours.range()) + .domain(names); + + brushScale = d3.scale.linear() + .range([0, width]); + ext = d3.extent(lanes, function (d) { + return d.id; + }); + yMain = d3.scale.linear() + .domain([ext[0], ext[1] + + 1 + ]) + .range([0, mainHeight]); + + + var ePlot; + + chart = d3.select('#' + div_name) + .append('svg:svg') + .attr('width', width + margin.right + + margin.left) + .attr('height', mainHeight + margin.top + + margin.bottom + 5) + .attr('class', 'chart') + + + main = chart.append('g') + .attr('transform', 'translate(' + margin.left + + ',' + margin.top + ')') + .attr('width', width) + .attr('height', mainHeight) + .attr('class', 'main') + + main.append('g') + .selectAll('.laneLines') + .data(lanes) + .enter() + .append('line') + .attr('x1', 0) + .attr('y1', function (d) { + return d3.round(yMain(d.id)) + 0.5; + }) + .attr('x2', width) + .attr('y2', function (d) { + return d3.round(yMain(d.id)) + 0.5; + }) + .attr('stroke', function (d) { + return d.label === '' ? 'white' : + 'lightgray' + }); + + main.append('g') + .selectAll('.laneText') + .data(lanes) + .enter() + .append('text') + .attr('x', 0) + .text(function (d) { + return d.label; + }) + .attr('y', function (d) { + return yMain(d.id + .5); + }) + .attr('dy', '0.5ex') + .attr('text-anchor', 'end') + .attr('class', 'laneText'); + + mainAxis = d3.svg.axis() + .scale(brushScale) + .orient('bottom'); + + tip = d3.tip() + .attr('class', 'd3-tip') + .html(function (d) { + return "<span style='color:white'>" + + d.name + "</span>"; + }) + + main.append('g') + .attr('transform', 'translate(0,' + + mainHeight + ')') + .attr('class', 'main axis') + .call(mainAxis); + + var ePlot; + + ePlot = { + div_name: div_name, + margin: margin, + chart: chart, + mainHeight: mainHeight, + width: width, + x: x, + brushScale: brushScale, + ext: ext, + yMain: yMain, + main: main, + mainAxis: mainAxis, + items: items, + colourAxis: colourAxis, + tip: tip, + lanes: lanes, + names: names, + }; + ePlot.zoomScale = zoomScale; + + if (showSummary) + ePlot.mini = drawMini(ePlot); + + var outgoing; + var zoomed = function () { + + if (zoomScale.domain()[0] < xMin) { + zoom.translate([zoom.translate()[ + 0] - zoomScale( + xMin) + + zoomScale.range()[0], + zoom.translate()[ + 1] + ]); + } else if (zoomScale.domain()[1] > + xMax) { + zoom.translate([zoom.translate()[ + 0] - zoomScale( + xMax) + + zoomScale.range()[1], + zoom.translate()[ + 1] + ]); + + } + + outgoing = main.selectAll(".mItem") + .attr("visibility", "hidden"); + drawMain(ePlot, zoomScale.domain()[0], + zoomScale.domain()[1]); + if (showSummary) { + brush.extent(zoomScale.domain()); + ePlot.mini.select(".brush") + .call( + brush); + } + + brushScale.domain(zoomScale.domain()); + ePlot.main.select('.main.axis') + .call(ePlot.mainAxis) + }; + + if (showSummary) { + var _brushed_event = function () { + main.selectAll("path") + .remove(); + var brush_xmin = brush.extent()[0]; + var brush_xmax = brush.extent()[1]; + + var t = zoom.translate(), + new_domain = brush.extent(), + scale; + + /* + * scale = x.range()[1] - x.range[0] + * -------------------------- + * x(x.domain()[1] - x.domain()[0]) + * + * _ _ + * new_domain[0] = x.invert | x.range()[0] - z.translate()[0] | + * | ------------------- | + * |_ z.scale() _| + * + * + * + * translate[0] = x.range()[0] - x(new_domain[0])) * zoom.scale() + */ + + scale = (width) / x(x.domain()[0] + + new_domain[1] - + new_domain[0]); + zoom.scale(scale); + t[0] = x.range()[0] - (x(new_domain[ + 0]) * scale); + zoom.translate(t); + + + brushScale.domain(brush.extent()) + drawMain(ePlot, brush_xmin, + brush_xmax); + ePlot.main.select('.main.axis') + .call(ePlot.mainAxis) + + }; + + brush = d3.svg.brush() + .x(x) + .extent(x.domain()) + .on("brush", _brushed_event); + + ePlot.mini.append('g') + .attr('class', 'brush') + .call(brush) + .selectAll('rect') + .attr('y', 1) + .attr('height', ePlot.miniHeight - 1); + } + + var zoom = d3.behavior.zoom() + .x(zoomScale) + .on( + "zoom", zoomed) + .on("zoomend", function () { + outgoing.remove() + }) + .scaleExtent([1, 4096]); + chart.call(zoom); + + drawMain(ePlot, xMin, xMax); + ePlot.main.select('.main.axis') + .call(ePlot.mainAxis) + return ePlot; + + }); + }; + + var drawMini = function (ePlot) { + + var miniHeight = ePlot.lanes.length * 12 + 50; + + var miniAxis = d3.svg.axis() + .scale(ePlot.x) + .orient('bottom'); + + var yMini = d3.scale.linear() + .domain([ePlot.ext[0], ePlot.ext[1] + + 1 + ]) + .range([0, miniHeight]); + + ePlot.yMini = yMini; + ePlot.miniAxis = miniAxis; + ePlot.miniHeight = miniHeight; + + var summary = d3.select("#" + ePlot.div_name) + .append( + "svg:svg") + .attr('width', ePlot.width + ePlot.margin.right + + ePlot.margin.left) + .attr('height', miniHeight + ePlot.margin.bottom + + ePlot.margin.top + 5) + .attr('class', 'chart') + + var mini = summary.append('g') + .attr("transform", "translate(" + ePlot.margin.left + + "," + ePlot.margin.top + ")") + .attr('width', ePlot.width) + .attr('height', ePlot.miniHeight) + .attr('class', 'mini'); + + mini.append('g') + .selectAll('.laneLines') + .data(ePlot.lanes) + .enter() + .append('line') + .attr('x1', 0) + .attr('y1', function (d) { + return d3.round(ePlot.yMini(d.id)) + 0.5; + }) + .attr('x2', ePlot.width) + .attr('y2', function (d) { + return d3.round(ePlot.yMini(d.id)) + 0.5; + }) + .attr('stroke', function (d) { + return d.label === '' ? 'white' : + 'lightgray' + }); + + mini.append('g') + .attr('transform', 'translate(0,' + + ePlot.miniHeight + ')') + .attr('class', 'axis') + .call(ePlot.miniAxis); + + + mini.append('g') + .selectAll('miniItems') + .data(getPaths(ePlot, ePlot.x, ePlot.yMini)) + .enter() + .append('path') + .attr('class', function (d) { + return 'miniItem' + }) + .attr('d', function (d) { + return d.path; + }) + .attr("stroke", function (d) { + return d.color + }) + + mini.append('g') + .selectAll('.laneText') + .data(ePlot.lanes) + .enter() + .append('text') + .text(function (d) { + return d.label; + }) + .attr('x', -10) + .attr('y', function (d) { + return ePlot.yMini(d.id + .5); + }) + .attr('dy', '0.5ex') + .attr('text-anchor', 'end') + .attr('class', 'laneText'); + + return mini; + }; + + + var drawMain = function (ePlot, xMin, xMax) { + + var rects, labels; + var dMin = 10000; + var paths = getPaths(ePlot, ePlot.zoomScale, ePlot.yMain); + ePlot.brushScale.domain([xMin, xMax]); + + if (paths.length == 0) + return; + + ePlot.main + .selectAll('mainItems') + .data(paths) + .enter() + .append('path') + .attr("shape-rendering", "crispEdges") + .attr('d', function (d) { + return d.path; + }) + .attr("class", "mItem") + .attr("stroke-width", function(d) { + return 0.8 * ePlot.yMain(1); + }) + .attr("stroke", function (d) { + return d.color + }) + .call(ePlot.tip) + .on("mouseover", ePlot.tip.show) + .on('mouseout', ePlot.tip.hide) + .on('mousemove', function () { + var xDisp = parseFloat(ePlot.tip.style("width")) / + 2.0 + ePlot.tip.style("left", (d3.event.pageX - xDisp) + + "px") + .style("top", Math.max(0, d3.event.pageY - + 47) + "px"); + }) + }; + + + function _handle_equality(d, xMin, xMax, x, y) { + var offset = 0.5 * y(1) + 0.5 + var bounds = [Math.max(d[0], xMin), Math.min(d[1], + xMax)] + if (bounds[0] < bounds[1]) + return 'M' + ' ' + x(bounds[0]) + ' ' + (y(d[2]) + offset) + ' H ' + x(bounds[1]); + else + return ''; + }; + + function _process(path, d, xMin, xMax, x, y, offset) { + + var start = d[0]; + if (start < xMin) + start = xMin; + var end = d[1]; + if (end > xMax) + end = xMax; + + start = x(start); + end = x(end); + + if ((end - start) < 0.01) + return path; + else if ((end - start) < 1) + end = start + 1; + + path += 'M' + ' ' + start + ' ' + (y(d[2]) + offset) + ' H ' + end; + return path; + } + + var _get_path = function(data, xMin, xMax, offset, x, y, stride) { + + var path = '' + var max_rects = 2000; + var right = search_data(data, 0, xMax, 0, data.length - + 1) + var left = search_data(data, 1, xMin, 0, right) + //Handle Equality + if (left == right) + return _handle_equality(data[left], xMin, xMax, x, y); + + data = data.slice(left, right + 1); + + var stride_length = 1; + if (stride) + stride_length = Math.max(Math.ceil(data.length / max_rects), 1); + + for (var i = 0; i < data.length; i+= stride_length) + path = _process(path, data[i], xMin, xMax, x, y, offset); + + return path; + } + + var getPaths = function (ePlot, x, y, stride) { + + var keys = ePlot.names; + var items = ePlot.items; + var colourAxis = ePlot.colourAxis; + + var xMin = x.domain()[0]; + var xMax = x.domain()[1]; + var paths = {}, + d, offset = 0.5 * y(1) + 0.5, + result = []; + + for (var i in keys) { + var name = keys[i]; + var path = _get_path(items[name], xMin, xMax, offset, x, y, stride) + /* This is critical. Adding paths for non + * existent processes in the window* can be + * very expensive as there is one SVG per process + * and SVG rendering is expensive + */ + if (!path || path == "") + continue + + result.push({ + color: colourAxis(name), + path: path, + name: name + }); + } + + return result; + + } + + return { + generate: generate, + }; + +}()); diff --git a/trappy/plotter/js/ILinePlot.js b/trappy/plotter/js/ILinePlot.js new file mode 100644 index 0000000..39da1d2 --- /dev/null +++ b/trappy/plotter/js/ILinePlot.js @@ -0,0 +1,199 @@ +/* + * Copyright 2015-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var ILinePlot = ( function() { + + var graphs = new Array(); + var syncObjs = new Array(); + + var convertToDataTable = function (d, index_col) { + + var columns = _.keys(d); + var out = []; + var index_col_default = false; + var index; + + if (index_col == undefined) { + + var index = []; + + columns.forEach(function(col) { + index = index.concat(Object.keys(d[col])); + }); + + index = $.unique(index); + index_col_default = true; + index = index.sort(function(a, b) { + return (parseFloat(a) - parseFloat(b)); + }); + } else { + index = d[index_col]; + columns.splice(columns.indexOf(index_col), 1); + } + + for (var ix in index) { + + var ix_val = ix; + + if (index_col_default) + ix_val = index[ix]; + + var row = [parseFloat(ix_val)]; + columns.forEach(function(col) { + + var val = d[col][ix_val]; + if (val == undefined) + val = null; + + row.push(val); + }); + out.push(row); + } + + var labels = ["index"].concat(columns); + return { + data: out, + labels: labels + } + }; + + var purge = function() { + for (var div_name in graphs) { + if (document.getElementById(div_name) == null) { + delete graphs[div_name]; + } + } + }; + + var sync = function(group) { + + var syncGraphs = Array(); + var xRange; + var yRange; + var syncZoom = true; + + for (var div_name in graphs) { + + if (graphs[div_name].group == group) { + syncGraphs.push(graphs[div_name].graph); + syncZoom = syncZoom & graphs[div_name].syncZoom; + + var xR = graphs[div_name].graph.xAxisRange(); + var yR = graphs[div_name].graph.yAxisRange(); + + if (xRange != undefined) { + if (xR[0] < xRange[0]) + xRange[0] = xR[0]; + if (xR[1] > xRange[1]) + xRange[1] = xR[1]; + } else + xRange = xR; + + if (yRange != undefined) { + if (yR[0] < yRange[0]) + yRange[0] = yR[0]; + if (yR[1] > yRange[1]) + yRange[1] = yR[1]; + } else + yRange = yR; + } + } + + if (syncGraphs.length >= 2) { + if (syncZoom) { + if (syncObjs[group] != undefined) + syncObjs[group].detach(); + + syncObjs[group] = Dygraph.synchronize(syncGraphs, { + zoom: true, + selection: false, + range: true + }); + } + + $.each(syncGraphs, function(g) { + var graph = syncGraphs[g]; + + graph.updateOptions({ + valueRange: yRange, + dateWindow: xRange + }); + + if (graph.padFront_ == undefined) { + graph.padFront_ = true; + var _decoy_elem = new Array(graph.rawData_[0].length); + graph.rawData_.unshift(_decoy_elem); + } + graph.rawData_[0][0] = xRange[0]; + + if (graph.padBack_ == undefined) { + graph.padBack_ = true; + var _decoy_elem = new Array(graph.rawData_[0].length); + graph.rawData_.push(_decoy_elem); + } + graph.rawData_[graph.rawData_.length - 1][0] = xRange[1]; + }); + } + }; + + var generate = function(div_name) { + var json_file = "/static/plotter_data/" + div_name + ".json"; + $.getJSON( json_file, function( data ) { + create_graph(data); + purge(); + if (data.syncGroup != undefined) + sync(data.syncGroup); + }); + }; + + var create_graph = function(t_info) { + var tabular = convertToDataTable(t_info.data, t_info.index_col); + + var graph = new Dygraph(document.getElementById(t_info.name), tabular.data, { + legend: 'always', + title: t_info.title, + labels: tabular.labels, + labelsDivStyles: { + 'textAlign': 'right' + }, + rollPeriod: 1, + animatedZooms: true, + connectSeparatedPoints: true, + showRangeSelector: t_info.rangesel, + rangeSelectorHeight: 50, + stepPlot: t_info.step_plot, + logscale: t_info.logscale, + fillGraph: t_info.fill_graph, + labelsDiv: t_info.name + "_legend", + errorBars: false, + valueRange: t_info.valueRange + + }); + + graphs[t_info.name] = + { + graph: graph, + group: t_info.syncGroup, + syncZoom: t_info.syncZoom + }; + + }; + + return { + generate: generate + }; + +}()); |