summaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tools')
-rw-r--r--tools/.gitignore1
-rw-r--r--tools/fio.service10
-rwxr-xr-xtools/fio_generate_plots126
-rw-r--r--tools/fio_generate_plots.145
-rwxr-xr-xtools/fiologparser.py221
-rwxr-xr-xtools/genfio355
-rw-r--r--tools/hist/.gitignore3
-rwxr-xr-xtools/hist/fiologparser_hist.py388
-rw-r--r--tools/hist/fiologparser_hist.py.1201
-rwxr-xr-xtools/hist/half-bins.py38
-rwxr-xr-xtools/plot/fio2gnuplot527
-rw-r--r--tools/plot/fio2gnuplot.1161
-rw-r--r--tools/plot/fio2gnuplot.manpage117
-rw-r--r--tools/plot/graph2D.gpm55
-rw-r--r--tools/plot/graph3D.gpm95
-rw-r--r--tools/plot/math.gpm42
-rw-r--r--tools/plot/samples/Makefile19
-rw-r--r--tools/plot/samples/fio-logs.tar.gzbin68085 -> 0 bytes
18 files changed, 0 insertions, 2404 deletions
diff --git a/tools/.gitignore b/tools/.gitignore
deleted file mode 100644
index b25c15b8..00000000
--- a/tools/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*~
diff --git a/tools/fio.service b/tools/fio.service
deleted file mode 100644
index 21de0b7a..00000000
--- a/tools/fio.service
+++ /dev/null
@@ -1,10 +0,0 @@
-[Unit]
-
-Description=flexible I/O tester server
-After=network.target
-
-[Service]
-
-Type=simple
-PIDFile=/run/fio.pid
-ExecStart=/usr/bin/fio --server
diff --git a/tools/fio_generate_plots b/tools/fio_generate_plots
deleted file mode 100755
index a47bfa5c..00000000
--- a/tools/fio_generate_plots
+++ /dev/null
@@ -1,126 +0,0 @@
-#!/bin/sh
-#
-# This script is an almost total rewrite by Louwrentius
-# of the original fio_generate_plots script provided as part of the FIO storage
-# benchmark utiliy. I only retained how GNUplot is used to generate graphs, as
-# that is something I know nothing about.
-#
-# The script uses the files generated by FIO to create nice graphs in the
-# SVG format. This output format is supported by most modern browsers and
-# allows resolution independent graphs to be generated.
-#
-# This script supports GNUPLOT 4.4 and higher.
-#
-# Version 1.0 @ 20121231
-#
-#
-#
-
-if [ -z "$1" ]; then
- echo "Usage: fio_generate_plots subtitle [xres yres]"
- exit 1
-fi
-
-GNUPLOT=$(which gnuplot)
-if [ ! -x "$GNUPLOT" ]
-then
- echo You need gnuplot installed to generate graphs
- exit 1
-fi
-
-TITLE="$1"
-
-# set resolution
-if [ ! -z "$2" ] && [ ! -z "$3" ]
-then
- XRES="$2"
- YRES="$3"
-else
- XRES=1280
- YRES=768
-fi
-
-if [ -z "$SAMPLE_DURATION" ]
-then
- SAMPLE_DURATION="*"
-fi
-
-DEFAULT_GRID_LINE_TYPE=3
-DEFAULT_LINE_WIDTH=2
-DEFAULT_LINE_COLORS="
-set object 1 rectangle from screen 0,0 to screen 1,1 fillcolor rgb\"#ffffff\" behind
-set style line 1 lc rgb \"#E41A1C\" lw $DEFAULT_LINE_WIDTH lt 1;
-set style line 2 lc rgb \"#377EB8\" lw $DEFAULT_LINE_WIDTH lt 1;
-set style line 3 lc rgb \"#4DAF4A\" lw $DEFAULT_LINE_WIDTH lt 1;
-set style line 4 lc rgb \"#984EA3\" lw $DEFAULT_LINE_WIDTH lt 1;
-set style line 5 lc rgb \"#FF7F00\" lw $DEFAULT_LINE_WIDTH lt 1;
-set style line 6 lc rgb \"#DADA33\" lw $DEFAULT_LINE_WIDTH lt 1;
-set style line 7 lc rgb \"#A65628\" lw $DEFAULT_LINE_WIDTH lt 1;
-set style line 20 lc rgb \"#000000\" lt $DEFAULT_GRID_LINE_TYPE lw $DEFAULT_LINE_WIDTH;
-"
-
-DEFAULT_TERMINAL="set terminal svg enhanced dashed size $XRES,$YRES dynamic"
-DEFAULT_TITLE_FONT="\"Helvetica,28\""
-DEFAULT_AXIS_FONT="\"Helvetica,14\""
-DEFAULT_AXIS_LABEL_FONT="\"Helvetica,16\""
-DEFAULT_XLABEL="set xlabel \"Time (sec)\" font $DEFAULT_AXIS_LABEL_FONT"
-DEFAULT_XTIC="set xtics font $DEFAULT_AXIS_FONT"
-DEFAULT_YTIC="set ytics font $DEFAULT_AXIS_FONT"
-DEFAULT_MXTIC="set mxtics 0"
-DEFAULT_MYTIC="set mytics 2"
-DEFAULT_XRANGE="set xrange [0:$SAMPLE_DURATION]"
-DEFAULT_YRANGE="set yrange [0:*]"
-DEFAULT_GRID="set grid ls 20"
-DEFAULT_KEY="set key outside bottom center ; set key box enhanced spacing 2.0 samplen 3 horizontal width 4 height 1.2 "
-DEFAULT_SOURCE="set label 30 \"Data source: http://example.com\" font $DEFAULT_AXIS_FONT tc rgb \"#00000f\" at screen 0.976,0.175 right"
-DEFAULT_OPTS="$DEFAULT_LINE_COLORS ; $DEFAULT_GRID_LINE ; $DEFAULT_GRID ; $DEFAULT_GRID_MINOR ; $DEFAULT_XLABEL ; $DEFAULT_XRANGE ; $DEFAULT_YRANGE ; $DEFAULT_XTIC ; $DEFAULT_YTIC ; $DEFAULT_MXTIC ; $DEFAULT_MYTIC ; $DEFAULT_KEY ; $DEFAULT_TERMINAL ; $DEFAULT_SOURCE"
-
-plot () {
-
- if [ -z "$TITLE" ]
- then
- PLOT_TITLE=" set title \"$1\" font $DEFAULT_TITLE_FONT"
- else
- PLOT_TITLE=" set title \"$TITLE\\\n\\\n{/*0.6 "$1"}\" font $DEFAULT_TITLE_FONT"
- fi
- FILETYPE="$2"
- YAXIS="set ylabel \"$3\" font $DEFAULT_AXIS_LABEL_FONT"
- SCALE=$4
-
- echo "Title: $PLOT_TITLE"
- echo "File type: $FILETYPE"
- echo "yaxis: $YAXIS"
-
- i=0
-
- for x in *_"$FILETYPE".log
- do
- i=$((i+1))
- PT=$(echo $x | sed s/_"$FILETYPE".log//g)
- if [ ! -z "$PLOT_LINE" ]
- then
- PLOT_LINE=$PLOT_LINE", "
- fi
-
- DEPTH=$(echo $PT | cut -d "-" -f 4)
- PLOT_LINE=$PLOT_LINE"'$x' using (\$1/1000):(\$2/$SCALE) title \"Queue depth $DEPTH\" with lines ls $i"
-
- done
-
- OUTPUT="set output \"$TITLE-$FILETYPE.svg\" "
-
- echo " $PLOT_TITLE ; $YAXIS ; $DEFAULT_OPTS ; show style lines ; $OUTPUT ; plot " $PLOT_LINE | $GNUPLOT -
- unset PLOT_LINE
-}
-
-#
-# plot <sub title> <file name tag> <y axis label> <y axis scale>
-#
-
-plot "I/O Latency" lat "Time (msec)" 1000
-plot "I/O Operations Per Second" iops "IOPS" 1
-plot "I/O Submission Latency" slat "Time (μsec)" 1
-plot "I/O Completion Latency" clat "Time (msec)" 1000
-plot "I/O Bandwidth" bw "Throughput (KB/s)" 1
-
-
diff --git a/tools/fio_generate_plots.1 b/tools/fio_generate_plots.1
deleted file mode 100644
index 9e3c1ff9..00000000
--- a/tools/fio_generate_plots.1
+++ /dev/null
@@ -1,45 +0,0 @@
-.\" Hey, EMACS: -*- nroff -*-
-.\" First parameter, NAME, should be all caps
-.\" Second parameter, SECTION, should be 1-8, maybe w/ subsection
-.\" other parameters are allowed: see man(7), man(1)
-.TH FIO_GENERATE_PLOTS 1 "May 19, 2009"
-.\" Please adjust this date whenever revising the manpage.
-.\"
-.\" Some roff macros, for reference:
-.\" .nh disable hyphenation
-.\" .hy enable hyphenation
-.\" .ad l left justify
-.\" .ad b justify to both left and right margins
-.\" .nf disable filling
-.\" .fi enable filling
-.\" .br insert line break
-.\" .sp <n> insert n+1 empty lines
-.\" for manpage-specific macros, see man(7)
-.SH NAME
-fio_generate_plots \- Generate plots for Flexible I/O Tester
-.SH SYNOPSIS
-.B fio_generate_plots
-.RI " title"
-.br
-.SH DESCRIPTION
-This manual page documents briefly the
-.B fio_generate_plots
-command. This manual page was written for the Debian distribution
-because the original program does not have a manual page.
-.PP
-.\" TeX users may be more comfortable with the \fB<whatever>\fP and
-.\" \fI<whatever>\fP escape sequences to invode bold face and italics,
-.\" respectively.
-\fBfio_generate_plots\fP is a shell script that uses gnuplot to
-generate plots from fio run with \-\-latency-log (\-l) and/or
-\-\-bandwidth-log (\-w). It expects the log files that fio
-generated in the current directory.
-.SH OPTIONS
-The script takes the title of the plot as only argument. It does
-not offer any additional options.
-.SH AUTHOR
-fio_generate_plots was written by Jens Axboe <jens.axboe@oracle.com>,
-now Jens Axboe <jaxboe@fusionio.com>.
-.PP
-This manual page was written by Martin Steigerwald <ms@teamix.de>,
-for the Debian project (but may be used by others).
diff --git a/tools/fiologparser.py b/tools/fiologparser.py
deleted file mode 100755
index 5a95009e..00000000
--- a/tools/fiologparser.py
+++ /dev/null
@@ -1,221 +0,0 @@
-#!/usr/bin/python
-#
-# fiologparser.py
-#
-# This tool lets you parse multiple fio log files and look at interaval
-# statistics even when samples are non-uniform. For instance:
-#
-# fiologparser.py -s *bw*
-#
-# to see per-interval sums for all bandwidth logs or:
-#
-# fiologparser.py -a *clat*
-#
-# to see per-interval average completion latency.
-
-import argparse
-import math
-
-def parse_args():
- parser = argparse.ArgumentParser()
- parser.add_argument('-i', '--interval', required=False, type=int, default=1000, help='interval of time in seconds.')
- parser.add_argument('-d', '--divisor', required=False, type=int, default=1, help='divide the results by this value.')
- parser.add_argument('-f', '--full', dest='full', action='store_true', default=False, help='print full output.')
- parser.add_argument('-A', '--all', dest='allstats', action='store_true', default=False,
- help='print all stats for each interval.')
- parser.add_argument('-a', '--average', dest='average', action='store_true', default=False, help='print the average for each interval.')
- parser.add_argument('-s', '--sum', dest='sum', action='store_true', default=False, help='print the sum for each interval.')
- parser.add_argument("FILE", help="collectl log output files to parse", nargs="+")
- args = parser.parse_args()
-
- return args
-
-def get_ftime(series):
- ftime = 0
- for ts in series:
- if ftime == 0 or ts.last.end < ftime:
- ftime = ts.last.end
- return ftime
-
-def print_full(ctx, series):
- ftime = get_ftime(series)
- start = 0
- end = ctx.interval
-
- while (start < ftime):
- end = ftime if ftime < end else end
- results = [ts.get_value(start, end) for ts in series]
- print("%s, %s" % (end, ', '.join(["%0.3f" % i for i in results])))
- start += ctx.interval
- end += ctx.interval
-
-def print_sums(ctx, series):
- ftime = get_ftime(series)
- start = 0
- end = ctx.interval
-
- while (start < ftime):
- end = ftime if ftime < end else end
- results = [ts.get_value(start, end) for ts in series]
- print("%s, %0.3f" % (end, sum(results)))
- start += ctx.interval
- end += ctx.interval
-
-def print_averages(ctx, series):
- ftime = get_ftime(series)
- start = 0
- end = ctx.interval
-
- while (start < ftime):
- end = ftime if ftime < end else end
- results = [ts.get_value(start, end) for ts in series]
- print("%s, %0.3f" % (end, float(sum(results))/len(results)))
- start += ctx.interval
- end += ctx.interval
-
-# FIXME: this routine is computationally inefficient
-# and has O(N^2) behavior
-# it would be better to make one pass through samples
-# to segment them into a series of time intervals, and
-# then compute stats on each time interval instead.
-# to debug this routine, use
-# # sort -n -t ',' -k 2 small.log
-# on your input.
-
-def my_extend( vlist, val ):
- vlist.extend(val)
- return vlist
-
-array_collapser = lambda vlist, val: my_extend(vlist, val)
-
-def print_all_stats(ctx, series):
- ftime = get_ftime(series)
- start = 0
- end = ctx.interval
- print('start-time, samples, min, avg, median, 90%, 95%, 99%, max')
- while (start < ftime): # for each time interval
- end = ftime if ftime < end else end
- sample_arrays = [ s.get_samples(start, end) for s in series ]
- samplevalue_arrays = []
- for sample_array in sample_arrays:
- samplevalue_arrays.append(
- [ sample.value for sample in sample_array ] )
- # collapse list of lists of sample values into list of sample values
- samplevalues = reduce( array_collapser, samplevalue_arrays, [] )
- # compute all stats and print them
- mymin = min(samplevalues)
- myavg = sum(samplevalues) / float(len(samplevalues))
- mymedian = median(samplevalues)
- my90th = percentile(samplevalues, 0.90)
- my95th = percentile(samplevalues, 0.95)
- my99th = percentile(samplevalues, 0.99)
- mymax = max(samplevalues)
- print( '%f, %d, %f, %f, %f, %f, %f, %f, %f' % (
- start, len(samplevalues),
- mymin, myavg, mymedian, my90th, my95th, my99th, mymax))
-
- # advance to next interval
- start += ctx.interval
- end += ctx.interval
-
-def median(values):
- s=sorted(values)
- return float(s[(len(s)-1)/2]+s[(len(s)/2)])/2
-
-def percentile(values, p):
- s = sorted(values)
- k = (len(s)-1) * p
- f = math.floor(k)
- c = math.ceil(k)
- if f == c:
- return s[int(k)]
- return (s[int(f)] * (c-k)) + (s[int(c)] * (k-f))
-
-def print_default(ctx, series):
- ftime = get_ftime(series)
- start = 0
- end = ctx.interval
- averages = []
- weights = []
-
- while (start < ftime):
- end = ftime if ftime < end else end
- results = [ts.get_value(start, end) for ts in series]
- averages.append(sum(results))
- weights.append(end-start)
- start += ctx.interval
- end += ctx.interval
-
- total = 0
- for i in range(0, len(averages)):
- total += averages[i]*weights[i]
- print('%0.3f' % (total/sum(weights)))
-
-class TimeSeries(object):
- def __init__(self, ctx, fn):
- self.ctx = ctx
- self.last = None
- self.samples = []
- self.read_data(fn)
-
- def read_data(self, fn):
- f = open(fn, 'r')
- p_time = 0
- for line in f:
- (time, value, foo, bar) = line.rstrip('\r\n').rsplit(', ')
- self.add_sample(p_time, int(time), int(value))
- p_time = int(time)
-
- def add_sample(self, start, end, value):
- sample = Sample(ctx, start, end, value)
- if not self.last or self.last.end < end:
- self.last = sample
- self.samples.append(sample)
-
- def get_samples(self, start, end):
- sample_list = []
- for s in self.samples:
- if s.start >= start and s.end <= end:
- sample_list.append(s)
- return sample_list
-
- def get_value(self, start, end):
- value = 0
- for sample in self.samples:
- value += sample.get_contribution(start, end)
- return value
-
-class Sample(object):
- def __init__(self, ctx, start, end, value):
- self.ctx = ctx
- self.start = start
- self.end = end
- self.value = value
-
- def get_contribution(self, start, end):
- # short circuit if not within the bound
- if (end < self.start or start > self.end):
- return 0
-
- sbound = self.start if start < self.start else start
- ebound = self.end if end > self.end else end
- ratio = float(ebound-sbound) / (end-start)
- return self.value*ratio/ctx.divisor
-
-
-if __name__ == '__main__':
- ctx = parse_args()
- series = []
- for fn in ctx.FILE:
- series.append(TimeSeries(ctx, fn))
- if ctx.sum:
- print_sums(ctx, series)
- elif ctx.average:
- print_averages(ctx, series)
- elif ctx.full:
- print_full(ctx, series)
- elif ctx.allstats:
- print_all_stats(ctx, series)
- else:
- print_default(ctx, series)
-
diff --git a/tools/genfio b/tools/genfio
deleted file mode 100755
index 68004520..00000000
--- a/tools/genfio
+++ /dev/null
@@ -1,355 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
-# Author: Erwan Velu <erwan@enovance.com>
-#
-# The license below covers all files distributed with fio unless otherwise
-# noted in the file itself.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-BLK_SIZE=
-BLOCK_SIZE=4k
-SEQ=-1
-TEMPLATE=/tmp/template.fio
-OUTFILE=
-DISKS=
-PRINTABLE_DISKS=
-RUNTIME=300
-ETA=0
-MODES="write,randwrite,read,randread"
-SHORT_HOSTNAME=
-CACHED_IO="FALSE"
-PREFIX=""
-PREFIX_FILENAME=""
-IODEPTH=1
-
-show_help() {
- PROG=$(basename $0)
- echo "usage of $PROG:"
- cat << EOF
--h : Show this help & exit
--c : Enable cached-based IOs
- Disabled by default
--a : Run sequential test then parallel one
- Disabled by default
--s : Run sequential test (default value)
- one test after another then one disk after another
- Disabled by default
--p : Run parallel test
- one test after anoter but all disks at the same time
- Enabled by default
--D iodepth : Run with the specified iodepth
- Default is $IODEPTH
--d disk1[,disk2,disk3,..] : Run the tests on the selected disks
- Separated each disk with a comma
--z filesize : Specify the working file size, if you are passing filepaths to -d
- Disabled by default
--r seconds : Time in seconds per benchmark
- 0 means till the end of the device
- Default is $RUNTIME seconds
--b blocksize[,blocksize1, ...] : The blocksizes to test under fio format (4k, 1m, ...)
- Separated each blocksize with a comma
- Default is $BLOCK_SIZE
--m mode1,[mode2,mode3, ...] : Define the fio IO profile to use like read, write, randread, randwrite
- Default is "$MODES"
--x prefix : Add a prefix to the fio filename
- Useful to let a context associated with the file
- If the prefix features a / (slash), prefix will be considered as a directory
--A cmd_to_run : System command to run after each job (exec_postrun in fio)
--B cmd_to_run : System command to run before each job (exec_prerun in fio)
-
-Example:
-
-$PROG -d /dev/sdb,/dev/sdc,/dev/sdd,/dev/sde -a -b 4k,128k,1m -r 100 -a -x dellr720-day2/
-
- Will generate an fio file that will run
- - a sequential bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 4k with write,randwrite,read,randread tests
- ETA ~ 4 tests * 4 disks * 100 seconds
- - a sequential bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 128k with write,randwrite,read,randread tests
- ETA ~ 4 tests * 4 disks * 100 seconds
- - a sequential bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 1m with write,randwrite,read,randread tests
- ETA ~ 4 tests * 4 disks * 100 seconds
- - a parallel bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 4k with write,randwrite,read,randread tests
- ETA ~ 4 tests * 100 seconds
- - a parallel bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 128k with write,randwrite,read,randread tests
- ETA ~ 4 tests * 100 seconds
- - a parallel bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 1m with write,randwrite,read,randread tests
- ETA ~ 4 tests * 100 seconds
-
-Generating dellr720-day2/localhost-4k,128k,1m-all-write,randwrite,read,randread-sdb,sdc,sdd,sde.fio
-Estimated Time = 6000 seconds : 1 hour 40 minutes
-EOF
-}
-
-finish_template() {
-echo "iodepth=$IODEPTH" >> $TEMPLATE
-
-if [ "$RUNTIME" != "0" ]; then
- echo "runtime=$RUNTIME" >> $TEMPLATE
- echo "time_based" >> $TEMPLATE
-fi
-
-if [ "$CACHED_IO" = "FALSE" ]; then
- echo "direct=1" >> $TEMPLATE
-fi
-}
-
-
-diskname_to_printable() {
-COUNT=0
-for disk in $(echo $@ | tr "," " "); do
- R=$(basename $disk | sed 's|/|_|g')
- COUNT=$(($COUNT + 1))
- if [ $COUNT -eq 1 ]; then
- P="$R"
- else
- P="$P,$R"
- fi
-done
-echo $P
-}
-
-gen_template() {
-cat >$TEMPLATE << EOF
-[global]
-ioengine=libaio
-invalidate=1
-ramp_time=5
-EOF
-}
-
-gen_seq_suite() {
-TYPE=$1
-disk=$2
-PRINTABLE_DISK=$(diskname_to_printable $disk)
-cat >> $OUTFILE << EOF
-[$TYPE-$PRINTABLE_DISK-$BLK_SIZE-seq]
-stonewall
-bs=$BLK_SIZE
-filename=$disk
-rw=$TYPE
-write_bw_log=${PREFIX_FILENAME}$SHORT_HOSTNAME-$BLK_SIZE-$PRINTABLE_DISK-$TYPE-seq.results
-write_iops_log=${PREFIX_FILENAME}$SHORT_HOSTNAME-$BLK_SIZE-$PRINTABLE_DISK-$TYPE-seq.results
-EOF
-ETA=$(($ETA + $RUNTIME))
-}
-
-gen_seq_fio() {
-for disk in $(echo $DISKS | tr "," " "); do
- for mode in $(echo $MODES | tr "," " "); do
- gen_seq_suite "$mode" "$disk"
- done
-done
-}
-
-
-gen_para_suite() {
-TYPE=$1
-NEED_WALL=$2
-D=0
-for disk in $(echo $DISKS | tr "," " "); do
- PRINTABLE_DISK=$(diskname_to_printable $disk)
- cat >> $OUTFILE << EOF
-[$TYPE-$PRINTABLE_DISK-$BLK_SIZE-para]
-bs=$BLK_SIZE
-EOF
-
-if [ "$D" = 0 ]; then
- echo "stonewall" >> $OUTFILE
- D=1
-fi
-
-cat >> $OUTFILE << EOF
-filename=$disk
-rw=$TYPE
-write_bw_log=${PREFIX_FILENAME}$SHORT_HOSTNAME-$BLK_SIZE-$PRINTABLE_DISK-$TYPE-para.results
-write_iops_log=${PREFIX_FILENAME}$SHORT_HOSTNAME-$BLK_SIZE-$PRINTABLE_DISK-$TYPE-para.results
-EOF
-done
-
-ETA=$(($ETA + $RUNTIME))
-echo >> $OUTFILE
-}
-
-gen_para_fio() {
-for mode in $(echo $MODES | tr "," " "); do
- gen_para_suite "$mode"
-done
-}
-
-gen_fio() {
-case $SEQ in
- 2)
- gen_seq_fio
- gen_para_fio
- ;;
- 1)
- gen_seq_fio
- ;;
- 0)
- gen_para_fio
- ;;
-esac
-}
-
-parse_cmdline() {
-while getopts "hacpsd:b:r:m:x:z:D:A:B:" opt; do
- case $opt in
- h)
- show_help
- exit 0
- ;;
- b)
- BLOCK_SIZE=$OPTARG
- ;;
- c)
- CACHED_IO="TRUE"
- ;;
- s)
- if [ "$SEQ" = "-1" ]; then
- SEQ=1
- fi
- ;;
- x)
- PREFIX=$OPTARG
- echo "$PREFIX" | grep -q "/"
- if [ "$?" -eq 0 ]; then
- mkdir -p $PREFIX
- # No need to keep the prefix for the log files
- # we do have a directory for that
- PREFIX_FILENAME=""
- else
- # We need to keep the prefix for the log files
- PREFIX_FILENAME=$PREFIX
- fi
- ;;
- r)
- RUNTIME=$OPTARG
- ;;
- p)
- if [ "$SEQ" = "-1" ]; then
- SEQ=0
- fi
- ;;
- m)
- MODES=$OPTARG;
- ;;
- d)
- DISKS=$OPTARG
- PRINTABLE_DISKS=$(diskname_to_printable "$DISKS")
- ;;
- D)
- IODEPTH=$OPTARG
- ;;
- a)
- SEQ=2
- ;;
- B)
- echo "exec_prerun=$OPTARG" >> $TEMPLATE
- ;;
- A)
- echo "exec_postrun=$OPTARG" >> $TEMPLATE
- ;;
- z)
- FSIZE=$OPTARG
- echo "size=$FSIZE" >> $TEMPLATE
- ;;
- \?)
- echo "Invalid option: -$OPTARG" >&2
- ;;
- esac
-done
-
-if [ "$SEQ" = "-1" ]; then
- SEQ=0
-fi
-
-SHORT_HOSTNAME=$(hostname -s)
-case $SEQ in
- 2)
- OUTFILE=${PREFIX}$SHORT_HOSTNAME-$BLOCK_SIZE-all-$MODES-$PRINTABLE_DISKS.fio
- ;;
-
- 1)
- OUTFILE=${PREFIX}$SHORT_HOSTNAME-$BLOCK_SIZE-sequential-$MODES-$PRINTABLE_DISKS.fio
- ;;
- 0)
- OUTFILE=${PREFIX}$SHORT_HOSTNAME-$BLOCK_SIZE-parallel-$MODES-$PRINTABLE_DISKS.fio
- ;;
-esac
-
-if [ -z "$DISKS" ]; then
- echo "Missing DISKS !"
- echo "Please read the help !"
- show_help
- exit 1
-fi
-
-}
-
-check_mode_order() {
-FOUND_WRITE="NO"
-CAUSE="You are reading data before writing them "
-
-# If no write occurs, let's show a different message
-echo $MODES | grep -q "write"
-if [ "$?" -ne 0 ]; then
- CAUSE="You are reading data while never wrote them before"
-fi
-
-for mode in $(echo $MODES | tr "," " "); do
- echo $mode | grep -q write
- if [ "$?" -eq 0 ]; then
- FOUND_WRITE="YES"
- fi
- echo $mode | grep -q "read"
- if [ "$?" -eq 0 ]; then
- if [ "$FOUND_WRITE" = "NO" ]; then
- echo "###############################################################"
- echo "# Warning : $CAUSE#"
- echo "# On some storage devices, this could lead to invalid results #"
- echo "# #"
- echo "# Press Ctrl-C to adjust pattern order if you have doubts #"
- echo "# Or Wait 5 seconds before the file will be created #"
- echo "###############################################################"
- sleep 5
- # No need to try showing the message more than one time
- return
- fi
- fi
-done
-}
-
-
-########## MAIN
-gen_template
-parse_cmdline "$@"
-finish_template
-check_mode_order
-
-echo "Generating $OUTFILE"
-cp -f $TEMPLATE $OUTFILE
-echo >> $OUTFILE
-
-for BLK_SIZE in $(echo $BLOCK_SIZE | tr "," " "); do
- gen_fio
-done
-ETA_H=$(($ETA / 3600))
-ETA_M=$((($ETA - ($ETA_H*3600)) / 60))
-if [ "$ETA" = "0" ]; then
- echo "Cannot estimate ETA as RUNTIME=0"
-else
- echo "Estimated Time = $ETA seconds : $ETA_H hour $ETA_M minutes"
-fi
diff --git a/tools/hist/.gitignore b/tools/hist/.gitignore
deleted file mode 100644
index 4f875dac..00000000
--- a/tools/hist/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-*.pyc
-*.ipynb
-.ipynb_checkpoints
diff --git a/tools/hist/fiologparser_hist.py b/tools/hist/fiologparser_hist.py
deleted file mode 100755
index ead5e543..00000000
--- a/tools/hist/fiologparser_hist.py
+++ /dev/null
@@ -1,388 +0,0 @@
-#!/usr/bin/env python2.7
-"""
- Utility for converting *_clat_hist* files generated by fio into latency statistics.
-
- Example usage:
-
- $ fiologparser_hist.py *_clat_hist*
- end-time, samples, min, avg, median, 90%, 95%, 99%, max
- 1000, 15, 192, 1678.107, 1788.859, 1856.076, 1880.040, 1899.208, 1888.000
- 2000, 43, 152, 1642.368, 1714.099, 1816.659, 1845.552, 1888.131, 1888.000
- 4000, 39, 1152, 1546.962, 1545.785, 1627.192, 1640.019, 1691.204, 1744
- ...
-
- @author Karl Cronburg <karl.cronburg@gmail.com>
-"""
-import os
-import sys
-import pandas
-import numpy as np
-
-err = sys.stderr.write
-
-def weighted_percentile(percs, vs, ws):
- """ Use linear interpolation to calculate the weighted percentile.
-
- Value and weight arrays are first sorted by value. The cumulative
- distribution function (cdf) is then computed, after which np.interp
- finds the two values closest to our desired weighted percentile(s)
- and linearly interpolates them.
-
- percs :: List of percentiles we want to calculate
- vs :: Array of values we are computing the percentile of
- ws :: Array of weights for our corresponding values
- return :: Array of percentiles
- """
- idx = np.argsort(vs)
- vs, ws = vs[idx], ws[idx] # weights and values sorted by value
- cdf = 100 * (ws.cumsum() - ws / 2.0) / ws.sum()
- return np.interp(percs, cdf, vs) # linear interpolation
-
-def weights(start_ts, end_ts, start, end):
- """ Calculate weights based on fraction of sample falling in the
- given interval [start,end]. Weights computed using vector / array
- computation instead of for-loops.
-
- Note that samples with zero time length are effectively ignored
- (we set their weight to zero).
-
- start_ts :: Array of start times for a set of samples
- end_ts :: Array of end times for a set of samples
- start :: int
- end :: int
- return :: Array of weights
- """
- sbounds = np.maximum(start_ts, start).astype(float)
- ebounds = np.minimum(end_ts, end).astype(float)
- ws = (ebounds - sbounds) / (end_ts - start_ts)
- if np.any(np.isnan(ws)):
- err("WARNING: zero-length sample(s) detected. Log file corrupt"
- " / bad time values? Ignoring these samples.\n")
- ws[np.where(np.isnan(ws))] = 0.0;
- return ws
-
-def weighted_average(vs, ws):
- return np.sum(vs * ws) / np.sum(ws)
-
-columns = ["end-time", "samples", "min", "avg", "median", "90%", "95%", "99%", "max"]
-percs = [50, 90, 95, 99]
-
-def fmt_float_list(ctx, num=1):
- """ Return a comma separated list of float formatters to the required number
- of decimal places. For instance:
-
- fmt_float_list(ctx.decimals=4, num=3) == "%.4f, %.4f, %.4f"
- """
- return ', '.join(["%%.%df" % ctx.decimals] * num)
-
-# Default values - see beginning of main() for how we detect number columns in
-# the input files:
-__HIST_COLUMNS = 1216
-__NON_HIST_COLUMNS = 3
-__TOTAL_COLUMNS = __HIST_COLUMNS + __NON_HIST_COLUMNS
-
-def read_chunk(rdr, sz):
- """ Read the next chunk of size sz from the given reader. """
- try:
- """ StopIteration occurs when the pandas reader is empty, and AttributeError
- occurs if rdr is None due to the file being empty. """
- new_arr = rdr.read().values
- except (StopIteration, AttributeError):
- return None
-
- """ Extract array of just the times, and histograms matrix without times column. """
- times, rws, szs = new_arr[:,0], new_arr[:,1], new_arr[:,2]
- hists = new_arr[:,__NON_HIST_COLUMNS:]
- times = times.reshape((len(times),1))
- arr = np.append(times, hists, axis=1)
-
- return arr
-
-def get_min(fps, arrs):
- """ Find the file with the current first row with the smallest start time """
- return min([fp for fp in fps if not arrs[fp] is None], key=lambda fp: arrs.get(fp)[0][0])
-
-def histogram_generator(ctx, fps, sz):
-
- # Create a chunked pandas reader for each of the files:
- rdrs = {}
- for fp in fps:
- try:
- rdrs[fp] = pandas.read_csv(fp, dtype=int, header=None, chunksize=sz)
- except ValueError as e:
- if e.message == 'No columns to parse from file':
- if ctx.warn: sys.stderr.write("WARNING: Empty input file encountered.\n")
- rdrs[fp] = None
- else:
- raise(e)
-
- # Initial histograms from disk:
- arrs = {fp: read_chunk(rdr, sz) for fp,rdr in rdrs.items()}
- while True:
-
- try:
- """ ValueError occurs when nothing more to read """
- fp = get_min(fps, arrs)
- except ValueError:
- return
- arr = arrs[fp]
- yield np.insert(arr[0], 1, fps.index(fp))
- arrs[fp] = arr[1:]
-
- if arrs[fp].shape[0] == 0:
- arrs[fp] = read_chunk(rdrs[fp], sz)
-
-def _plat_idx_to_val(idx, edge=0.5, FIO_IO_U_PLAT_BITS=6, FIO_IO_U_PLAT_VAL=64):
- """ Taken from fio's stat.c for calculating the latency value of a bin
- from that bin's index.
-
- idx : the value of the index into the histogram bins
- edge : fractional value in the range [0,1]** indicating how far into
- the bin we wish to compute the latency value of.
-
- ** edge = 0.0 and 1.0 computes the lower and upper latency bounds
- respectively of the given bin index. """
-
- # MSB <= (FIO_IO_U_PLAT_BITS-1), cannot be rounded off. Use
- # all bits of the sample as index
- if (idx < (FIO_IO_U_PLAT_VAL << 1)):
- return idx
-
- # Find the group and compute the minimum value of that group
- error_bits = (idx >> FIO_IO_U_PLAT_BITS) - 1
- base = 1 << (error_bits + FIO_IO_U_PLAT_BITS)
-
- # Find its bucket number of the group
- k = idx % FIO_IO_U_PLAT_VAL
-
- # Return the mean (if edge=0.5) of the range of the bucket
- return base + ((k + edge) * (1 << error_bits))
-
-def plat_idx_to_val_coarse(idx, coarseness, edge=0.5):
- """ Converts the given *coarse* index into a non-coarse index as used by fio
- in stat.h:plat_idx_to_val(), subsequently computing the appropriate
- latency value for that bin.
- """
-
- # Multiply the index by the power of 2 coarseness to get the bin
- # bin index with a max of 1536 bins (FIO_IO_U_PLAT_GROUP_NR = 24 in stat.h)
- stride = 1 << coarseness
- idx = idx * stride
- lower = _plat_idx_to_val(idx, edge=0.0)
- upper = _plat_idx_to_val(idx + stride, edge=1.0)
- return lower + (upper - lower) * edge
-
-def print_all_stats(ctx, end, mn, ss_cnt, vs, ws, mx):
- ps = weighted_percentile(percs, vs, ws)
-
- avg = weighted_average(vs, ws)
- values = [mn, avg] + list(ps) + [mx]
- row = [end, ss_cnt] + map(lambda x: float(x) / ctx.divisor, values)
- fmt = "%d, %d, %d, " + fmt_float_list(ctx, 5) + ", %d"
- print (fmt % tuple(row))
-
-def update_extreme(val, fncn, new_val):
- """ Calculate min / max in the presence of None values """
- if val is None: return new_val
- else: return fncn(val, new_val)
-
-# See beginning of main() for how bin_vals are computed
-bin_vals = []
-lower_bin_vals = [] # lower edge of each bin
-upper_bin_vals = [] # upper edge of each bin
-
-def process_interval(ctx, samples, iStart, iEnd):
- """ Construct the weighted histogram for the given interval by scanning
- through all the histograms and figuring out which of their bins have
- samples with latencies which overlap with the given interval
- [iStart,iEnd].
- """
-
- times, files, hists = samples[:,0], samples[:,1], samples[:,2:]
- iHist = np.zeros(__HIST_COLUMNS)
- ss_cnt = 0 # number of samples affecting this interval
- mn_bin_val, mx_bin_val = None, None
-
- for end_time,file,hist in zip(times,files,hists):
-
- # Only look at bins of the current histogram sample which
- # started before the end of the current time interval [start,end]
- start_times = (end_time - 0.5 * ctx.interval) - bin_vals / 1000.0
- idx = np.where(start_times < iEnd)
- s_ts, l_bvs, u_bvs, hs = start_times[idx], lower_bin_vals[idx], upper_bin_vals[idx], hist[idx]
-
- # Increment current interval histogram by weighted values of future histogram:
- ws = hs * weights(s_ts, end_time, iStart, iEnd)
- iHist[idx] += ws
-
- # Update total number of samples affecting current interval histogram:
- ss_cnt += np.sum(hs)
-
- # Update min and max bin values seen if necessary:
- idx = np.where(hs != 0)[0]
- if idx.size > 0:
- mn_bin_val = update_extreme(mn_bin_val, min, l_bvs[max(0, idx[0] - 1)])
- mx_bin_val = update_extreme(mx_bin_val, max, u_bvs[min(len(hs) - 1, idx[-1] + 1)])
-
- if ss_cnt > 0: print_all_stats(ctx, iEnd, mn_bin_val, ss_cnt, bin_vals, iHist, mx_bin_val)
-
-def guess_max_from_bins(ctx, hist_cols):
- """ Try to guess the GROUP_NR from given # of histogram
- columns seen in an input file """
- max_coarse = 8
- if ctx.group_nr < 19 or ctx.group_nr > 26:
- bins = [ctx.group_nr * (1 << 6)]
- else:
- bins = [1216,1280,1344,1408,1472,1536,1600,1664]
- coarses = range(max_coarse + 1)
- fncn = lambda z: list(map(lambda x: z/2**x if z % 2**x == 0 else -10, coarses))
-
- arr = np.transpose(list(map(fncn, bins)))
- idx = np.where(arr == hist_cols)
- if len(idx[1]) == 0:
- table = repr(arr.astype(int)).replace('-10', 'N/A').replace('array',' ')
- err("Unable to determine bin values from input clat_hist files. Namely \n"
- "the first line of file '%s' " % ctx.FILE[0] + "has %d \n" % (__TOTAL_COLUMNS,) +
- "columns of which we assume %d " % (hist_cols,) + "correspond to histogram bins. \n"
- "This number needs to be equal to one of the following numbers:\n\n"
- + table + "\n\n"
- "Possible reasons and corresponding solutions:\n"
- " - Input file(s) does not contain histograms.\n"
- " - You recompiled fio with a different GROUP_NR. If so please specify this\n"
- " new GROUP_NR on the command line with --group_nr\n")
- exit(1)
- return bins[idx[1][0]]
-
-def main(ctx):
-
- if ctx.job_file:
- try:
- from configparser import SafeConfigParser, NoOptionError
- except ImportError:
- from ConfigParser import SafeConfigParser, NoOptionError
-
- cp = SafeConfigParser(allow_no_value=True)
- with open(ctx.job_file, 'r') as fp:
- cp.readfp(fp)
-
- if ctx.interval is None:
- # Auto detect --interval value
- for s in cp.sections():
- try:
- hist_msec = cp.get(s, 'log_hist_msec')
- if hist_msec is not None:
- ctx.interval = int(hist_msec)
- except NoOptionError:
- pass
-
- if ctx.interval is None:
- ctx.interval = 1000
-
- # Automatically detect how many columns are in the input files,
- # calculate the corresponding 'coarseness' parameter used to generate
- # those files, and calculate the appropriate bin latency values:
- with open(ctx.FILE[0], 'r') as fp:
- global bin_vals,lower_bin_vals,upper_bin_vals,__HIST_COLUMNS,__TOTAL_COLUMNS
- __TOTAL_COLUMNS = len(fp.readline().split(','))
- __HIST_COLUMNS = __TOTAL_COLUMNS - __NON_HIST_COLUMNS
-
- max_cols = guess_max_from_bins(ctx, __HIST_COLUMNS)
- coarseness = int(np.log2(float(max_cols) / __HIST_COLUMNS))
- bin_vals = np.array(map(lambda x: plat_idx_to_val_coarse(x, coarseness), np.arange(__HIST_COLUMNS)), dtype=float)
- lower_bin_vals = np.array(map(lambda x: plat_idx_to_val_coarse(x, coarseness, 0.0), np.arange(__HIST_COLUMNS)), dtype=float)
- upper_bin_vals = np.array(map(lambda x: plat_idx_to_val_coarse(x, coarseness, 1.0), np.arange(__HIST_COLUMNS)), dtype=float)
-
- fps = [open(f, 'r') for f in ctx.FILE]
- gen = histogram_generator(ctx, fps, ctx.buff_size)
-
- print(', '.join(columns))
-
- try:
- start, end = 0, ctx.interval
- arr = np.empty(shape=(0,__TOTAL_COLUMNS - 1))
- more_data = True
- while more_data or len(arr) > 0:
-
- # Read up to ctx.max_latency (default 20 seconds) of data from end of current interval.
- while len(arr) == 0 or arr[-1][0] < ctx.max_latency * 1000 + end:
- try:
- new_arr = next(gen)
- except StopIteration:
- more_data = False
- break
- arr = np.append(arr, new_arr.reshape((1,__TOTAL_COLUMNS - 1)), axis=0)
- arr = arr.astype(int)
-
- if arr.size > 0:
- # Jump immediately to the start of the input, rounding
- # down to the nearest multiple of the interval (useful when --log_unix_epoch
- # was used to create these histograms):
- if start == 0 and arr[0][0] - ctx.max_latency > end:
- start = arr[0][0] - ctx.max_latency
- start = start - (start % ctx.interval)
- end = start + ctx.interval
-
- process_interval(ctx, arr, start, end)
-
- # Update arr to throw away samples we no longer need - samples which
- # end before the start of the next interval, i.e. the end of the
- # current interval:
- idx = np.where(arr[:,0] > end)
- arr = arr[idx]
-
- start += ctx.interval
- end = start + ctx.interval
- finally:
- map(lambda f: f.close(), fps)
-
-
-if __name__ == '__main__':
- import argparse
- p = argparse.ArgumentParser()
- arg = p.add_argument
- arg("FILE", help='space separated list of latency log filenames', nargs='+')
- arg('--buff_size',
- default=10000,
- type=int,
- help='number of samples to buffer into numpy at a time')
-
- arg('--max_latency',
- default=20,
- type=float,
- help='number of seconds of data to process at a time')
-
- arg('-i', '--interval',
- type=int,
- help='interval width (ms), default 1000 ms')
-
- arg('-d', '--divisor',
- required=False,
- type=int,
- default=1,
- help='divide the results by this value.')
-
- arg('--decimals',
- default=3,
- type=int,
- help='number of decimal places to print floats to')
-
- arg('--warn',
- dest='warn',
- action='store_true',
- default=False,
- help='print warning messages to stderr')
-
- arg('--group_nr',
- default=19,
- type=int,
- help='FIO_IO_U_PLAT_GROUP_NR as defined in stat.h')
-
- arg('--job-file',
- default=None,
- type=str,
- help='Optional argument pointing to the job file used to create the '
- 'given histogram files. Useful for auto-detecting --log_hist_msec and '
- '--log_unix_epoch (in fio) values.')
-
- main(p.parse_args())
-
diff --git a/tools/hist/fiologparser_hist.py.1 b/tools/hist/fiologparser_hist.py.1
deleted file mode 100644
index ed22c747..00000000
--- a/tools/hist/fiologparser_hist.py.1
+++ /dev/null
@@ -1,201 +0,0 @@
-.TH fiologparser_hist.py 1 "August 18, 2016"
-.SH NAME
-fiologparser_hist.py \- Calculate statistics from fio histograms
-.SH SYNOPSIS
-.B fiologparser_hist.py
-[\fIoptions\fR] [clat_hist_files]...
-.SH DESCRIPTION
-.B fiologparser_hist.py
-is a utility for converting *_clat_hist* files
-generated by fio into a CSV of latency statistics including minimum,
-average, maximum latency, and 50th, 95th, and 99th percentiles.
-.SH EXAMPLES
-.PP
-.nf
-$ fiologparser_hist.py *_clat_hist*
-end-time, samples, min, avg, median, 90%, 95%, 99%, max
-1000, 15, 192, 1678.107, 1788.859, 1856.076, 1880.040, 1899.208, 1888.000
-2000, 43, 152, 1642.368, 1714.099, 1816.659, 1845.552, 1888.131, 1888.000
-4000, 39, 1152, 1546.962, 1545.785, 1627.192, 1640.019, 1691.204, 1744
-...
-.fi
-.PP
-
-.SH OPTIONS
-.TP
-.BR \-\-help
-Print these options.
-.TP
-.BR \-\-buff_size \fR=\fPint
-Number of samples to buffer into numpy at a time. Default is 10,000.
-This can be adjusted to help performance.
-.TP
-.BR \-\-max_latency \fR=\fPint
-Number of seconds of data to process at a time. Defaults to 20 seconds,
-in order to handle the 17 second upper bound on latency in histograms
-reported by fio. This should be increased if fio has been
-run with a larger maximum latency. Lowering this when a lower maximum
-latency is known can improve performance. See NOTES for more details.
-.TP
-.BR \-i ", " \-\-interval \fR=\fPint
-Interval at which statistics are reported. Defaults to 1000 ms. This
-should be set a minimum of the value for \fBlog_hist_msec\fR as given
-to fio.
-.TP
-.BR \-d ", " \-\-divisor \fR=\fPint
-Divide statistics by this value. Defaults to 1. Useful if you want to
-convert latencies from milliseconds to seconds (\fBdivisor\fR=\fP1000\fR).
-.TP
-.BR \-\-warn
-Enables warning messages printed to stderr, useful for debugging.
-.TP
-.BR \-\-group_nr \fR=\fPint
-Set this to the value of \fIFIO_IO_U_PLAT_GROUP_NR\fR as defined in
-\fPstat.h\fR if fio has been recompiled. Defaults to 19, the
-current value used in fio. See NOTES for more details.
-
-.SH NOTES
-end-times are calculated to be uniform increments of the \fB\-\-interval\fR value given,
-regardless of when histogram samples are reported. Of note:
-
-.RS
-Intervals with no samples are omitted. In the example above this means
-"no statistics from 2 to 3 seconds" and "39 samples influenced the statistics
-of the interval from 3 to 4 seconds".
-.LP
-Intervals with a single sample will have the same value for all statistics
-.RE
-
-.PP
-The number of samples is unweighted, corresponding to the total number of samples
-which have any effect whatsoever on the interval.
-
-Min statistics are computed using value of the lower boundary of the first bin
-(in increasing bin order) with non-zero samples in it. Similarly for max,
-we take the upper boundary of the last bin with non-zero samples in it.
-This is semantically identical to taking the 0th and 100th percentiles with a
-50% bin-width buffer (because percentiles are computed using mid-points of
-the bins). This enforces the following nice properties:
-
-.RS
-min <= 50th <= 90th <= 95th <= 99th <= max
-.LP
-min and max are strict lower and upper bounds on the actual
-min / max seen by fio (and reported in *_clat.* with averaging turned off).
-.RE
-
-.PP
-Average statistics use a standard weighted arithmetic mean.
-
-Percentile statistics are computed using the weighted percentile method as
-described here: \fIhttps://en.wikipedia.org/wiki/Percentile#Weighted_percentile\fR.
-See weights() method for details on how weights are computed for individual
-samples. In process_interval() we further multiply by the height of each bin
-to get weighted histograms.
-
-We convert files given on the command line, assumed to be fio histogram files,
-An individual histogram file can contain the
-histograms for multiple different r/w directions (notably when \fB\-\-rw\fR=\fPrandrw\fR). This
-is accounted for by tracking each r/w direction separately. In the statistics
-reported we ultimately merge *all* histograms (regardless of r/w direction).
-
-The value of *_GROUP_NR in \fIstat.h\fR (and *_BITS) determines how many latency bins
-fio outputs when histogramming is enabled. Namely for the current default of
-GROUP_NR=19, we get 1,216 bins with a maximum latency of approximately 17
-seconds. For certain applications this may not be sufficient. With GROUP_NR=24
-we have 1,536 bins, giving us a maximum latency of 541 seconds (~ 9 minutes). If
-you expect your application to experience latencies greater than 17 seconds,
-you will need to recompile fio with a larger GROUP_NR, e.g. with:
-
-.RS
-.PP
-.nf
-sed -i.bak 's/^#define FIO_IO_U_PLAT_GROUP_NR 19\n/#define FIO_IO_U_PLAT_GROUP_NR 24/g' stat.h
-make fio
-.fi
-.PP
-.RE
-
-.PP
-Quick reference table for the max latency corresponding to a sampling of
-values for GROUP_NR:
-
-.RS
-.PP
-.nf
-GROUP_NR | # bins | max latency bin value
-19 | 1216 | 16.9 sec
-20 | 1280 | 33.8 sec
-21 | 1344 | 67.6 sec
-22 | 1408 | 2 min, 15 sec
-23 | 1472 | 4 min, 32 sec
-24 | 1536 | 9 min, 4 sec
-25 | 1600 | 18 min, 8 sec
-26 | 1664 | 36 min, 16 sec
-.fi
-.PP
-.RE
-
-.PP
-At present this program automatically detects the number of histogram bins in
-the log files, and adjusts the bin latency values accordingly. In particular if
-you use the \fB\-\-log_hist_coarseness\fR parameter of fio, you get output files with
-a number of bins according to the following table (note that the first
-row is identical to the table above):
-
-.RS
-.PP
-.nf
-coarse \\ GROUP_NR
- 19 20 21 22 23 24 25 26
- -------------------------------------------------------
- 0 [[ 1216, 1280, 1344, 1408, 1472, 1536, 1600, 1664],
- 1 [ 608, 640, 672, 704, 736, 768, 800, 832],
- 2 [ 304, 320, 336, 352, 368, 384, 400, 416],
- 3 [ 152, 160, 168, 176, 184, 192, 200, 208],
- 4 [ 76, 80, 84, 88, 92, 96, 100, 104],
- 5 [ 38, 40, 42, 44, 46, 48, 50, 52],
- 6 [ 19, 20, 21, 22, 23, 24, 25, 26],
- 7 [ N/A, 10, N/A, 11, N/A, 12, N/A, 13],
- 8 [ N/A, 5, N/A, N/A, N/A, 6, N/A, N/A]]
-.fi
-.PP
-.RE
-
-.PP
-For other values of GROUP_NR and coarseness, this table can be computed like this:
-
-.RS
-.PP
-.nf
-bins = [1216,1280,1344,1408,1472,1536,1600,1664]
-max_coarse = 8
-fncn = lambda z: list(map(lambda x: z/2**x if z % 2**x == 0 else nan, range(max_coarse + 1)))
-np.transpose(list(map(fncn, bins)))
-.fi
-.PP
-.RE
-
-.PP
-If you have not adjusted GROUP_NR for your (high latency) application, then you
-will see the percentiles computed by this tool max out at the max latency bin
-value as in the first table above, and in this plot (where GROUP_NR=19 and thus we see
-a max latency of ~16.7 seconds in the red line):
-
-.RS
-\fIhttps://www.cronburg.com/fio/max_latency_bin_value_bug.png
-.RE
-
-.PP
-Motivation for, design decisions, and the implementation process are
-described in further detail here:
-
-.RS
-\fIhttps://www.cronburg.com/fio/cloud-latency-problem-measurement/
-.RE
-
-.SH AUTHOR
-.B fiologparser_hist.py
-and this manual page were written by Karl Cronburg <karl.cronburg@gmail.com>.
-.SH "REPORTING BUGS"
-Report bugs to the \fBfio\fR mailing list <fio@vger.kernel.org>.
diff --git a/tools/hist/half-bins.py b/tools/hist/half-bins.py
deleted file mode 100755
index d592af00..00000000
--- a/tools/hist/half-bins.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python2.7
-""" Cut the number bins in half in fio histogram output. Example usage:
-
- $ half-bins.py -c 2 output_clat_hist.1.log > smaller_clat_hist.1.log
-
- Which merges e.g. bins [0 .. 3], [4 .. 7], ..., [1212 .. 1215] resulting in
- 304 = 1216 / (2**2) merged bins per histogram sample.
-
- @author Karl Cronburg <karl.cronburg@gmail.com>
-"""
-import sys
-
-def main(ctx):
- stride = 1 << ctx.coarseness
- with open(ctx.FILENAME, 'r') as fp:
- for line in fp.readlines():
- vals = line.split(', ')
- sys.stdout.write("%s, %s, %s, " % tuple(vals[:3]))
-
- hist = list(map(int, vals[3:]))
- for i in range(0, len(hist) - stride, stride):
- sys.stdout.write("%d, " % sum(hist[i : i + stride],))
- sys.stdout.write("%d\n" % sum(hist[len(hist) - stride:]))
-
-if __name__ == '__main__':
- import argparse
- p = argparse.ArgumentParser()
- arg = p.add_argument
- arg( 'FILENAME', help='clat_hist file for which we will reduce'
- ' (by half or more) the number of bins.')
- arg('-c', '--coarseness',
- default=1,
- type=int,
- help='number of times to reduce number of bins by half, '
- 'e.g. coarseness of 4 merges each 2^4 = 16 consecutive '
- 'bins.')
- main(p.parse_args())
-
diff --git a/tools/plot/fio2gnuplot b/tools/plot/fio2gnuplot
deleted file mode 100755
index a703ae33..00000000
--- a/tools/plot/fio2gnuplot
+++ /dev/null
@@ -1,527 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
-# Author: Erwan Velu <erwan@enovance.com>
-#
-# The license below covers all files distributed with fio unless otherwise
-# noted in the file itself.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-import os
-import fnmatch
-import sys
-import getopt
-import re
-import math
-import shutil
-
-def find_file(path, pattern):
- fio_data_file=[]
- # For all the local files
- for file in os.listdir(path):
- # If the file matches the glob
- if fnmatch.fnmatch(file, pattern):
- # Let's consider this file
- fio_data_file.append(file)
-
- return fio_data_file
-
-def generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir):
- if verbose: print "Generating rendering scripts"
- filename=gnuplot_output_dir+'mygraph'
- temporary_files.append(filename)
- f=open(filename,'w')
-
- # Plotting 3D or comparing graphs doesn't have a meaning unless if there is at least 2 traces
- if len(fio_data_file) > 1:
- f.write("call \'%s/graph3D.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\'\n" % (gpm_dir,title,gnuplot_output_filename,gnuplot_output_filename,mode))
-
- # Setting up the compare files that will be plot later
- compare=open(gnuplot_output_dir + 'compare.gnuplot','w')
- compare.write('''
-set title '%s'
-set terminal png size 1280,1024
-set ytics axis out auto
-set key top left reverse
-set xlabel "Time (Seconds)"
-set ylabel '%s'
-set yrange [0:]
-set style line 1 lt 1 lw 3 pt 3 linecolor rgb "green"
-'''% (title,mode))
- compare.close()
- #Copying the common file for all kind of graph (raw/smooth/trend)
- compare_raw_filename="compare-%s-2Draw" % (gnuplot_output_filename)
- compare_smooth_filename="compare-%s-2Dsmooth" % (gnuplot_output_filename)
- compare_trend_filename="compare-%s-2Dtrend" % (gnuplot_output_filename)
-
- shutil.copy(gnuplot_output_dir+'compare.gnuplot',gnuplot_output_dir+compare_raw_filename+".gnuplot")
- shutil.copy(gnuplot_output_dir+'compare.gnuplot',gnuplot_output_dir+compare_smooth_filename+".gnuplot")
- shutil.copy(gnuplot_output_dir+'compare.gnuplot',gnuplot_output_dir+compare_trend_filename+".gnuplot")
- temporary_files.append(gnuplot_output_dir+compare_raw_filename+".gnuplot")
- temporary_files.append(gnuplot_output_dir+compare_smooth_filename+".gnuplot")
- temporary_files.append(gnuplot_output_dir+compare_trend_filename+".gnuplot")
-
- #Setting up a different output filename for each kind of graph
- compare_raw=open(gnuplot_output_dir+compare_raw_filename + ".gnuplot",'a')
- compare_raw.write("set output '%s.png'\n" % compare_raw_filename)
- compare_smooth=open(gnuplot_output_dir+compare_smooth_filename+".gnuplot",'a')
- compare_smooth.write("set output '%s.png'\n" % compare_smooth_filename)
- compare_trend=open(gnuplot_output_dir+compare_trend_filename+".gnuplot",'a')
- compare_trend.write("set output '%s.png'\n" % compare_trend_filename)
-
- # Let's plot the average value for all the traces
- global_disk_perf = sum(disk_perf, [])
- global_avg = average(global_disk_perf)
- compare_raw.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg));
- compare_smooth.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg));
- compare_trend.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg));
-
- pos=0
- # Let's create a temporary file for each selected fio file
- for file in fio_data_file:
- tmp_filename = "gnuplot_temp_file.%d" % pos
-
- # Plotting comparing graphs doesn't have a meaning unless if there is at least 2 traces
- if len(fio_data_file) > 1:
- # Adding the plot instruction for each kind of comparing graphs
- compare_raw.write(",\\\n'%s' using 2:3 with linespoints title '%s'" % (tmp_filename,fio_data_file[pos]))
- compare_smooth.write(",\\\n'%s' using 2:3 smooth csplines title '%s'" % (tmp_filename,fio_data_file[pos]))
- compare_trend.write(",\\\n'%s' using 2:3 smooth bezier title '%s'" % (tmp_filename,fio_data_file[pos]))
-
- png_file=file.replace('.log','')
- raw_filename = "%s-2Draw" % (png_file)
- smooth_filename = "%s-2Dsmooth" % (png_file)
- trend_filename = "%s-2Dtrend" % (png_file)
- avg = average(disk_perf[pos])
- f.write("call \'%s/graph2D.gpm\' \'%s' \'%s\' \'%s\' \'%s\' \'%s\' \'%s\' \'%s\' \'%f\'\n" % (gpm_dir,title,tmp_filename,fio_data_file[pos],raw_filename,mode,smooth_filename,trend_filename,avg))
- pos = pos +1
-
- # Plotting comparing graphs doesn't have a meaning unless if there is at least 2 traces
- if len(fio_data_file) > 1:
- os.remove(gnuplot_output_dir+"compare.gnuplot")
- compare_raw.close()
- compare_smooth.close()
- compare_trend.close()
- f.close()
-
-def generate_gnuplot_math_script(title,gnuplot_output_filename,mode,average,gnuplot_output_dir,gpm_dir):
- filename=gnuplot_output_dir+'mymath';
- temporary_files.append(filename)
- f=open(filename,'a')
- f.write("call \'%s/math.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\' %s\n" % (gpm_dir,title,gnuplot_output_filename,gnuplot_output_filename,mode,average))
- f.close()
-
-def compute_aggregated_file(fio_data_file, gnuplot_output_filename, gnuplot_output_dir):
- if verbose: print "Processing data file 2/2"
- temp_files=[]
- pos=0
-
- # Let's create a temporary file for each selected fio file
- for file in fio_data_file:
- tmp_filename = "%sgnuplot_temp_file.%d" % (gnuplot_output_dir, pos)
- temp_files.append(open(tmp_filename,'r'))
- pos = pos +1
-
- f = open(gnuplot_output_dir+gnuplot_output_filename, "w")
- temporary_files.append(gnuplot_output_dir+gnuplot_output_filename)
- index=0
- # Let's add some information
- for tempfile in temp_files:
- f.write("# Disk%d was coming from %s\n" % (index,fio_data_file[index]))
- f.write(tempfile.read())
- f.write("\n")
- tempfile.close()
- index = index + 1
- f.close()
-
-def average(s): return sum(s) * 1.0 / len(s)
-
-def compute_temp_file(fio_data_file,disk_perf,gnuplot_output_dir, min_time, max_time):
- end_time=max_time
- if end_time == -1:
- end_time="infinite"
- if verbose: print "Processing data file 1/2 with %s<time<%s" % (min_time,end_time)
- files=[]
- temp_outfile=[]
- blk_size=0
- for file in fio_data_file:
- files.append(open(file))
- pos = len(files) - 1
- tmp_filename = "%sgnuplot_temp_file.%d" % (gnuplot_output_dir,pos)
- temporary_files.append(tmp_filename)
- gnuplot_file=open(tmp_filename,'w')
- temp_outfile.append(gnuplot_file)
- gnuplot_file.write("#Temporary file based on file %s\n" % file)
- disk_perf.append([])
-
- shall_break = False
- while True:
- current_line=[]
- nb_empty_files=0
- nb_files=len(files)
- for myfile in files:
- s=myfile.readline().replace(',',' ').split()
- if not s:
- nb_empty_files+=1
- s="-1, 0, 0, 0".replace(',',' ').split()
-
- if (nb_empty_files == nb_files):
- shall_break=True
- break;
-
- current_line.append(s);
-
- if shall_break == True:
- break
-
- last_time = -1
- index=-1
- perfs=[]
- for line in enumerate(current_line):
- # Index will be used to remember what file was featuring what value
- index=index+1
-
- time, perf, x, block_size = line[1]
- if (blk_size == 0):
- try:
- blk_size=int(block_size)
- except:
- print "Error while reading the following line :"
- print line
- sys.exit(1);
-
- # We ignore the first 500msec as it doesn't seems to be part of the real benchmark
- # Time < 500 usually reports BW=0 breaking the min computing
- if (min_time == 0):
- min_time==0.5
-
- # Then we estimate if the data we got is part of the time range we want to plot
- if ((float(time)>(float(min_time)*1000)) and ((int(time) < (int(max_time)*1000)) or max_time==-1)):
- disk_perf[index].append(int(perf))
- perfs.append("%d %s %s"% (index, time, perf))
-
- # If we reach this point, it means that all the traces are coherent
- for p in enumerate(perfs):
- index, perf_time,perf = p[1].split()
- temp_outfile[int(index)].write("%s %.2f %s\n" % (index, float(float(perf_time)/1000), perf))
-
-
- for file in files:
- file.close()
- for file in temp_outfile:
- file.close()
- return blk_size
-
-def compute_math(fio_data_file, title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir):
- if verbose: print "Computing Maths"
- global_min=[]
- global_max=[]
- average_file=open(gnuplot_output_dir+gnuplot_output_filename+'.average', 'w')
- min_file=open(gnuplot_output_dir+gnuplot_output_filename+'.min', 'w')
- max_file=open(gnuplot_output_dir+gnuplot_output_filename+'.max', 'w')
- stddev_file=open(gnuplot_output_dir+gnuplot_output_filename+'.stddev', 'w')
- global_file=open(gnuplot_output_dir+gnuplot_output_filename+'.global','w')
- temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.average')
- temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.min')
- temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.max')
- temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.stddev')
- temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.global')
-
- min_file.write('DiskName %s\n' % mode)
- max_file.write('DiskName %s\n'% mode)
- average_file.write('DiskName %s\n'% mode)
- stddev_file.write('DiskName %s\n'% mode )
- for disk in xrange(len(fio_data_file)):
-# print disk_perf[disk]
- min_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk]))
- max_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk]))
- average_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk]))
- stddev_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk]))
- avg = average(disk_perf[disk])
- variance = map(lambda x: (x - avg)**2, disk_perf[disk])
- standard_deviation = math.sqrt(average(variance))
-# print "Disk%d [ min=%.2f max=%.2f avg=%.2f stddev=%.2f \n" % (disk,min(disk_perf[disk]),max(disk_perf[disk]),avg, standard_deviation)
- average_file.write('%d %d\n' % (disk, avg))
- stddev_file.write('%d %d\n' % (disk, standard_deviation))
- local_min=min(disk_perf[disk])
- local_max=max(disk_perf[disk])
- min_file.write('%d %d\n' % (disk, local_min))
- max_file.write('%d %d\n' % (disk, local_max))
- global_min.append(int(local_min))
- global_max.append(int(local_max))
-
- global_disk_perf = sum(disk_perf, [])
- avg = average(global_disk_perf)
- variance = map(lambda x: (x - avg)**2, global_disk_perf)
- standard_deviation = math.sqrt(average(variance))
-
- global_file.write('min=%.2f\n' % min(global_disk_perf))
- global_file.write('max=%.2f\n' % max(global_disk_perf))
- global_file.write('avg=%.2f\n' % avg)
- global_file.write('stddev=%.2f\n' % standard_deviation)
- global_file.write('values_count=%d\n' % len(global_disk_perf))
- global_file.write('disks_count=%d\n' % len(fio_data_file))
- #print "Global [ min=%.2f max=%.2f avg=%.2f stddev=%.2f \n" % (min(global_disk_perf),max(global_disk_perf),avg, standard_deviation)
-
- average_file.close()
- min_file.close()
- max_file.close()
- stddev_file.close()
- global_file.close()
- try:
- os.remove(gnuplot_output_dir+'mymath')
- except:
- True
-
- generate_gnuplot_math_script("Average values of "+title,gnuplot_output_filename+'.average',mode,int(avg),gnuplot_output_dir,gpm_dir)
- generate_gnuplot_math_script("Min values of "+title,gnuplot_output_filename+'.min',mode,average(global_min),gnuplot_output_dir,gpm_dir)
- generate_gnuplot_math_script("Max values of "+title,gnuplot_output_filename+'.max',mode,average(global_max),gnuplot_output_dir,gpm_dir)
- generate_gnuplot_math_script("Standard Deviation of "+title,gnuplot_output_filename+'.stddev',mode,int(standard_deviation),gnuplot_output_dir,gpm_dir)
-
-def parse_global_files(fio_data_file, global_search):
- max_result=0
- max_file=''
- for file in fio_data_file:
- f=open(file)
- disk_count=0
- search_value=-1
-
- # Let's read the complete file
- while True:
- try:
- # We do split the name from the value
- name,value=f.readline().split("=")
- except:
- f.close()
- break
- # If we ended the file
- if not name:
- # Let's process what we have
- f.close()
- break
- else:
- # disks_count is not global_search item
- # As we need it for some computation, let's save it
- if name=="disks_count":
- disks_count=int(value)
-
- # Let's catch the searched item
- if global_search in name:
- search_value=float(value)
-
- # Let's process the avg value by estimated the global bandwidth per file
- # We keep the biggest in memory for reporting
- if global_search == "avg":
- if (disks_count > 0) and (search_value != -1):
- result=disks_count*search_value
- if (result > max_result):
- max_result=result
- max_file=file
- # Let's print the avg output
- if global_search == "avg":
- print "Biggest aggregated value of %s was %2.f in file %s\n" % (global_search, max_result, max_file)
- else:
- print "Global search %s is not yet implemented\n" % global_search
-
-def render_gnuplot(fio_data_file, gnuplot_output_dir):
- print "Running gnuplot Rendering"
- try:
- # Let's render all the compared files if some
- if len(fio_data_file) > 1:
- if verbose: print " |-> Rendering comparing traces"
- os.system("cd %s; for i in *.gnuplot; do gnuplot $i; done" % gnuplot_output_dir)
- if verbose: print " |-> Rendering math traces"
- os.system("cd %s; gnuplot mymath" % gnuplot_output_dir)
- if verbose: print " |-> Rendering 2D & 3D traces"
- os.system("cd %s; gnuplot mygraph" % gnuplot_output_dir)
-
- name_of_directory="the current"
- if gnuplot_output_dir != "./":
- name_of_directory=gnuplot_output_dir
- print "\nRendering traces are available in %s directory" % name_of_directory
- global keep_temp_files
- keep_temp_files=False
- except:
- print "Could not run gnuplot on mymath or mygraph !\n"
- sys.exit(1);
-
-def print_help():
- print 'fio2gnuplot -ghbiodvk -t <title> -o <outputfile> -p <pattern> -G <type> -m <time> -M <time>'
- print
- print '-h --help : Print this help'
- print '-p <pattern> or --pattern <pattern> : A glob pattern to select fio input files'
- print '-b or --bandwidth : A predefined pattern for selecting *_bw.log files'
- print '-i or --iops : A predefined pattern for selecting *_iops.log files'
- print '-g or --gnuplot : Render gnuplot traces before exiting'
- print '-o or --outputfile <file> : The basename for gnuplot traces'
- print ' - Basename is set with the pattern if defined'
- print '-d or --outputdir <dir> : The directory where gnuplot shall render files'
- print '-t or --title <title> : The title of the gnuplot traces'
- print ' - Title is set with the block size detected in fio traces'
- print '-G or --Global <type> : Search for <type> in .global files match by a pattern'
- print ' - Available types are : min, max, avg, stddev'
- print ' - The .global extension is added automatically to the pattern'
- print '-m or --min_time <time> : Only consider data starting from <time> seconds (default is 0)'
- print '-M or --max_time <time> : Only consider data ending before <time> seconds (default is -1 aka nolimit)'
- print '-v or --verbose : Increasing verbosity'
- print '-k or --keep : Keep all temporary files from gnuplot\'s output dir'
-
-def main(argv):
- mode='unknown'
- pattern=''
- pattern_set_by_user=False
- title='No title'
- gnuplot_output_filename='result'
- gnuplot_output_dir='./'
- gpm_dir="/usr/share/fio/"
- disk_perf=[]
- run_gnuplot=False
- parse_global=False
- global_search=''
- min_time=0
- max_time=-1
- global verbose
- verbose=False
- global temporary_files
- temporary_files=[]
- global keep_temp_files
- keep_temp_files=True
- force_keep_temp_files=False
-
- if not os.path.isfile(gpm_dir+'math.gpm'):
- gpm_dir="/usr/local/share/fio/"
- if not os.path.isfile(gpm_dir+'math.gpm'):
- print "Looks like fio didn't get installed properly as no gpm files found in '/usr/share/fio' or '/usr/local/share/fio'\n"
- sys.exit(3)
-
- try:
- opts, args = getopt.getopt(argv[1:],"ghkbivo:d:t:p:G:m:M:",['bandwidth', 'iops', 'pattern', 'outputfile', 'outputdir', 'title', 'min_time', 'max_time', 'gnuplot', 'Global', 'help', 'verbose','keep'])
- except getopt.GetoptError:
- print "Error: One of the options passed to the cmdline was not supported"
- print "Please fix your command line or read the help (-h option)"
- sys.exit(2)
-
- for opt, arg in opts:
- if opt in ("-b", "--bandwidth"):
- pattern='*_bw.log'
- elif opt in ("-i", "--iops"):
- pattern='*_iops.log'
- elif opt in ("-v", "--verbose"):
- verbose=True
- elif opt in ("-k", "--keep"):
- #User really wants to keep the temporary files
- force_keep_temp_files=True
- elif opt in ("-p", "--pattern"):
- pattern_set_by_user=True
- pattern=arg
- pattern=pattern.replace('\\','')
- elif opt in ("-o", "--outputfile"):
- gnuplot_output_filename=arg
- elif opt in ("-d", "--outputdir"):
- gnuplot_output_dir=arg
- if not gnuplot_output_dir.endswith('/'):
- gnuplot_output_dir=gnuplot_output_dir+'/'
- if not os.path.exists(gnuplot_output_dir):
- os.makedirs(gnuplot_output_dir)
- elif opt in ("-t", "--title"):
- title=arg
- elif opt in ("-m", "--min_time"):
- min_time=arg
- elif opt in ("-M", "--max_time"):
- max_time=arg
- elif opt in ("-g", "--gnuplot"):
- run_gnuplot=True
- elif opt in ("-G", "--Global"):
- parse_global=True
- global_search=arg
- elif opt in ("-h", "--help"):
- print_help()
- sys.exit(1)
-
- # Adding .global extension to the file
- if parse_global==True:
- if not gnuplot_output_filename.endswith('.global'):
- pattern = pattern+'.global'
-
- fio_data_file=find_file('.',pattern)
- if len(fio_data_file) == 0:
- print "No log file found with pattern %s!" % pattern
- # Try numjob log file format if per_numjob_logs=1
- if (pattern == '*_bw.log'):
- fio_data_file=find_file('.','*_bw.*.log')
- if (pattern == '*_iops.log'):
- fio_data_file=find_file('.','*_iops.*.log')
- if len(fio_data_file) == 0:
- sys.exit(1)
- else:
- print "Using log file per job format instead"
- else:
- print "%d files Selected with pattern '%s'" % (len(fio_data_file), pattern)
-
- fio_data_file=sorted(fio_data_file, key=str.lower)
- for file in fio_data_file:
- print ' |-> %s' % file
- if "_bw.log" in file :
- mode="Bandwidth (KB/sec)"
- if "_iops.log" in file :
- mode="IO per Seconds (IO/sec)"
- if (title == 'No title') and (mode != 'unknown'):
- if "Bandwidth" in mode:
- title='Bandwidth benchmark with %d fio results' % len(fio_data_file)
- if "IO" in mode:
- title='IO benchmark with %d fio results' % len(fio_data_file)
-
- print
- #We need to adjust the output filename regarding the pattern required by the user
- if (pattern_set_by_user == True):
- gnuplot_output_filename=pattern
- # As we do have some glob in the pattern, let's make this simpliest
- # We do remove the simpliest parts of the expression to get a clear file name
- gnuplot_output_filename=gnuplot_output_filename.replace('-*-','-')
- gnuplot_output_filename=gnuplot_output_filename.replace('*','-')
- gnuplot_output_filename=gnuplot_output_filename.replace('--','-')
- gnuplot_output_filename=gnuplot_output_filename.replace('.log','')
- # Insure that we don't have any starting or trailing dash to the filename
- gnuplot_output_filename = gnuplot_output_filename[:-1] if gnuplot_output_filename.endswith('-') else gnuplot_output_filename
- gnuplot_output_filename = gnuplot_output_filename[1:] if gnuplot_output_filename.startswith('-') else gnuplot_output_filename
- if (gnuplot_output_filename == ''):
- gnuplot_output_filename='default'
-
- if parse_global==True:
- parse_global_files(fio_data_file, global_search)
- else:
- blk_size=compute_temp_file(fio_data_file,disk_perf,gnuplot_output_dir,min_time,max_time)
- title="%s @ Blocksize = %dK" % (title,blk_size/1024)
- compute_aggregated_file(fio_data_file, gnuplot_output_filename, gnuplot_output_dir)
- compute_math(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir)
- generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir)
-
- if (run_gnuplot==True):
- render_gnuplot(fio_data_file, gnuplot_output_dir)
-
- # Shall we clean the temporary files ?
- if keep_temp_files==False and force_keep_temp_files==False:
- # Cleaning temporary files
- if verbose: print "Cleaning temporary files"
- for f in enumerate(temporary_files):
- if verbose: print " -> %s"%f[1]
- try:
- os.remove(f[1])
- except:
- True
-
-#Main
-if __name__ == "__main__":
- sys.exit(main(sys.argv))
diff --git a/tools/plot/fio2gnuplot.1 b/tools/plot/fio2gnuplot.1
deleted file mode 100644
index 1a33167b..00000000
--- a/tools/plot/fio2gnuplot.1
+++ /dev/null
@@ -1,161 +0,0 @@
-.\" Text automatically generated by txt2man
-.TH fio2gnuplot "07 août 2013" "" ""
-.SH NAME
-\fBfio2gnuplot \fP- Render fio's output files with gnuplot
-.SH SYNOPSIS
-.nf
-.fam C
-\fBfio2gnuplot\fP [\fB-ghbiodvk\fP] [\fB-t\fP \fItitle\fP] [\fB-o\fP \fIoutputfile\fP]
- [\fB-d\fP \fIoutput_dir\fP] [\fB-p\fP \fIpattern\fP]
- [\fB-G\fP \fItype\fP] [\fB-m\fP \fImin_time\fP] [\fB-M\fP \fImax_time\fP]
-
-.fam T
-.fi
-.fam T
-.fi
-.SH DESCRIPTION
-\fBfio2gnuplot\fP analyze a set of fio's log files to turn them into a set of graphical traces using gnuplot tool.
-Several flavor of plotting are produced
-.TP
-.B
-Individual 2D Graph
-Each file is plotted in a separate image file with several option
-.RS
-.IP \(bu 3
-raw : Plot the exact reported performance. This plotting could be difficult to read
-.IP \(bu 3
-smooth :a smoother version of the raw print
-Using csplines option of gnuplot, the rendering is
-filtered to get an easier to read graph.
-.IP \(bu 3
-trend : an even smoother version of the raw print to get trends
-Bezier's curves makes much more filtered plots
-The resulting graph helps at understanding trends.
-.RE
-.TP
-.B
-Grouped 2D graph
-All files are plotted in a single image to ease the comparaison. The same rendering options as per the individual 2D graph are used :
-.RS
-.IP \(bu 3
-raw
-.IP \(bu 3
-smooth
-.IP \(bu 3
-trend
-.RE
-.TP
-.B
-Grouped 3D graph
-All files are plotted into a single 3D graph.
-The 3D plotting generates a 'surface' to estimate how close were
-the performance.
-A flat surface means a good coherency between traces.
-A rugged surface means a lack of coherency between traces
-.TP
-.B
-Mathemical Plotting
-.RS
-.TP
-.B
-Average graph
-A bar graph to show the average performance of each file.
-A green line is added to show the global average performance.
-This green line helps at understanding how far from the average is
-every individual file.
-.TP
-.B
-Min graph
-A green line is added to show the global average of minimal performance.
-This green line helps at understanding how far from the average is
-every individual file.
-.TP
-.B
-Max graph
-A bar graph to show the maximum performance of each file.
-A green line is added to show the global average of maximal performance.
-This green line helps at understanding how far from the average is
-every individual file.
-.TP
-.B
-Standard Deviation
-A bar graph to show the standard deviation of each file.
-A green line is added to show the global average of standard deviation.
-This green line helps at understanding how far from the average is
-every individual file.
-.SH OPTIONS
-.TP
-.B
-\fB-h\fP or \fB--help\fP
-The option \fB-h\fP displays help
-.TP
-.B
-\fB-p\fP '\fIpattern\fP' or --\fIpattern\fP '\fIpattern\fP'
-A \fIpattern\fP in regexp to select fio input files.
-Don't forget the simple quotes to avoid shell's interactions
-.TP
-.B
-\fB-b\fP or \fB--bandwidth\fP
-A predefined \fIpattern\fP for selecting *_bw.log files
-.TP
-.B
-\fB-i\fP or \fB--iops\fP
-A predefined \fIpattern\fP for selecting *_iops.log files
-.TP
-.B
-\fB-g\fP or \fB--gnuplot\fP
-Render gnuplot traces before exiting
-.TP
-.B
-\fB-o\fP file or --\fIoutputfile\fP file
-The basename for gnuplot traces (set with the \fIpattern\fP if defined)
-.TP
-.B
-\fB-d\fP dir or \fB--outputdir\fP dir
-The directory where gnuplot shall render files.
-.TP
-.B
-\fB-t\fP \fItitle\fP or --\fItitle\fP \fItitle\fP
-The \fItitle\fP of the gnuplot traces.
-Title is set with the block size detected in fio trace
-.TP
-.B
-\fB-G\fP \fItype\fP or \fB--Global\fP \fItype\fP
-Search for '\fItype\fP' in .global files match by a \fIpattern\fP.
-Available types are : min, max, avg, stddev.
-The .global extension is added automatically to the \fIpattern\fP
-.TP
-.B
-\fB-m\fP time or --\fImin_time\fP time
-Only consider data starting from 'time' seconds. Default is 0
-.TP
-.B
-\fB-M\fP time or --\fImax_time\fP time
-Only consider data ending before 'time' seconds. Default is \fB-1\fP aka nolimit
-.TP
-.B
-\fB-v\fP or \fB--verbose\fP
-Increasing verbosity
-.TP
-.B
-\fB-k\fP or \fB--keep\fP
-Keep all temporary files from gnuplot's output dir
-.SH EXAMPLE
-.TP
-.B
-To plot all the traces named like 'host*_read_4k_iops.log'
-$ \fBfio2gnuplot\fP \fB-p\fP 'host*_read_4k_iops.log' \fB-g\fP
-.TP
-.B
-To plot all IO oriented log files from the current directory
-$ \fBfio2gnuplot\fP \fB-g\fP \fB-i\fP
-.TP
-.B
-To plot all Bandwidth oriented log files from the current directory
-$ \fBfio2gnuplot\fP \fB-g\fP \fB-b\fP
-.TP
-.B
-To plot all Bandwidth oriented log files in a directory name 'outdir'
-$ \fBfio2gnuplot\fP \fB-g\fP \fB-b\fP \fB-d\fP outdir
-.SH AUTHOR
-Erwan Velu <erwan@enovance.com>
diff --git a/tools/plot/fio2gnuplot.manpage b/tools/plot/fio2gnuplot.manpage
deleted file mode 100644
index 6a12cf81..00000000
--- a/tools/plot/fio2gnuplot.manpage
+++ /dev/null
@@ -1,117 +0,0 @@
-NAME
-fio2gnuplot - Render fio's output files with gnuplot
-SYNOPSIS
-fio2gnuplot [-ghbiodvk] [-t title] [-o outputfile]
- [-d output_dir] [-p pattern]
- [-G type] [-m min_time] [-M max_time]
-
-DESCRIPTION
- fio2gnuplot analyze a set of fio's log files to turn them into a set of graphical traces using gnuplot tool.
- Several flavor of plotting are produced
-
- Individual 2D Graph
- Each file is plotted in a separate image file with several option
- - raw : Plot the exact reported performance. This plotting could be difficult to read
- - smooth :a smoother version of the raw print
- Using csplines option of gnuplot, the rendering is
- filtered to get an easier to read graph.
- - trend : an even smoother version of the raw print to get trends
- Bezier's curves makes much more filtered plots
- The resulting graph helps at understanding trends.
-
- Grouped 2D graph
- All files are plotted in a single image to ease the comparaison. The same rendering options as per the individual 2D graph are used :
- - raw
- - smooth
- - trend
-
- Grouped 3D graph
- All files are plotted into a single 3D graph.
- The 3D plotting generates a 'surface' to estimate how close were
- the performance.
- A flat surface means a good coherency between traces.
- A rugged surface means a lack of coherency between traces
-
- Mathemical Plotting
- Average graph
- A bar graph to show the average performance of each file.
- A green line is added to show the global average performance.
- This green line helps at understanding how far from the average is
- every individual file.
-
- Min graph
- A green line is added to show the global average of minimal performance.
- This green line helps at understanding how far from the average is
- every individual file.
-
- Max graph
- A bar graph to show the maximum performance of each file.
- A green line is added to show the global average of maximal performance.
- This green line helps at understanding how far from the average is
- every individual file.
-
- Standard Deviation
- A bar graph to show the standard deviation of each file.
- A green line is added to show the global average of standard deviation.
- This green line helps at understanding how far from the average is
- every individual file.
-
-OPTIONS
- -h or --help
- The option -h displays help
-
- -p 'pattern' or --pattern 'pattern'
- A pattern in regexp to select fio input files.
- Don't forget the simple quotes to avoid shell's interactions
-
- -b or --bandwidth
- A predefined pattern for selecting *_bw.log files
-
- -i or --iops
- A predefined pattern for selecting *_iops.log files
-
- -g or --gnuplot
- Render gnuplot traces before exiting
-
- -o file or --outputfile file
- The basename for gnuplot traces (set with the pattern if defined)
-
- -d dir or --outputdir dir
- The directory where gnuplot shall render files.
-
- -t title or --title title
- The title of the gnuplot traces.
- Title is set with the block size detected in fio trace
-
- -G type or --Global type
- Search for 'type' in .global files match by a pattern.
- Available types are : min, max, avg, stddev.
- The .global extension is added automatically to the pattern
-
- -m time or --min_time time
- Only consider data starting from 'time' seconds. Default is 0
-
- -M time or --max_time time
- Only consider data ending before 'time' seconds. Default is -1 aka nolimit
-
- -v or --verbose
- Increasing verbosity
-
- -k or --keep
- Keep all temporary files from gnuplot's output dir
-
-EXAMPLE
-To plot all the traces named like 'host*_read_4k_iops.log'
- $ fio2gnuplot -p 'host*_read_4k_iops.log' -g
-
-To plot all IO oriented log files from the current directory
- $ fio2gnuplot -g -i
-
-To plot all Bandwidth oriented log files from the current directory
- $ fio2gnuplot -g -b
-
-To plot all Bandwidth oriented log files in a directory name 'outdir'
- $ fio2gnuplot -g -b -d outdir
-
-AUTHOR
- Erwan Velu <erwan@enovance.com>
diff --git a/tools/plot/graph2D.gpm b/tools/plot/graph2D.gpm
deleted file mode 100644
index 769b754a..00000000
--- a/tools/plot/graph2D.gpm
+++ /dev/null
@@ -1,55 +0,0 @@
-# This Gnuplot file has been generated by eNovance
-
-needed_args = 8
-if (exists("ARGC") && ARGC >= needed_args) \
- found_args = 1; \
-else if (strlen("$$#") < 3 && "$#" >= needed_args) \
- found_args = 1; \
- ARG1 = "$0"; \
- ARG2 = "$1"; \
- ARG3 = "$2"; \
- ARG4 = "$3"; \
- ARG5 = "$4"; \
- ARG6 = "$5"; \
- ARG7 = "$6"; \
- ARG8 = "$7"; \
-else \
- found_args = 0; \
- print "Aborting: could not find all arguments"; \
- exit
-
-avg_num = ARG8 + 0
-avg_str = sprintf("%g", avg_num)
-
-set title ARG1
-
-set terminal png size 1280,1024
-set output ARG4 . '.png'
-#set terminal x11
-
-#Preparing Axes
-#set logscale x
-set ytics axis out auto
-#set data style lines
-set key top left reverse
-set xlabel "Time (Seconds)"
-set ylabel ARG5
-set xrange [0:]
-set yrange [0:]
-
-#Set Color style
-#set palette rgbformulae 22,9,23
-#set palette rgbformulae 7,5,15
-set style line 100 lt 7 lw 0.5
-set style line 1 lt 1 lw 3 pt 3 linecolor rgb "green"
-
-plot ARG2 using 2:3 with linespoints title ARG3, avg_num w l ls 1 ti 'Global average value (' . avg_str . ')'
-
-set output ARG6 . '.png'
-plot ARG2 using 2:3 smooth csplines title ARG3, avg_num w l ls 1 ti 'Global average value (' . avg_str . ')'
-
-set output ARG7 . '.png'
-plot ARG2 using 2:3 smooth bezier title ARG3, avg_num w l ls 1 ti 'Global average value (' . avg_str .')'
-
-#pause -1
-#The End
diff --git a/tools/plot/graph3D.gpm b/tools/plot/graph3D.gpm
deleted file mode 100644
index ac2cdf6c..00000000
--- a/tools/plot/graph3D.gpm
+++ /dev/null
@@ -1,95 +0,0 @@
-# This Gnuplot file has been generated by eNovance
-
-needed_args = 5
-if (exists("ARGC") && ARGC >= needed_args) \
- found_args = 1; \
-else if (strlen("$$#") < 3 && "$#" >= needed_args) \
- found_args = 1; \
- ARG1 = "$0"; \
- ARG2 = "$1"; \
- ARG3 = "$2"; \
- ARG4 = "$3"; \
- ARG5 = "$4"; \
-else \
- found_args = 0; \
- print "Aborting: could not find all arguments"; \
- exit
-
-set title ARG1
-
-set terminal png size 1280,1024
-set output ARG4 . '.png'
-#set terminal x11
-#3D Config
-set isosamples 30
-set hidden3d
-set pm3d at s solid hidden3d 100 scansbackward
-set pm3d depthorder
-
-#Preparing Axes
-#set logscale x
-set ytics axis out 0,1
-#set data style lines
-set grid back
-set key top left reverse
-set ylabel "Disk"
-set xlabel "Time (Seconds)"
-set zlabel ARG5
-set cbrange [0:]
-set zrange [0:]
-
-#Set Color style
-#set palette rgbformulae 22,9,23
-set palette rgbformulae 7,5,15
-set style line 100 lt 7 lw 0.5
-
-#Multiploting
-set multiplot
-
-#Top Left View
-set size 0.5,0.5
-set view 64,216
-set origin 0,0.5
-splot ARG2 using 2:1:3 with linespoints title ARG3
-
-#Top Right View
-set size 0.5,0.5
-set origin 0.5,0.5
-set view 90,0
-set pm3d at s solid hidden3d 100 scansbackward
-set pm3d depthorder
-splot ARG2 using 2:1:3 with linespoints title ARG3
-
-#Bottom Right View
-set size 0.5,0.5
-set origin 0.5,0
-set view 63,161
-set pm3d at s solid hidden3d 100 scansbackward
-set pm3d depthorder
-splot ARG2 using 2:1:3 with linespoints title ARG3
-
-#Bottom Left View
-set size 0.5,0.5
-set origin 0,0
-set pm3d map
-splot ARG2 using 2:1:3 with linespoints title ARG3
-
-#Unsetting multiplotting
-unset multiplot
-#pause -1
-
-#Preparing 3D Interactive view
-set mouse
-set terminal png size 1024,768
-set output ARG4 . '-3D.png'
-
-#set term x11
-set view 64,216
-set origin 0,0
-set size 1,1
-set pm3d at bs solid hidden3d 100 scansbackward
-set pm3d depthorder
-splot ARG2 using 2:1:3 with linespoints title ARG3
-
-#pause -1
-#The End
diff --git a/tools/plot/math.gpm b/tools/plot/math.gpm
deleted file mode 100644
index 0a2aff56..00000000
--- a/tools/plot/math.gpm
+++ /dev/null
@@ -1,42 +0,0 @@
-# This Gnuplot file has been generated by eNovance
-if (exists("ARGC") && ARGC > 5) \
- found_args = 1; \
-else if (strlen("$$#") < 3 && "$#" > 5) \
- found_args = 1; \
- ARG1 = "$0"; \
- ARG2 = "$1"; \
- ARG3 = "$2"; \
- ARG4 = "$3"; \
- ARG5 = "$4"; \
- ARG6 = "$5"; \
-else \
- found_args = 0; \
- print "Aborting: could not find all arguments"; \
- exit
-
-avg_num = ARG6 + 0
-avg_str = sprintf("%g", avg_num)
-
-set title ARG1
-
-set terminal png size 1280,1024
-set output ARG4 . '.png'
-
-set palette rgbformulae 7,5,15
-set style line 100 lt 7 lw 0.5
-set style fill transparent solid 0.9 noborder
-set auto x
-set ylabel ARG5
-set xlabel "Disk"
-set yrange [0:]
-set style data histogram
-set style histogram cluster gap 1
-set style fill solid border -1
-set boxwidth 2
-#set xtic rotate by -10 scale 10 font ",8"
-set bmargin 3
-set xtics axis out
-set xtic rotate by 45 scale 0 font ",8" autojustify
-set xtics offset 0,-1 border -5,1,5
-set style line 1 lt 1 lw 3 pt 3 linecolor rgb "green"
-plot ARG2 using 2:xtic(1) ti col, avg_num w l ls 1 ti 'Global average value (' . avg_str . ')'
diff --git a/tools/plot/samples/Makefile b/tools/plot/samples/Makefile
deleted file mode 100644
index df0480f1..00000000
--- a/tools/plot/samples/Makefile
+++ /dev/null
@@ -1,19 +0,0 @@
-all: clean m2sw1-128k-sdb-randwrite-para.results_bw.log io bandwidth
-
-m2sw1-128k-sdb-randwrite-para.results_bw.log:
- tar -xf fio-logs.tar.gz
-
-io: setup
- ./fio2gnuplot.py -p 'm2sw1-128k-*-read-para*iops.log' -g
-
-bandwidth: setup
- ./fio2gnuplot.py -p 'm2sw1-128k-*-read-para*bw.log' -g
-
-setup:
- ln -sf ../*py ../*gpm .
-
-clean:
- rm -rf *png mygraph mymath *py *gpm gnuplot_temp_file* *~
- rm -rf *.average *.stddev *.min *.max *.global
- rm -rf m2sw1-128k-read-para-bw m2sw1-128k-read-para-iops
- rm -rf *log
diff --git a/tools/plot/samples/fio-logs.tar.gz b/tools/plot/samples/fio-logs.tar.gz
deleted file mode 100644
index 2237f5e3..00000000
--- a/tools/plot/samples/fio-logs.tar.gz
+++ /dev/null
Binary files differ
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-09 01:31:37 +0000