[Telemetry] Add HTML output and make it the default.

This is based on Blink's run-perf-tests HTML output. BUG=264695 Review URL: https://chromiumcodereview.appspot.com/20565003 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@215542 0039d316-1c4b-4281-b951-d872f2087c98
author: tonyg@chromium.org <tonyg@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> 2013-08-04 21:33:29 +0000
committer: tonyg@chromium.org <tonyg@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> 2013-08-04 21:33:29 +0000
commit: 9694696cd263109e42565a0598adddee1bbf4bdc (patch)
tree: eaf6585deea4065a1f623a41aab74cff8b1b71b3 /tools/telemetry/support
parent: 872e8e1683de933fe09e948ce42c2dfc6bcdf5b7 (diff)
download: chromium_src-9694696cd263109e42565a0598adddee1bbf4bdc.zip
chromium_src-9694696cd263109e42565a0598adddee1bbf4bdc.tar.gz
chromium_src-9694696cd263109e42565a0598adddee1bbf4bdc.tar.bz2
1 files changed, 596 insertions, 0 deletions
diff --git a/tools/telemetry/support/html_output/results-template.html b/tools/telemetry/support/html_output/results-template.html
new file mode 100644
index 0000000..b0b84c7
--- /dev/null
+++ b/tools/telemetry/support/html_output/results-template.html
@@ -0,0 +1,596 @@
+<!DOCTYPE html>
+<html>
+<head>
+<title>Telemetry Performance Test Results</title>
+<style type="text/css">
+
+section {
+    background: white;
+    padding: 10px;
+    position: relative;
+}
+
+.time-plots {
+    padding-left: 25px;
+}
+
+.time-plots > div {
+    display: inline-block;
+    width: 90px;
+    height: 40px;
+    margin-right: 10px;
+}
+
+section h1 {
+    text-align: center;
+    font-size: 1em;
+}
+
+section .tooltip {
+    position: absolute;
+    text-align: center;
+    background: #ffcc66;
+    border-radius: 5px;
+    padding: 0px 5px;
+}
+
+body {
+    padding: 0px;
+    margin: 0px;
+    font-family: sans-serif;
+}
+
+table {
+    background: white;
+    width: 100%;
+}
+
+table, td, th {
+    border-collapse: collapse;
+    padding: 5px;
+    white-space: nowrap;
+}
+
+tr.even {
+    background: #f6f6f6;
+}
+
+table td {
+    position: relative;
+    font-family: monospace;
+}
+
+th, td {
+    cursor: pointer;
+    cursor: hand;
+}
+
+th {
+    background: #e6eeee;
+    background: -webkit-gradient(linear, left top, left bottom, from(rgb(244, 244, 244)), to(rgb(217, 217, 217)));
+    border: 1px solid #ccc;
+}
+
+th:after {
+    content: ' \25B8';
+}
+
+th.headerSortUp:after {
+    content: ' \25BE';
+}
+
+th.headerSortDown:after {
+    content: ' \25B4';
+}
+
+td.comparison, td.result {
+    text-align: right;
+}
+
+td.better {
+    color: #6c6;
+}
+
+td.worse {
+    color: #c66;
+}
+
+td.missing {
+    text-align: center;
+}
+
+.checkbox {
+    display: inline-block;
+    background: #eee;
+    background: -webkit-gradient(linear, left bottom, left top, from(rgb(220, 220, 220)), to(rgb(200, 200, 200)));
+    border: inset 1px #ddd;
+    border-radius: 5px;
+    margin: 10px;
+    font-size: small;
+    cursor: pointer;
+    cursor: hand;
+    -webkit-user-select: none;
+    font-weight: bold;
+}
+
+.checkbox span {
+    display: inline-block;
+    line-height: 100%;
+    padding: 5px 8px;
+    border: outset 1px transparent;
+}
+
+.checkbox .checked {
+    background: #e6eeee;
+    background: -webkit-gradient(linear, left top, left bottom, from(rgb(255, 255, 255)), to(rgb(235, 235, 235)));
+    border: outset 1px #eee;
+    border-radius: 5px;
+}
+
+</style>
+</head>
+<body onload="init()">
+<div style="padding: 0 10px; white-space: nowrap;">
+Result <span id="time-memory" class="checkbox"><span class="checked">Time</span><span>Memory</span></span>
+Reference <span id="reference" class="checkbox"></span>
+Run Telemetry with --reset-html-results to clear all runs
+</div>
+<table id="container"></table>
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.6.4/jquery.min.js"></script>
+<script>
+%plugins%
+</script>
+<script>
+function TestResult(metric, values, associatedRun) {
+    if (values[0] instanceof Array) {
+        var flattenedValues = [];
+        for (var i = 0; i < values.length; i++)
+            flattenedValues = flattenedValues.concat(values[i]);
+        values = flattenedValues;
+    }
+
+    this.test = function () { return metric; }
+    this.values = function () { return values.map(function (value) { return metric.scalingFactor() * value; }); }
+    this.unscaledMean = function () { return Statistics.sum(values) / values.length; }
+    this.mean = function () { return metric.scalingFactor() * this.unscaledMean(); }
+    this.min = function () { return metric.scalingFactor() * Statistics.min(values); }
+    this.max = function () { return metric.scalingFactor() * Statistics.max(values); }
+    this.confidenceIntervalDelta = function () {
+        return metric.scalingFactor() * Statistics.confidenceIntervalDelta(0.95, values.length,
+            Statistics.sum(values), Statistics.squareSum(values));
+    }
+    this.confidenceIntervalDeltaRatio = function () { return this.confidenceIntervalDelta() / this.mean(); }
+    this.percentDifference = function(other) { return (other.unscaledMean() - this.unscaledMean()) / this.unscaledMean(); }
+    this.isStatisticallySignificant = function (other) {
+        var diff = Math.abs(other.mean() - this.mean());
+        return diff > this.confidenceIntervalDelta() && diff > other.confidenceIntervalDelta();
+    }
+    this.run = function () { return associatedRun; }
+}
+
+function TestRun(entry) {
+    this.description = function () { return entry['description']; }
+    this.revision = function () { return entry['revision']; }
+    this.label = function () {
+        var label = 'r' + this.revision();
+        if (this.description())
+            label += ' &dash; ' + this.description();
+        return label;
+    }
+}
+
+function PerfTestMetric(name, metric, unit, isImportant) {
+    var testResults = [];
+    var cachedUnit = null;
+    var cachedScalingFactor = null;
+
+    // We can't do this in TestResult because all results for each test need to share the same unit and the same scaling factor.
+    function computeScalingFactorIfNeeded() {
+        // FIXME: We shouldn't be adjusting units on every test result.
+        // We can only do this on the first test.
+        if (!testResults.length || cachedUnit)
+            return;
+
+        var mean = testResults[0].unscaledMean(); // FIXME: We should look at all values.
+        var kilo = unit == 'bytes' ? 1024 : 1000;
+        if (mean > 10 * kilo * kilo && unit != 'ms') {
+            cachedScalingFactor = 1 / kilo / kilo;
+            cachedUnit = 'M ' + unit;
+        } else if (mean > 10 * kilo) {
+            cachedScalingFactor = 1 / kilo;
+            cachedUnit = unit == 'ms' ? 's' : ('K ' + unit);
+        } else {
+            cachedScalingFactor = 1;
+            cachedUnit = unit;
+        }
+    }
+
+    this.name = function () { return name + ':' + metric; }
+    this.isImportant = isImportant;
+    this.isMemoryTest = function () {
+        return (unit == 'kb' ||
+                unit == 'KB' ||
+                unit == 'MB' ||
+                unit == 'bytes');
+    }
+    this.addResult = function (newResult) {
+        testResults.push(newResult);
+        cachedUnit = null;
+        cachedScalingFactor = null;
+    }
+    this.results = function () { return testResults; }
+    this.scalingFactor = function() {
+        computeScalingFactorIfNeeded();
+        return cachedScalingFactor;
+    }
+    this.unit = function () {
+        computeScalingFactorIfNeeded();
+        return cachedUnit;
+    }
+    this.biggerIsBetter = function () {
+        if (window.unitToBiggerIsBetter == undefined) {
+            window.unitToBiggerIsBetter = {};
+            var units = JSON.parse(document.getElementById('units-json').textContent);
+            for (var unit in units) {
+                if (units[unit].improvement_direction == 'up') {
+                    window.unitToBiggerIsBetter[unit] = true;
+                }
+            }
+        }
+        return window.unitToBiggerIsBetter[unit];
+    }
+}
+
+var plotColor = 'rgb(230,50,50)';
+var subpointsPlotOptions = {
+    lines: {show:true, lineWidth: 0},
+    color: plotColor,
+    points: {show: true, radius: 1},
+    bars: {show: false}};
+
+var mainPlotOptions = {
+    xaxis: {
+        min: -0.5,
+        tickSize: 1,
+    },
+    crosshair: { mode: 'y' },
+    series: { shadowSize: 0 },
+    bars: {show: true, align: 'center', barWidth: 0.5},
+    lines: { show: false },
+    points: { show: true },
+    grid: {
+        borderWidth: 1,
+        borderColor: '#ccc',
+        backgroundColor: '#fff',
+        hoverable: true,
+        autoHighlight: false,
+    }
+};
+
+var timePlotOptions = {
+    yaxis: { show: false },
+    xaxis: { show: false },
+    lines: { show: true },
+    grid: { borderWidth: 1, borderColor: '#ccc' },
+    colors: [ plotColor ]
+};
+
+function createPlot(container, test) {
+    var section = $('<section><div class="plot"></div><div class="time-plots"></div>'
+        + '<span class="tooltip"></span></section>');
+    section.children('.plot').css({'width': (100 * test.results().length + 25) + 'px', 'height': '300px'});
+    $(container).append(section);
+
+    var plotContainer = section.children('.plot');
+    var minIsZero = true;
+    attachPlot(test, plotContainer, minIsZero);
+
+    attachTimePlots(test, section.children('.time-plots'));
+
+    var tooltip = section.children('.tooltip');
+    plotContainer.bind('plothover', function (event, position, item) {
+        if (item) {
+            var postfix = item.series.id ? ' (' + item.series.id + ')' : '';
+            tooltip.html(item.datapoint[1].toPrecision(4) + postfix);
+            var sectionOffset = $(section).offset();
+            tooltip.css({left: item.pageX - sectionOffset.left - tooltip.outerWidth() / 2, top: item.pageY - sectionOffset.top + 10});
+            tooltip.fadeIn(200);
+        } else
+            tooltip.hide();
+    });
+    plotContainer.mouseout(function () {
+        tooltip.hide();
+    });
+    plotContainer.click(function (event) {
+        event.preventDefault();
+        minIsZero = !minIsZero;
+        attachPlot(test, plotContainer, minIsZero);
+    });
+
+    return section;
+}
+
+function attachTimePlots(test, container) {
+    var results = test.results();
+    var attachedPlot = false;
+    for (var i = 0; i < results.length; i++) {
+        container.append('<div></div>');
+        var values = results[i].values();
+        if (!values)
+            continue;
+        attachedPlot = true;
+
+        $.plot(container.children().last(), [values.map(function (value, index) { return [index, value]; })],
+            $.extend(true, {}, timePlotOptions, {yaxis: {min: Math.min.apply(Math, values) * 0.9, max: Math.max.apply(Math, values) * 1.1},
+                xaxis: {min: -0.5, max: values.length - 0.5}}));
+    }
+    if (!attachedPlot)
+        container.children().remove();
+}
+
+function attachPlot(test, plotContainer, minIsZero) {
+    var results = test.results();
+
+    var values = results.reduce(function (values, result, index) {
+        var newValues = result.values();
+        return newValues ? values.concat(newValues.map(function (value) { return [index, value]; })) : values;
+    }, []);
+
+    var plotData = [$.extend(true, {}, subpointsPlotOptions, {data: values})];
+    plotData.push({id: '&mu;', data: results.map(function (result, index) { return [index, result.mean()]; }), color: plotColor});
+
+    var overallMax = Statistics.max(results.map(function (result, index) { return result.max(); }));
+    var overallMin = Statistics.min(results.map(function (result, index) { return result.min(); }));
+    var margin = (overallMax - overallMin) * 0.1;
+    var currentPlotOptions = $.extend(true, {}, mainPlotOptions, {yaxis: {
+        min: minIsZero ? 0 : overallMin - margin,
+        max: minIsZero ? overallMax * 1.1 : overallMax + margin}});
+
+    currentPlotOptions.xaxis.max = results.length - 0.5;
+    currentPlotOptions.xaxis.ticks = results.map(function (result, index) { return [index, result.run().label()]; });
+
+    $.plot(plotContainer, plotData, currentPlotOptions);
+}
+
+function toFixedWidthPrecision(value) {
+    var decimal = value.toFixed(2);
+    return decimal;
+}
+
+function formatPercentage(fraction) {
+    var percentage = fraction * 100;
+    return (fraction * 100).toFixed(2) + '%';
+}
+
+function createTable(tests, runs, shouldIgnoreMemory, referenceIndex) {
+    $('#container').html('<thead><tr><th>Test</th><th>Unit</th>' + runs.map(function (run, index) {
+        return '<th colspan="' + (index == referenceIndex ? 2 : 3) + '" class="{sorter: \'comparison\'}">' + run.label() + '</th>';
+    }).reduce(function (markup, cell) { return markup + cell; }, '') + '</tr></head><tbody></tbody>');
+
+    var testNames = [];
+    for (testName in tests)
+        testNames.push(testName);
+
+    testNames.sort().map(function (testName) {
+        var test = tests[testName];
+        if (test.isMemoryTest() != shouldIgnoreMemory)
+            createTableRow(runs, test, referenceIndex);
+    });
+
+    $('#container').tablesorter({widgets: ['zebra']});
+}
+
+function linearRegression(points) {
+    // Implement http://www.easycalculation.com/statistics/learn-correlation.php.
+    // x = magnitude
+    // y = iterations
+    var sumX = 0;
+    var sumY = 0;
+    var sumXSquared = 0;
+    var sumYSquared = 0;
+    var sumXTimesY = 0;
+
+    for (var i = 0; i < points.length; i++) {
+        var x = i;
+        var y = points[i];
+        sumX += x;
+        sumY += y;
+        sumXSquared += x * x;
+        sumYSquared += y * y;
+        sumXTimesY += x * y;
+    }
+
+    var r = (points.length * sumXTimesY - sumX * sumY) /
+        Math.sqrt((points.length * sumXSquared - sumX * sumX) *
+                  (points.length * sumYSquared - sumY * sumY));
+
+    if (isNaN(r) || r == Math.Infinity)
+        r = 0;
+
+    var slope = (points.length * sumXTimesY - sumX * sumY) / (points.length * sumXSquared - sumX * sumX);
+    var intercept = sumY / points.length - slope * sumX / points.length;
+    return {slope: slope, intercept: intercept, rSquared: r * r};
+}
+
+var warningSign = '<svg viewBox="0 0 100 100" style="width: 18px; height: 18px; vertical-align: bottom;" version="1.1">'
+    + '<polygon fill="red" points="50,10 90,80 10,80 50,10" stroke="red" stroke-width="10" stroke-linejoin="round" />'
+    + '<polygon fill="white" points="47,30 48,29, 50, 28.7, 52,29 53,30 50,60" stroke="white" stroke-width="10" stroke-linejoin="round" />'
+    + '<circle cx="50" cy="73" r="6" fill="white" />'
+    + '</svg>';
+
+function createTableRow(runs, test, referenceIndex) {
+    var tableRow = $('<tr><td class="test"' + (test.isImportant ? ' style="font-weight:bold"' : '') + '>' + test.name() + '</td><td class="unit">' + test.unit() + '</td></tr>');
+
+    function markupForRun(result, referenceResult) {
+        var comparisonCell = '';
+        var hiddenValue = '';
+        var shouldCompare = result !== referenceResult;
+        if (shouldCompare && referenceResult) {
+            var percentDifference = referenceResult.percentDifference(result);
+            var better = test.biggerIsBetter() ? percentDifference > 0 : percentDifference < 0;
+            var comparison = '';
+            var className = 'comparison';
+            if (referenceResult.isStatisticallySignificant(result)) {
+                comparison = formatPercentage(Math.abs(percentDifference)) + (better ? ' Better' : ' Worse&nbsp;');
+                className += better ? ' better' : ' worse';
+            }
+            hiddenValue = '<span style="display: none">|' + comparison + '</span>';
+            comparisonCell = '<td class="' + className + '">' + comparison + '</td>';
+        } else if (shouldCompare)
+            comparisonCell = '<td class="comparison"></td>';
+
+        var values = result.values();
+        var warning = '';
+        var regressionAnalysis = '';
+        if (values && values.length > 3) {
+            regressionResult = linearRegression(values);
+            regressionAnalysis = 'slope=' + toFixedWidthPrecision(regressionResult.slope)
+                + ', R^2=' + toFixedWidthPrecision(regressionResult.rSquared);
+            if (regressionResult.rSquared > 0.6 && Math.abs(regressionResult.slope) > 0.01) {
+                warning = ' <span class="regression-warning" title="Detected a time dependency with ' + regressionAnalysis + '">' + warningSign + ' </span>';
+            }
+        }
+
+        var statistics = '&sigma;=' + toFixedWidthPrecision(result.confidenceIntervalDelta()) + ', min=' + toFixedWidthPrecision(result.min())
+            + ', max=' + toFixedWidthPrecision(result.max()) + '\n' + regressionAnalysis;
+
+        // Tablesorter doesn't know about the second cell so put the comparison in the invisible element.
+        return '<td class="result" title="' + statistics + '">' + toFixedWidthPrecision(result.mean()) + hiddenValue
+            + '</td><td class="confidenceIntervalDelta" title="' + statistics + '">&plusmn; '
+            + formatPercentage(result.confidenceIntervalDeltaRatio()) + warning + '</td>' + comparisonCell;
+    }
+
+    function markupForMissingRun(isReference) {
+        return '<td colspan="' + (isReference ? 2 : 3) + '" class="missing">Missing</td>';
+    }
+
+    var runIndex = 0;
+    var results = test.results();
+    var referenceResult = undefined;
+    var resultIndexMap = {};
+    for (var i = 0; i < results.length; i++) {
+        while (runs[runIndex] !== results[i].run())
+            runIndex++;
+        if (runIndex == referenceIndex)
+            referenceResult = results[i];
+        resultIndexMap[runIndex] = i;
+    }
+    for (var i = 0; i < runs.length; i++) {
+        var resultIndex = resultIndexMap[i];
+        if (resultIndex == undefined)
+            tableRow.append(markupForMissingRun(i == referenceIndex));
+        else
+            tableRow.append(markupForRun(results[resultIndex], referenceResult));
+    }
+
+    $('#container').children('tbody').last().append(tableRow);
+
+    function toggle() {
+        var firstCell = tableRow.children('td').first();
+        if (firstCell.children('section').length) {
+            firstCell.children('section').remove();
+            tableRow.children('td').css({'padding-bottom': ''});
+        } else {
+            var plot = createPlot(firstCell, test);
+            plot.css({'position': 'absolute', 'z-index': 2});
+            var offset = tableRow.offset();
+            offset.left += 1;
+            offset.top += tableRow.outerHeight();
+            plot.offset(offset);
+            tableRow.children('td').css({'padding-bottom': plot.outerHeight() + 5});
+        }
+
+        return false;
+    };
+
+    tableRow.click(function(event) {
+        if (event.target != tableRow[0] && event.target.parentNode != tableRow[0])
+            return;
+
+        event.preventDefault();
+
+        toggle();
+    });
+
+    if (test.isImportant) {
+        toggle();
+    }
+}
+
+function init() {
+    $.tablesorter.addParser({
+        id: 'comparison',
+        is: function(s) {
+            return s.indexOf('|') >= 0;
+        },
+        format: function(s) {
+            var parsed = parseFloat(s.substring(s.indexOf('|') + 1));
+            return isNaN(parsed) ? 0 : parsed;
+        },
+        type: 'numeric',
+    });
+
+    var runs = [];
+    var metrics = {};
+    $.each(JSON.parse(document.getElementById('results-json').textContent), function (index, entry) {
+        var run = new TestRun(entry);
+        runs.push(run);
+
+        function addTests(tests, parentFullName) {
+            for (var testName in tests) {
+                var fullTestName = parentFullName + '/' + testName;
+                var rawMetrics = tests[testName].metrics;
+
+                for (var metricName in rawMetrics) {
+                    var fullMetricName = fullTestName + ':' + metricName;
+                    var metric = metrics[fullMetricName];
+                    if (!metric) {
+                        metric = new PerfTestMetric(fullTestName, metricName, rawMetrics[metricName].units, rawMetrics[metricName].important);
+                        metrics[fullMetricName] = metric;
+                    }
+                    metric.addResult(new TestResult(metric, rawMetrics[metricName].current, run));
+                }
+
+                if (tests[testName].tests)
+                    addTests(tests[testName].tests, fullTestName);
+            }
+        }
+
+        addTests(entry.tests, '');
+    });
+
+    var shouldIgnoreMemory= true;
+    var referenceIndex = 0;
+
+    createTable(metrics, runs, shouldIgnoreMemory, referenceIndex);
+
+    $('#time-memory').bind('change', function (event, checkedElement) {
+        shouldIgnoreMemory = checkedElement.textContent == 'Time';
+        createTable(metrics, runs, shouldIgnoreMemory, referenceIndex);
+    });
+
+    runs.map(function (run, index) {
+        $('#reference').append('<span value="' + index + '"' + (index == referenceIndex ? ' class="checked"' : '') + '>' + run.label() + '</span>');
+    })
+
+    $('#reference').bind('change', function (event, checkedElement) {
+        referenceIndex = parseInt(checkedElement.getAttribute('value'));
+        createTable(metrics, runs, shouldIgnoreMemory, referenceIndex);
+    });
+
+    $('.checkbox').each(function (index, checkbox) {
+        $(checkbox).children('span').click(function (event) {
+            if ($(this).hasClass('checked'))
+                return;
+            $(checkbox).children('span').removeClass('checked');
+            $(this).addClass('checked');
+            $(checkbox).trigger('change', $(this));
+        });
+    });
+}
+
+</script>
+<script id="results-json" type="application/json">%json_results%</script>
+<script id="units-json" type="application/json">%json_units%</script>
+</body>
+</html>
author	tonyg@chromium.org <tonyg@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98>	2013-08-04 21:33:29 +0000
committer	tonyg@chromium.org <tonyg@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98>	2013-08-04 21:33:29 +0000
commit	9694696cd263109e42565a0598adddee1bbf4bdc (patch)
tree	eaf6585deea4065a1f623a41aab74cff8b1b71b3 /tools/telemetry/support
parent	872e8e1683de933fe09e948ce42c2dfc6bcdf5b7 (diff)
download	chromium_src-9694696cd263109e42565a0598adddee1bbf4bdc.zip chromium_src-9694696cd263109e42565a0598adddee1bbf4bdc.tar.gz chromium_src-9694696cd263109e42565a0598adddee1bbf4bdc.tar.bz2