2015-01-13  Youenn Fablet  <youenn.fablet@crf.canon.fr>

        Fix for 134766

        Add support for automated W3C conformance testing

        https://bugs.webkit.org/show_bug.cgi?id=134766

        Reviewed by NOBODY (OOPS!).

        Scripts/check-w3c-conformance is the executable to test W3C test conformance.

        Results are stored by default in a WebKitBuild sub-folder (see Port.w3c_conformance_results_directory).

        Scripts/webkitpy/w3c/conformance_checker.py is the main script.

        It uses test importer to clone and import the tests.

        It then uses run-webkit-test to run the tests and generate the -expected.txt files for testharness tests.

        From the -expected.txt files, it computes whether the test is passing, partially passing or failing.

        Results are outputted through TestResultsManager as a json file that can be loaded using w3c_conformance_results.html.

        Draft Test expectations and import rules are also generated in the same folder.

        * Scripts/check-w3c-conformance: Added.

        * Scripts/webkitpy/port/base.py:

        (Port.w3c_conformance_results_directory):

        * Scripts/webkitpy/w3c/conformance_checker.py: Added.

        (create_argument_parser):

        (ConformanceChecker):

        (ConformanceChecker.__init__):

        (ConformanceChecker._print_rwt_output_line):

        (ConformanceChecker.run_tests):

        (ConformanceChecker.results_manager):

        (ConformanceChecker.init_test_importer):

        (ConformanceChecker.test_suites):

        (ConformanceChecker.tests):

        (ConformanceChecker.conformance_result):

        (ConformanceChecker.check_conformance):

        (ConformanceChecker.print_results_to_console):

        (ConformanceChecker.write_import_expectation_updates):

        (ConformanceChecker.write_test_expectation_updates):

        (ConformanceChecker.write_conformance_results):

        (configure_logging):

        (configure_logging.LogHandler):

        (configure_logging.LogHandler.format):

        (main):

        * Scripts/webkitpy/w3c/test_results_manager.py: Added.

        (TestResultsManager):

        (TestResultsManager.__init__):

        (TestResultsManager.w3c_test_name):

        (TestResultsManager._get_test_results):

        (TestResultsManager.process_test_results):

        (TestResultsManager._js_test_conformance):

        (TestResultsManager._conformance_status):

        (TestResultsManager.conformance_results):

        (TestResultsManager.summary):

        (TestResultsManager.write_conformance_results):

#!/usr/bin/env python

# Copyright (c) 2015, Canon Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

# 3.  Neither the name of Canon Inc. nor the names of

#     its contributors may be used to endorse or promote products derived

#     from this software without specific prior written permission.

#

# THIS SOFTWARE IS PROVIDED BY CANON INC. AND ITS CONTRIBUTORS "AS IS" AND ANY

# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL CANON INC. AND ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Wrapper around webkitpy/w3c/conformance_checker.py"""

from webkitpy.common import multiprocessing_bootstrap

multiprocessing_bootstrap.run('webkitpy', 'w3c', 'conformance_checker.py')

    def w3c_conformance_results_directory(self):

        """Absolute path to the place to store the test results (uses --w3c-conformance-directory)."""

        return self._build_path('w3c-conformance-results')

# Copyright (c) 2014, Canon Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

# 3.  Neither the name of Canon Inc. nor the names of

#     its contributors may be used to endorse or promote products derived

#     from this software without specific prior written permission.

#

# THIS SOFTWARE IS PROVIDED BY CANON INC. AND ITS CONTRIBUTORS "AS IS" AND ANY

# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL CANON INC. AND ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""

 This script evaluates W3C tests conformance by:

    - importing W3C tests as WebKit tests

    - generating expected.txt files (using run-webkit-tests)

    - computing whether a js test is passing or not by checking PASS/FAIL assertions outputted within the expected.txt files

"""

import argparse

import logging

import optparse

import sys

from webkitpy.common.host import Host

from webkitpy.common.system.executive import Executive

from webkitpy.layout_tests.models.test_expectations import TestExpectations

from webkitpy.port import platform_options

import webkitpy.w3c.test_importer as test_importer

from webkitpy.w3c.test_results_manager import TestResultsManager

from webkitpy.w3c.test_downloader import TestDownloader

_log = logging.getLogger(__name__)

def create_argument_parser():

    parser = argparse.ArgumentParser()

    for opt in platform_options(use_globs=False):

        parser.add_argument(opt.get_opt_string(), action=opt.action, dest=opt.dest, const=getattr(opt, 'const'), help=getattr(opt, 'help'))

    parser.add_argument('--webkit-test-runner', '-2', action='store_true', default=False,

        help='Set WebKit architecture WebKit1 or WebKit2 (default WK1).')

    parser.add_argument('-q', dest='quiet', action='store_true', default=False,

        help='Print minimal log.')

    parser.add_argument('-v', dest='verbose', action='store_true', default=False,

        help='Print maximal log.')

    parser.add_argument('-t', '--test-path', action='append', dest='test_paths', default=[],

        help='Import and run only tests in the supplied subdirectory of the w3c_test_directory. Can be supplied multiple times to give multiple paths')

    parser.add_argument('-s', '--skip-refresh', action='store_false', default=True, dest='refresh',

        help='Disable refresh of tests (git checkout  + import). By default, tests are refreshed.')

    parser.add_argument('--no-fetch', action='store_false', dest='fetch', default=True,

        help='Do not fetch the CSS/WPT repositories. By default, repositories are fetched if a W3c test directory is not provided')

    return parser

class ConformanceChecker(object):

    def __init__(self, port, options):

        self._port = port

        self._filesystem = port.host.filesystem

        self._options = options

        self._results_manager = None

        self._tests = None

        self._conformance_results = None

        self.layout_w3c_directory = self._filesystem.join('imported', 'w3c')

        self.results_directory = self._port.w3c_conformance_results_directory()

        self.layout_tests_result_directory = self._filesystem.join(self.results_directory, 'layout-test-results')

        self.imported_tests_directory = self._filesystem.join(self.results_directory, 'imported-tests')

        self.runned_tests_directory = self._filesystem.join(self.results_directory, 'runned-tests')

        self.init_test_importer()

    def _print_rwt_output_line(self, line):

        if not self._options.verbose:

            if line.startswith('['):

                tokens = line.split(' ')

                sys.stdout.write('\r' + tokens[0] + ' ' + tokens[1].replace(self.runned_tests_directory, '')[1:] + ' ')

                sys.stdout.flush()

            elif line == '':

                sys.stdout.write('\n')

        else:

            sys.stdout.write(line)

    def run_tests(self, tests):

        run_webkit_args = [self._port.path_to_script('run-webkit-tests'), '--no-retry-failures', '--no-show-results', '--full-results-html']

        test_list_filename = self._filesystem.join(self.runned_tests_directory, 'test_list.txt')

        self._filesystem.write_text_file(test_list_filename, '\n'.join(tests))

        run_webkit_args.append('--test-list=' + test_list_filename)

        if self._filesystem.exists(self.layout_tests_result_directory):

            self._filesystem.rmtree(self.layout_tests_result_directory)

        run_webkit_args.append('--results-directory=' + self.layout_tests_result_directory)

        run_webkit_args.append('--wptserver-doc-root=' + self._filesystem.join(self.runned_tests_directory, TestDownloader.WPT_NAME))

        if self._options.platform:

            run_webkit_args.append('--' + self._options.platform)

        if self._options.webkit_test_runner is True:

            run_webkit_args.append('-2')

        if not self._options.verbose:

            run_webkit_args.append('-q')

        process = Executive().popen(run_webkit_args, stdout=Executive.PIPE, stderr=Executive.STDOUT)

        while True:

            line = process.stdout.readline()

            self._print_rwt_output_line(line)

            if line == '':

                break

        process.wait()

    def results_manager(self):

        return self._results_manager

    def init_test_importer(self):

        args = ['-d', self.imported_tests_directory, '--ignore-expectations']

        for test_path in self._options.test_paths:

            args.append('-t')

            args.append(test_path)

        if not self._options.fetch:

            args.append('--no-fetch')

        if self._options.verbose:

            args.append('-v')

        elif self._options.quiet:

            logging.getLogger('webkitpy.w3c.test_importer').setLevel(logging.WARN)

            logging.getLogger('webkitpy.w3c.test_converter').setLevel(logging.WARN)

            logging.getLogger('webkitpy.w3c.test_downloader').setLevel(logging.WARN)

        options, args = test_importer.parse_args(args)

        self.test_importer = test_importer.TestImporter(self._port.host, None, options)

    def test_suites(self):

        return self._options.test_paths if self._options.test_paths else self.test_importer.test_downloader().all_test_paths()

    def tests(self):

        test_suites = self.test_suites()

        if self._tests is None:

            test_suite_paths = [self._filesystem.join(self.runned_tests_directory, test_suite) for test_suite in test_suites]

            self._tests = self._port.tests(test_suite_paths)

        return self._tests

    def conformance_result(self, test):

        if self._conformance_results is None:

            self._conformance_results = self.results_manager().conformance_results()

        return self._conformance_results[self.results_manager().w3c_test_name(test)]

    def check_conformance(self):

        if self._options.refresh is True:

            _log.info('Importing tests...')

            self.test_importer.do_import()

        else:

            _log.info('Skipping test import...')

        # Regenerate runned_test_directory to start in a clean state (no -expected.txt files in particular).

        self._filesystem.rmtree(self.runned_tests_directory)

        self._filesystem.copytree(self.imported_tests_directory, self.runned_tests_directory)

        _log.info('Running tests...')

        self.run_tests(self.tests())

        _log.info('Parsing test results...')

        self._results_manager = TestResultsManager(self._port, self.runned_tests_directory, self.layout_tests_result_directory, self.tests())

        self._results_manager.process_test_results()

        _log.info('Generating conformance results...')

        self._filesystem.maybe_make_directory(self.results_directory)

        self.write_conformance_results()

        _log.info('Generating expectation updates...')

        self.write_test_expectation_updates(self.results_directory)

        self.write_import_expectation_updates(self.results_directory)

        return True

    def print_results_to_console(self):

        summary = self.results_manager().summary()

        _log.info('Run ' + str(summary['total']) + ' tests')

        _log.info('Pass: ' + str(summary['num_pass']))

        _log.info('Partial: ' + str(summary['num_partial']))

        _log.info('Fail: ' + str(summary['num_fail']))

        _log.info('Error: ' + str(summary['num_error']))

    def write_import_expectation_updates(self, directory_path):

        import_expectations = []

        for test_suite in self.test_suites():

            import_expectations.append('# %s [ Pass ]' % self._filesystem.join(self.layout_w3c_directory, test_suite))

        import_expectations.append('')

        for test in self.tests():

            if not self.conformance_result(test) == 'PASS':

                import_expectations.append('%s [ Skip ]' % self._filesystem.join(self.layout_w3c_directory, self.results_manager().w3c_test_name(test)))

        self._filesystem.write_text_file(self._filesystem.join(directory_path, 'ImportExpectations'), '\n'.join(import_expectations))

    def write_test_expectation_updates(self, directory_path):

        test_expectations = ['# W3C test infrastructure folders']

        for path in self.test_importer.test_downloader().infrastructure_paths:

            file_path = self._filesystem.join(self.runned_tests_directory, path)

            if (self._filesystem.isdir(file_path)):

                test_expectations.append('%s [ Skip ]' % self._filesystem.join(self.layout_w3c_directory, path))

        self._filesystem.write_text_file(self._filesystem.join(directory_path, 'TestExpectations'), '\n'.join(test_expectations))

    def write_conformance_results(self):

        test_run_info = {}

        test_run_info['port_name'] = self._port.port_name

        test_run_info['web_platform_tests_version'] = self.test_importer.test_downloader().tests_version('WPT')

        test_run_info['csswg_tests_version'] = self.test_importer.test_downloader().tests_version('CSS')

        test_run_info['results_directory'] = self._filesystem.relpath(self.layout_tests_result_directory, self.results_directory)

        test_run_info['tests_directory'] = self._filesystem.relpath(self.runned_tests_directory, self.results_directory)

        self.results_manager().write_conformance_results(self.results_directory, test_run_info)

def configure_logging():

    class LogHandler(logging.StreamHandler):

        def format(self, record):

            if record.levelno > logging.INFO:

                return "%s: %s" % (record.levelname, record.getMessage())

            return record.getMessage()

    logger = logging.getLogger()

    logger.setLevel(logging.INFO)

    handler = LogHandler()

    handler.setLevel(logging.INFO)

    logger.addHandler(handler)

    return handler

def main(argv, stdout, stderr):

    configure_logging()

    host = Host()

    parser = create_argument_parser()

    options, args = parser.parse_known_args(argv)

    port = host.port_factory.get(options.platform, optparse.Values({'webkit_test_runner': options.webkit_test_runner}))

    runner = ConformanceChecker(port, options)

    if runner.check_conformance():

        runner.print_results_to_console()

        _log.info("Results exported in '" + runner.results_directory + "'")

if '__main__' == __name__:

    sys.exit(main(sys.argv[1:], sys.stdout, sys.stderr))

# Copyright (c) 2014, Canon Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

# 3.  Neither the name of Canon Inc. nor the names of

#     its contributors may be used to endorse or promote products derived

#     from this software without specific prior written permission.

#

# THIS SOFTWARE IS PROVIDED BY CANON INC. AND ITS CONTRIBUTORS "AS IS" AND ANY

# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL CANON INC. AND ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""

 This script manage JSON and results.html files for W3C test results.

"""

import json

from webkitpy.layout_tests.layout_package import json_results_generator

class TestResultsManager(object):

    _JSON_PREFIX = 'ADD_CONFORMANCE_RESULTS('

    _JSON_SUFFIX = ');'

    def __init__(self, port, base_directory, result_dir, tests):

        self._port = port

        self._filesystem = port.host.filesystem

        self._results_directory = result_dir

        self._tests = tests

        # Root directory of W3C imported test

        self._base_directory = base_directory

    def w3c_test_name(self, test):

        return self._filesystem.relpath(test, self._base_directory)

    def _get_test_results(self, tests, test_dict, path):

        for key in test_dict.keys():

            _path = self._filesystem.join(path, key)

            _potential_test = test_dict[key]

            # Is it a test entry or a sub-directory

            if 'report' in _potential_test:

                # Store the test

                tests[_path] = _potential_test

            else:

                self._get_test_results(tests, _potential_test, _path)

    def process_test_results(self):

        test_results = {}

        results_json_path = self._filesystem.join(self._results_directory, 'full_results.json')

        results_json = json_results_generator.load_json(self._filesystem, results_json_path)

        path = self._filesystem.sep if self._filesystem.isabs(self._base_directory) else ''

        self._get_test_results(test_results, results_json['tests'], path)

        return test_results

    def _js_test_conformance(self, expected_text):

        # Test is said to PASS if at least one line is PASS and no line is FAIL

        hasFail = False

        hasPass = False

        for line in expected_text.splitlines():

            if line.startswith('PASS'):

                hasPass = True

            elif line.startswith('FAIL'):

                hasFail = True

        if hasPass:

            return 'PASS' if not hasFail else 'PARTIAL'

        if hasFail:

            return 'FAIL'

        return 'ERROR'

    def _conformance_status(self, test_name, test_results):

        # FIXME: We should improve reftest/jstest detection.

        if not test_name in test_results:

            return {'type': 'reftest', 'status': 'PASS'}

        test_result = test_results[test_name]

        report = test_result['report']

        if report == 'MISSING':

            text = self._port.expected_text(test_name)

            if not(text is None or len(text) == 0):

                return {'type': 'jstest', 'status': self._js_test_conformance(text)}

        elif report == 'REGRESSION':

            return {'type': 'reftest', 'status': 'FAIL'}

        reason = test_result['actual']

        if reason == 'CRASH' or reason == 'TIMEOUT':

            return {'type': 'jstest', 'status': reason}

        return {'type': 'jstest', 'status': 'ERROR'}

    def conformance_results(self):

        test_wkr_results = self.process_test_results()

        results = {}

        for test_name in self._tests:

            results[self.w3c_test_name(test_name)] = self._conformance_status(test_name, test_wkr_results)

        return results

    def summary(self):

        num_pass = num_partial = num_fail = num_error = 0

        conformance_results = self.conformance_results()

        for key, value in conformance_results.iteritems():

            result = value['status']

            if result == 'PASS':

                num_pass += 1

            elif result == 'PARTIAL':

                num_partial += 1

            elif result == 'FAIL':

                num_fail += 1

            else:

                num_error += 1

        summary = {}

        summary['total'] = num_pass + num_fail + num_partial + num_error

        summary['num_pass'] = num_pass

        summary['num_fail'] = num_fail

        summary['num_partial'] = num_partial

        summary['num_error'] = num_error

        return summary

    def write_conformance_results(self, path, test_run_info):

        results = {}

        results['conformance_results'] = self.conformance_results()

        results.update(test_run_info)

        json_string = json.dumps(results, indent=4, sort_keys=True, separators=(',', ':'))

        json_string = self._JSON_PREFIX + json_string + self._JSON_SUFFIX

        self._filesystem.write_text_file(self._filesystem.join(path, 'w3c_conformance_results.json'), json_string)

        results_file = self._port.path_from_webkit_base('LayoutTests', 'imported', 'w3c', 'resources', 'w3c_conformance_results.html')

        if self._filesystem.exists(results_file):

            self._filesystem.copyfile(results_file, self._filesystem.join(path, 'w3c_conformance_results.html'))

2015-01-13  Youenn Fablet  <youenn.fablet@crf.canon.fr>

        Add support for automated W3C conformance testing

        https://bugs.webkit.org/show_bug.cgi?id=134766

        Reviewed by NOBODY (OOPS!).

        The template to display conformance results.

        * resources/w3c_conformance_results.html: Added.

<!DOCTYPE html>

<html>

<head>

<style>

#results {

    display: table;

    padding: 20px;

    clear: both;

}

#summary {

    margin-top: 20px;

    margin-bottom: 20px;

}

.test, .firstline {

    display: table;

}

.testname, .testresult, .firstLine > div {

    background-color: lavender;

    border-bottom:solid 1px white;

}

.firstLineCell{

    display: table-cell;

    text-align: center;

    width: 800px;

}

.secondLineCell {

    display: table-cell;

    padding-left: 50px;

    width: 750px;

}

.testname {

    display: table-cell;

    width: 900px;

    margin-left:20px;

}

.testnamelink {

    float:left;

    margin-right:10px;

}

.testresultlink  {

    float: left;

}

.testresultlink a {

    text-transform: capitalize;

}

.bugfilinglink  {

    text-align: right;

}

.port {

    display: table-cell;

    width: 250px;

    text-align: center;

}

.testresult {

    display: table-cell;

    width: 150px;

    text-align: center;

}

.testresult:not(:last-child) {

    padding-right: 100px;

}

.partial, .pass, .fail, .timeout, .error {

    text-align: center;

}

.pass a {

    color: green;

}

.partial a {

    color: yellow;

}

.fail a {

    color: orange;

}

.timeout a {

    color: red;

}

.crash a {

    color: red;

}

.error a {

    color: red;

}

.error-results-hist {

    background-color: red;

    float: left; 

}

.fail-results-hist {

    background-color: orange;

    float: left;

}

.partial-results-hist {

    background-color: yellow;

    float: left;

}

.pass-results-hist {

    background-color: green;

    float: left;

}

.module-name {

    display:table-cell;

    width: 800px;

}

.number-of-tests {

    display:inline-block;

    width: 50px;

}

.module-results {

    text-align: center;

    display:table-cell;

    width: 250px;

}

.histogram {

    display:inline-block;

    margin-left: 40px;

    margin-top: 5px;

    width:150px;

}

.module{

    background-color: lavender;

    border:solid 1px white;

    display:table;

}

.module-summary, .secondLine {

    display:table;

    background: lavender;

    border:solid 1px white;

    height: 25px;

    cursor:pointer;

}

.box{

    display: table;

}

.hidden-data {

    display: none;

}

.legend > div {

    float: left;

    margin-right: 5px;

    min-width: 20px;

}

.legend {

}

</style>

<script>

function testsDirectory(results)

{

    return (results.root_directory ? results.root_directory : results.port_name) + "/" + 

            (results.tests_directory ? results.tests_directory : "imported-tests")

}

function resultsDirectory(results)

{

    return (results.root_directory ? results.root_directory : results.port_name) + "/" + 

            (results.results_directory ? results.results_directory : "layout-test-results")

}

function githubTestURL(testname, results)

{

    return "https://github.com/w3c/web-platform-tests/blob/" + results.tests_version + "/" + testname

}

function bugzillaURL(title, comment)

{

    return "https://bugs.webkit.org/enter_bug.cgi?assigned_to=webkit-unassigned%40lists.webkit.org&product=WebKit&form_name=enter_bug&short_desc=" + title + "&comment=" + comment

}

function testFileLink(testname, results)

{

    return "<div class='testnamelink'><a href='" + testsDirectory(results) + "/" + testname + "'>" + testname + "</a></div>"

        + "<div class='githublink'>(<a href='" + githubTestURL(testname, results) + "'>github</a>)</div>" 

}

function testResultFileLink(testname, result, results)

{

    var suffix = "." + testname.split('.').pop()

    var new_suffix = "-expected.txt"

    result_status = result.status.toLowerCase().trim()

    if (result_status == "crash")

        new_suffix = "-crash-log.txt"

    if (result.type == "reftest")

        new_suffix = "-diffs.html"

    var link = "<div class='testresultlink'><a href='" + testsDirectory(results) + "/" + testname.replace(suffix, new_suffix) + "'>" + result_status + "</a></div>"

    if (result != "pass") {

        var title = encodeURIComponent("W3C " + testname + " test is " + result_status + "ing")

        var comment = encodeURIComponent(githubTestURL(testname, results) + " is " + result_status + "ing on " + results.port_name.toUpperCase() + ".")

        link += "<div class='bugfilinglink'><a href='" + bugzillaURL(title, comment) + "'>file bug</a></div>"

    }

    return link

}

function testSuiteStateHistogram(numError, numFail, numPartial, numPass)

{

    var total = numError + numPass + numPartial + numFail

    var html = '<div class="module-results">'

    html += '<div class="number-of-tests">' + numPass + '/' + total + '</div>'

    html += '<div class="histogram">'

    if (numError)

        html += '<div class="error-results-hist" style="width:' + (100 * numError / total) + '%">&nbsp;</div>'

    if (numFail)

        html += '<div class="fail-results-hist" style="width:' + (100 * numFail / total)  + '%">&nbsp;</div>'

    if (numPartial)

        html += '<div class="partial-results-hist" style="width:' + (100 * numPartial / total) + '%">&nbsp;</div>'

    if (numPass)

        html += '<div class="pass-results-hist" style="width:' + (100 * numPass / total) + '%">&nbsp;</div>'

    html += '</div>'

    return html + '</div>'

}

function testsuiteModules(results)

 {

    var modules = {}

    // first try to only show module from URL hash

    var hash = window.location.hash.substring(1)

    var hasTest = false

    if (hash.length) {

        Object.keys(results.conformance_results).sort().forEach(function(key, i) {

            var testName = key

            if (testName.indexOf(hash) == 0) {

                hasTest = true

                var items = testName.split("/")

                var name = items[0]

                if (modules[name])

                    modules[name].push(key)

                else

                    modules[name] = [key]

            }

        })

    }

    if (hasTest)

        return modules

    // If no tests, show all modules

    Object.keys(results.conformance_results).sort().forEach(function(key, i) {

        var testName = key

        var items = testName.split("/")

        var name = items[0]

        // In case of html module, we split display in sub-modules

        if (name == 'web-platform-tests' || name == 'csswg-tests' || name == 'html')

            name += '/' + items[1]

        if (modules[name])

            modules[name].push(key)

        else

            modules[name] = [key]

    })

    return modules

 }

function toggleTests(div)

{

    var box = div.nextElementSibling

    if (box)

        box.className = (box.className == 'box') ? 'hidden-data' : 'box'

}

function incrementSummary(result, summary)

{

    if (result.status == "PASS")

        summary.numPass++

    else if (result.status == "PARTIAL")

        summary.numPartial++

    else if (result.status == "FAIL")

        summary.numFail++

    else

        summary.numError++

}

function computeGlobalConformance(results)

{

    var summary = {numPass: 0, numPartial: 0, numFail: 0, numError: 0}

    for (var test in results.conformance_results) {

        if (results.conformance_results.hasOwnProperty(test))

            incrementSummary(results.conformance_results[test], summary)

    }

    return summary

}

function computeStatistics(resultsSummary, result, results)

{

    var summary = resultsSummary[results.port_name]

    if (!summary)

        resultsSummary[results.port_name] = summary = {numPass: 0, numPartial: 0, numFail: 0, numError: 0}

    incrementSummary(result, summary)

}

function addTestSuiteListOfResults(moduleName, tests, listOfResults, isOpen)

{

    var innerHTML = ""

    var resultsSummary =  {}

    tests.forEach(function(key, i) {

        innerHTML += "<div class='test'><div class='testname'>" + testFileLink(key, listOfResults[0]) + "</div>"

        listOfResults.forEach(function(results) {

            var result = results.conformance_results[key]

            if (!result) {

                result = "Missing"

                console.log("WARNING: test " + key + " is missing")

            }

            innerHTML += "<div class='testresult " + result.status.toLowerCase() + "'>" + testResultFileLink(key, result, results) + "</div>"

            computeStatistics(resultsSummary, result, results)

        })

        innerHTML += "</div>"

    })

    var html = '<div class="module"><div class="module-summary" onclick="toggleTests(event.currentTarget)"><div class="module-name">' + moduleName + '</div>'

    listOfResults.forEach(function(results) {

        var summary = resultsSummary[results.port_name]

        html += testSuiteStateHistogram(summary.numError, summary.numFail, summary.numPartial, summary.numPass)

    })

    html += '</div><div class="' + (isOpen ? "box" : "hidden-data") + '">'

    html += innerHTML

    html += '</div></div>'

    document.getElementById("results").innerHTML += html

}

function writeSummary()

{

    var html = "<div>W3C conformance results from run-webkit-test ("

    listOfResults.forEach(function(results) {

        html += "<a href='" + resultsDirectory(results) + "'>" + results.port_name + "</a> &nbsp;"

    })

    html += ")</div>"

    html += "<div> Generated <a href='TestExpectations'>test expectations</a> and <a href='ImportExpectations'>import expectations</a>./div>"

    if (listOfResults.length > 1) {

        var sameTestVersion = true

        listOfResults.forEach(function(results) {

            if (listOfResults[0].tests_version != results.tests_version) {

                sameTestVersion = false

                html += "<div>Warning: W3C test version are not the same for all platforms: "

                html += listOfResults[0].tests_version + " vs " + results.tests_version + " of platform " + results.port_name

                console.log(results)

                html += "</div>"

            }

        })

        if (!sameTestVersion) {

            html += "<div>Warning: W3C test version are not the same for all platforms: "

            html += listOfResults[0].tests_version + " vs " + results.tests_version + " of platform " + results.port_name

            console.log(results)

            html += "</div>"

        }

    }

    document.getElementById("summary").innerHTML = html

}

function displayResults()

{

    setTimeout(function()  {

        var firstLineHTML = "<div class='firstLine'><div class='firstLineCell'>Module/Test Name</div>"

        listOfResults.forEach(function(results) {

            firstLineHTML += "<div class='port'>" + results.port_name.toUpperCase() + "</div>"

        })

        firstLineHTML += "</div>"

        var secondLineHTML = "<div class='secondLine'><div class='secondLineCell'>Overall conformance</div>"

        listOfResults.forEach(function(results) {

            summary = computeGlobalConformance(results)

            secondLineHTML += testSuiteStateHistogram(summary.numError, summary.numFail, summary.numPartial, summary.numPass)

        })

        secondLineHTML += "</div>"

        document.getElementById("results").innerHTML = firstLineHTML + secondLineHTML

        var modules =  testsuiteModules(listOfResults[0])

        var keys = Object.keys(modules)

        keys.sort().forEach(function(key, i) {

            addTestSuiteListOfResults(key, modules[key], listOfResults, keys.length == 1)

        })

        writeSummary(listOfResults)

    }, 0)

}

listOfResults = [] 

function ADD_CONFORMANCE_RESULTS(results)

{

    results.root_directory = dirs[counter]

    listOfResults.push(results)

    setTimeout(loadScripts, 0)    

}

var counter

var dirs

function loadScripts(directories)

{

    if (directories) {

        dirs = directories

        counter = -1

    }

    if (counter < dirs.length - 1) {

        document.getElementById('summary').innerHTML = "Loading " + dirs[++counter] + "..."

        var head= document.getElementsByTagName('head')[0]

        var script= document.createElement('script')

        script.type= 'text/javascript'

        script.src= dirs[counter] + '/w3c_conformance_results.json'

        head.appendChild(script)

    }

    else {

        document.getElementById('summary').innerHTML = "Displaying results..."

        setTimeout(displayResults, 0)

    }

}

</script>

</head>

<body>

<div id="summary">

    Loading...

</div>

<div class="legend">

    <div class="error-results-hist legend">&nbsp;</div>

    <div>Error, Timeout or Crash</div>

    <div class="fail-results-hist legend">&nbsp;</div>

    <div> All assertions failing</div>

    <div class="partial-results-hist legend">&nbsp;</div>

    <div> Some assertions passing</div>

    <div class="pass-results-hist legend">&nbsp;</div>

    <div> All assertions passing</div>

</div>

<div id="results">

</div>

<!-- Add as many folder containing w3c_conformance_results.json as needed in loadScripts array to display and compare the results -->

<script>

    setTimeout(function() {

        loadScripts(["."])

    }, 0)

</script>

</body>

</html>