2014-09-16  Youenn Fablet  <youenn.fablet@crf.canon.fr>

        Add support for automated W3C conformance testing

        https://bugs.webkit.org/show_bug.cgi?id=134766

        Reviewed by NOBODY (OOPS!).

        * Scripts/run-w3c-conformance-tests: Added (main script).

        * Scripts/webkitpy/port/base.py:

        (Port.w3c_conformance_results_directory): Path to the working directory for conformance testing.

        * Scripts/webkitpy/w3c/run_w3c_conformance_tests.py: Added (Handle conformance test runnning using rwt).

        (W3CConformanceRunner):

        (W3CConformanceRunner.__init__):

        (W3CConformanceRunner.conformance_results_directory):

        (W3CConformanceRunner.layout_tests_result_directory):

        (W3CConformanceRunner.test_suites):

        (W3CConformanceRunner.run_webkit_tests):

        (W3CConformanceRunner.run_test_list):

        (W3CConformanceRunner.results_manager):

        (W3CConformanceRunner.tests):

        (W3CConformanceRunner.conformance_result):

        (W3CConformanceRunner.run_tests):

        (W3CConformanceRunner.print_results_to_console):

        (W3CConformanceRunner.print_potential_test_expectations_update):

        (W3CConformanceRunner.write_conformance_results):

        (configure_logging):

        (configure_logging.LogHandler):

        (configure_logging.LogHandler.format):

        (main):

        * Scripts/webkitpy/w3c/test_results_manager.py: Added (Parse rwt results to compute conformance results).

        (TestResultsManager):

        (TestResultsManager.__init__):

        (TestResultsManager.w3c_test_name):

        (TestResultsManager._get_test_results):

        (TestResultsManager.process_test_results):

        (TestResultsManager._compute_conformance_status):

        (TestResultsManager.compute_conformance_results):

        (TestResultsManager.summary):

        (TestResultsManager.write_conformance_results):

        * Scripts/webkitpy/w3c/web_platform_test_setup.py: Added (update wpt files, wpt testharness-report.js and wpt server config file).

        (WebPlatformTestSetup):

        (WebPlatformTestSetup.__init__):

        (WebPlatformTestSetup.restore_w3c_testharnessreport):

        (WebPlatformTestSetup.copy_webkit_testharnessreport):

        (WebPlatformTestSetup.copy_wpt_json_config):

#!/usr/bin/env python

# Copyright (c) 2014, Canon Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

# 3.  Neither the name of Canon Inc. nor the names of

#     its contributors may be used to endorse or promote products derived

#     from this software without specific prior written permission.

#

# THIS SOFTWARE IS PROVIDED BY CANON INC. AND ITS CONTRIBUTORS "AS IS" AND ANY

# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL CANON INC. AND ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Wrapper around webkitpy/w3c/run_w3c_conformance_tests.py"""

from webkitpy.common import multiprocessing_bootstrap

multiprocessing_bootstrap.run('webkitpy', 'w3c', 'run_w3c_conformance_tests.py')

    def w3c_conformance_results_directory(self):

        """Absolute path to the place to store the test results (uses --w3c-conformance-directory)."""

        return self._build_path('w3c-conformance-results')

# Copyright (c) 2014, Canon Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

# 3.  Neither the name of Canon Inc. nor the names of

#     its contributors may be used to endorse or promote products derived

#     from this software without specific prior written permission.

#

# THIS SOFTWARE IS PROVIDED BY CANON INC. AND ITS CONTRIBUTORS "AS IS" AND ANY

# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL CANON INC. AND ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""

 This script evaluates W3C tests conformance by:

    - importing W3C tests as WebKit tests (using web_platform_test_importer)

    - generating expected.txt files (using run-webkit-tests)

    - computing whether a test is passing or not by checking PASS/FAIL assertions outputted within the expected.txt files

"""

import argparse

import logging

import optparse

import sys

from webkitpy.common.host import Host

from webkitpy.common.system.executive import Executive

from webkitpy.layout_tests.models.test_expectations import TestExpectations

from webkitpy.port import platform_options

from webkitpy.w3c.factory import w3c_create_argument_parser

from webkitpy.w3c.test_results_manager import TestResultsManager

from webkitpy.w3c.web_platform_test_setup import WebPlatformTestSetup

from webkitpy.w3c.web_platform_test_importer import WebPlatformTestImporter

_log = logging.getLogger(__name__)

class W3CConformanceRunner(object):

    def __init__(self, port, options):

        self._port = port

        self._filesystem = port.host.filesystem

        self._options = options

        self._results_manager = None

        self._tests = None

        self._conformance_results = None

        self.web_platform_test_importer = WebPlatformTestImporter(port, options)

        self.imported_tests_directory = self.web_platform_test_importer.imported_tests_directory

        self.runned_tests_directory = self._filesystem.join(port.w3c_conformance_results_directory(), "runned-tests")

    def conformance_results_directory(self):

        return self._port.w3c_conformance_results_directory()

    def layout_tests_result_directory(self):

        return self._filesystem.join(self.conformance_results_directory(), "layout-test-results")

    def test_suites(self):

        return self.web_platform_test_importer.test_suites()

    def run_webkit_tests(self, parameters):

        tmp_args = [self._port.path_to_script('run-webkit-tests')] + parameters

        if self._options.platform:

            tmp_args.append('--' + self._options.platform)

        if self._options.webkit_test_runner is True:

            tmp_args.append('-2')

        Executive().run_command(tmp_args, error_handler=Executive.ignore_error, return_stderr=(self._options.__dict__['w3c_quiet']), decode_output=False)

    def run_test_list(self, tests):

        test_list_filename = self._filesystem.join(self.runned_tests_directory, 'test_list.txt')

        with open(test_list_filename, 'w+') as out:

            for test in tests:

                out.write(test + '\n')

            out.close()

        run_webkit_args = ['-q', '--no-retry-failures', '--no-show-results', '--full-results-html', '--results-directory=' + self.layout_tests_result_directory(), '--wptserver-doc-root=' + self.runned_tests_directory, '--test-list=' + test_list_filename]

        self.run_webkit_tests(run_webkit_args)

        self._results_manager = TestResultsManager(self._port, self.runned_tests_directory, self.layout_tests_result_directory(), self.tests())

        return self._results_manager.process_test_results()

    def results_manager(self):

        return self._results_manager

    def tests(self):

        if self._tests is None:

            test_suite_paths = [self._filesystem.join(self.runned_tests_directory, test_suite) for test_suite in self.test_suites()]

            self._tests = self._port.tests(test_suite_paths)

        return self._tests

    def conformance_result(self, test):

        if self._conformance_results is None:

            self._conformance_results = self.results_manager().compute_conformance_results()

        return self._conformance_results[self.results_manager().w3c_test_name(test)]

    def run_tests(self):

        if self._options.__dict__['w3c_list'] is True:

            return False

        if self._options.__dict__['w3c_refresh'] is True:

            _log.info('Importing tests...')

            self.web_platform_test_importer.import_tests()

        else:

            _log.info('Skipping test import...')

        _log.info('Setting up configuration files...')

        wpt_setup = WebPlatformTestSetup(self._port.host, self.imported_tests_directory)

        wpt_setup.copy_webkit_testharnessreport()

        wpt_setup.copy_wpt_json_config()

        _log.info('Copying imported_tests_directory as runned_test_directory')

        # A different directory, runned_test_directory, is used to run tests so that imported_tests_directory remains clean.

        # runned_test_directory is also recreated each time to start in a clean state.

        self._filesystem.rmtree(self.runned_tests_directory)

        self._filesystem.copytree(self.imported_tests_directory, self.runned_tests_directory)

        _log.info('Running tests...')

        self.run_test_list(self.tests())

        _log.info('Generating conformance results...')

        self._filesystem.maybe_make_directory(self.conformance_results_directory())

        self.write_conformance_results()

        return True

    def print_results_to_console(self):

        summary = self.results_manager().summary()

        _log.info('Run ' + str(summary['total']) + ' tests')

        _log.info('Pass: ' + str(summary['num_pass']))

        _log.info('Partial: ' + str(summary['num_partial']))

        _log.info('Fail: ' + str(summary['num_fail']))

        _log.info('Error: ' + str(summary['num_error']))

    def print_potential_test_expectations_update(self):

        test_expectations = TestExpectations(self._port).model()

        conformance = self.results_manager().compute_conformance_results()

        for test in self.tests():

            w3c_test_name = self.results_manager().w3c_test_name(test)

            wk_test_name = self._filesystem.join('imported', 'w3c', 'wpt', w3c_test_name)

            if not test_expectations.has_test(wk_test_name):

                if not conformance[w3c_test_name] == 'PASS':

                    _log.info('test ' + wk_test_name + ' is not passing')

            elif conformance[w3c_test_name] == 'PASS':

                _log.info('test ' + wk_test_name + ' is passing')

    def write_conformance_results(self):

        test_run_info = {}

        test_run_info['port_name'] = self._port.port_name

        test_run_info['tests_version'] = self.web_platform_test_importer.wptsha()

        test_run_info['results_directory'] = self._filesystem.relpath(self.layout_tests_result_directory(), self.conformance_results_directory())

        test_run_info['tests_directory'] = self._filesystem.relpath(self.runned_tests_directory, self.conformance_results_directory())

        self.results_manager().write_conformance_results(self.conformance_results_directory(), test_run_info)

def configure_logging():

    class LogHandler(logging.StreamHandler):

        def format(self, record):

            if record.levelno > logging.INFO:

                return "%s: %s" % (record.levelname, record.getMessage())

            return record.getMessage()

    logger = logging.getLogger()

    logger.setLevel(logging.INFO)

    handler = LogHandler()

    handler.setLevel(logging.INFO)

    logger.addHandler(handler)

    return handler

def main(argv, stdout, stderr):

    configure_logging()

    host = Host()

    parser = w3c_create_argument_parser(host, host.filesystem.basename(__file__))

    options, args = parser.parse_known_args(argv)

    port = host.port_factory.get(options.platform, optparse.Values({'webkit_test_runner': options.webkit_test_runner}))

    runner = W3CConformanceRunner(port, options)

    if runner.run_tests():

        runner.print_results_to_console()

        runner.print_potential_test_expectations_update()

        _log.info("Results exported in '" + runner.conformance_results_directory() + "'")

if '__main__' == __name__:

    sys.exit(main(sys.argv[1:], sys.stdout, sys.stderr))

# Copyright (c) 2014, Canon Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

# 3.  Neither the name of Canon Inc. nor the names of

#     its contributors may be used to endorse or promote products derived

#     from this software without specific prior written permission.

#

# THIS SOFTWARE IS PROVIDED BY CANON INC. AND ITS CONTRIBUTORS "AS IS" AND ANY

# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL CANON INC. AND ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""

 This script manage JSON and results.html files for W3C test results.

"""

import io

import json

from webkitpy.layout_tests.layout_package import json_results_generator

class TestResultsManager(object):

    _JSON_PREFIX = 'ADD_CONFORMANCE_RESULTS('

    _JSON_SUFFIX = ');'

    def __init__(self, port, base_directory, result_dir, tests):

        self._port = port

        self._filesystem = port.host.filesystem

        self._results_directory = result_dir

        self._tests = tests

        # Root directory of W3C imported test

        self._base_directory = base_directory

    def w3c_test_name(self, test):

        return self._filesystem.relpath(test, self._base_directory)

    def _get_test_results(self, tests, test_dict, path):

        for key in test_dict.keys():

            _path = self._filesystem.join(path, key)

            _potential_test = test_dict[key]

            if 'actual' in _potential_test:

                # Store the test

                tests[_path] = _potential_test

            else:

                # Process subdirectories

                self._get_test_results(tests, _potential_test, _path)

    def process_test_results(self):

        test_results = {}

        results_json_path = self._filesystem.join(self._results_directory, 'full_results.json')

        results_json = json_results_generator.load_json(self._filesystem, results_json_path)

        path = self._filesystem.sep if self._filesystem.isabs(self._base_directory) else ''

        self._get_test_results(test_results, results_json['tests'], path)

        return test_results

    def _compute_conformance_status(self, test_name, test_results):

        text = self._port.expected_text(test_name)

        if text is None or len(text) == 0:

            # if the empty expected file, test may be failing or a ref test.

            # Let's find out from json results if the test is failing.

            if test_name in test_results and 'actual' in test_results[test_name]:

                reason = test_results[test_name]['actual']

                if reason == 'CRASH' or reason == 'TIMEOUT':

                    return reason

                return 'ERROR'

            # TODO: We may want to validate that this is actually a ref test

            return 'PASS'

        # Let's check whether some lines start with PASS or FAIL

        # Test is said to PASS if at least one line is PASS an no line is FAIL

        hasFail = False

        hasPass = False

        buf = io.StringIO(unicode(text, "utf-8"))

        for line in buf.readlines():

            if line.startswith('PASS'):

                hasPass = True

            elif line.startswith('FAIL'):

                hasFail = True

        if hasPass:

            return 'PARTIAL' if hasFail else 'PASS'

        if hasFail:

            return 'FAIL'

        return 'ERROR'

    def compute_conformance_results(self):

        status = {}

        test_results = self.process_test_results()

        for test_name in self._tests:

            status[self.w3c_test_name(test_name)] = self._compute_conformance_status(test_name, test_results)

        return status

    def summary(self):

        num_pass = num_partial = num_fail = num_error = 0

        status = self.compute_conformance_results()

        for test_name in self._tests:

            result = status[self.w3c_test_name(test_name)]

            if result == 'PASS':

                num_pass += 1

            elif result == 'PARTIAL':

                num_partial += 1

            elif result == 'FAIL':

                num_fail += 1

            else:

                num_error += 1

        summary = {}

        summary['total'] = num_pass + num_fail + num_partial + num_error

        summary['num_pass'] = num_pass

        summary['num_fail'] = num_fail

        summary['num_partial'] = num_partial

        summary['num_error'] = num_error

        return summary

    def write_conformance_results(self, path, test_run_info):

        results = {}

        results['conformance_results'] = self.compute_conformance_results()

        results.update(test_run_info)

        # copy conformance results as json data

        json_string = json.dumps(results, separators=(',', ':'))

        json_string = self._JSON_PREFIX + json_string + self._JSON_SUFFIX

        self._filesystem.write_text_file(self._filesystem.join(path, 'w3c_conformance_results.json'), json_string)

        # copy html template

        results_file = self._port.path_from_webkit_base('LayoutTests', 'imported', 'w3c', 'resources', 'w3c_conformance_results.html')

        if self._filesystem.exists(results_file):

            self._filesystem.copyfile(results_file, self._filesystem.join(path, 'w3c_conformance_results.html'))

#!/usr/bin/env python

# Copyright (c) 2014, Canon Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

# 3.  Neither the name of Canon Inc. nor the names of

#     its contributors may be used to endorse or promote products derived

#     from this software without specific prior written permission.

#

# THIS SOFTWARE IS PROVIDED BY CANON INC. AND ITS CONTRIBUTORS "AS IS" AND ANY

# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL CANON INC. AND ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""

 This script allows setting up the web platform test parameters such as:

    - The config.json file used by the WPT http server

    - The testharnessreport.js used for web platform tests (copy of webkit testharnessreport.js).

"""

import logging

from webkitpy.common.host import Host

from webkitpy.common.webkit_finder import WebKitFinder

_log = logging.getLogger(__name__)

class WebPlatformTestSetup(object):

    def __init__(self, host, test_directory):

        self._filesystem = host.filesystem

        self._wpt_root = test_directory

        self._layout_root = self._filesystem.join(WebKitFinder(self._filesystem).webkit_base(), "LayoutTests")

    def restore_w3c_testharnessreport(self):

        _log.debug('Using W3C WPT testharnessreport.js')

        testharnessreport_wpt_filename = self._filesystem.join(self._wpt_root, "resources", "testharnessreport.js~")

        if self._filesystem.isfile(testharnessreport_wpt_filename):

            self._filesystem.copyfile(testharnessreport_wpt_filename, self._filesystem.join(self._wpt_root, "resources", "testharnessreport.js"))

            self._filesystem.remove(testharnessreport_wpt_filename)

    def copy_webkit_testharnessreport(self):

        _log.debug('Using WebKit testharnessreport.js')

        testharnessreport_wk_filename = self._filesystem.join(self._layout_root, "resources", "testharnessreport.js")

        if self._filesystem.isfile(testharnessreport_wk_filename):

            self._filesystem.copyfile(self._filesystem.join(self._wpt_root, "resources", "testharnessreport.js"), self._filesystem.join(self._wpt_root, "resources", "testharnessreport.js~"))

            self._filesystem.copyfile(self._filesystem.join(self._layout_root, "resources", "testharnessreport.js"), self._filesystem.join(self._wpt_root, "resources", "testharnessreport.js"))

    def copy_wpt_json_config(self):

        _log.debug('Copying json web platform server config')

        config_wpt_filename = self._filesystem.join(self._layout_root, "imported", "w3c", "resources", "config.json")

        if self._filesystem.isfile(config_wpt_filename):

            self._filesystem.copyfile(config_wpt_filename, self._filesystem.join(self._wpt_root, "config.json"))

2014-09-16  Youenn Fablet  <youenn.fablet@crf.canon.fr>

        Add support for automated W3C conformance testing

        https://bugs.webkit.org/show_bug.cgi?id=134766

        Reviewed by NOBODY (OOPS!).

        * resources/config.json: Added (test server config file).

        * resources/w3c_conformance_results.html: Added (conformance result template file).

{"host": "localhost",

 "ports":{"http":[8800, 8801],

          "https":[],

          "ws":[49001]},

 "check_subdomains": false,

 "log_level":"debug",

 "bind_hostname": false}

<!DOCTYPE html>

<html>

<head>

<style>

#results {

	display: table;

    padding: 20px;

	clear: both;

}

#summary {

    margin-top: 20px;

    margin-bottom: 20px;

}

.test, .firstline {

	display: table;

}

.testname, .testresult, .firstLine > div {

    background-color: lavender;

    border-bottom:solid 1px white;

}

.firstLineCell{

	display: table-cell;

	text-align: center;

	width: 800px;

}

.secondLineCell {

	display: table-cell;

	padding-left: 50px;

	width: 750px;

}

.testname {

	display: table-cell;

	width: 800px;

	margin-left:20px;

}

.testresult, .port {

	display: table-cell;

	width: 250px;

	text-align: center;

}

.testresult a {

    text-transform: capitalize;

}

.partial, .pass, .fail, .timeout, .error {

	text-align: center;

}

.pass a {

    color: green;

}

.partial a {

    color: yellow;

}

.fail a {

    color: orange;

}

.timeout a {

    color: red;

}

.crash a {

    color: red;

}

.error a {

    color: red;

}

.error-results-hist {

    background-color: red;

    float: left; 

}

.fail-results-hist {

    background-color: orange;

    float: left;

}

.partial-results-hist {

    background-color: yellow;

    float: left;

}

.pass-results-hist {

    background-color: green;

    float: left;

}

.module-name {

	display:table-cell;

	width: 800px;

}

.number-of-tests {

	display:inline-block;

	width: 50px;

}

.module-results {

	text-align: center;

	display:table-cell;

	width: 250px;

}

.histogram {

	display:inline-block;

	margin-left: 40px;

	margin-top: 5px;

	width:150px;

}

.module{

    background-color: lavender;

    border:solid 1px white;

	display:table;

}

.module-summary, .secondLine {

	display:table;

    background: lavender;

    border:solid 1px white;

    height: 25px;

    cursor:pointer;

}

.box{

	display: table;

}

.hidden-data {

    display: none

}

.legend > div {

	float: left;

	margin-right: 5px;

	min-width: 20px;

}

.legend {

}

</style>

<script>

function getTestsDirectory(results)

{

    return (results.root_directory ? results.root_directory : results.port_name) + "/" + 

            (results.tests_directory ? results.tests_directory : "imported-tests")

}

function getResultsDirectory(results)

{

    return (results.root_directory ? results.root_directory : results.port_name) + "/" + 

            (results.results_directory ? results.results_directory : "layout-test-results")

}

function getTestLink(testname, results)

{

	return "<div style='float:left; margin-right:10px'><a href='" + getTestsDirectory(results) + "/" + testname + "'>" + testname + "</a></div>"

		+ "<div>(<a href='https://github.com/w3c/web-platform-tests/blob/" + results.tests_version + "/" + testname + "'>github</a>)</div>" 

}

function getTestResultLink(testname, result, results)

{

	suffix = "." + testname.split('.').pop();

	return "<a href='" + getTestsDirectory(results) + "/" + testname.replace(suffix,"-expected.txt") + "'>" + result.toLowerCase() + "</a>"

}

function testSuiteHist(numError, numFail, numPartial, numPass)

{

    var total = numError + numPass + numPartial + numFail;

    html = '<div class="module-results">'

    html += '<div class="number-of-tests">' + numPass + '/' + total + '</div>'

	html += '<div class="histogram">'

    if (numError)

        html += '<div class="error-results-hist" style="width:' + (100 * numError / total) + '%">&nbsp;</div>'

    if (numFail)

        html += '<div class="fail-results-hist" style="width:' + (100 * numFail / total)  + '%">&nbsp;</div>'

    if (numPartial)

        html += '<div class="partial-results-hist" style="width:' + (100 * numPartial / total) + '%">&nbsp;</div>'

    if (numPass)

        html += '<div class="pass-results-hist" style="width:' + (100 * numPass / total) + '%">&nbsp;</div>'

    html += '</div>'

    return html + '</div>';

}

function getTestsuiteModules(results)

 {

	var modules = {}

	// first try to only show module from URL hash

	var hash = window.location.hash.substring(1)

	var hasTest = false

	if (hash.length)

		Object.keys(results.conformance_results)

			.sort()

			.forEach(function(key, i) {

				var testName = key

				if (testName.indexOf(hash) == 0) {

					hasTest = true

					var items = testName.split("/")

					var name = items[0]

					if (modules[name])

						modules[name].push(key)

					else

						modules[name] = [key]

				}

		})

	if (hasTest)

		return modules

	// if no tests, show all modules

    Object.keys(results.conformance_results)

        .sort()

        .forEach(function(key, i) {

			var testName = key

			var items = testName.split("/")

			var name = items[0]

			// In case of html module, we split again

			if (name == 'html')

				name += '/' + items[1]

			if (modules[name])

				modules[name].push(key)

			else

				modules[name] = [key]

	})

	return modules

 }

function toggleTests(div)

{

	var box = div.nextElementSibling

	if (box) {

		if (box.className == 'box')

            box.className ='hidden-data'

		else

			box.className = 'box'

	}

}

function computeGlobalConformance(results)

{

	var summary = {numPass: 0, numPartial: 0, numFail: 0, numError: 0}

	for (var test in results.conformance_results) {

		if (results.conformance_results.hasOwnProperty(test)) {

			result = results.conformance_results[test]

			if (result == "PASS") summary.numPass++

			else if (result == "PARTIAL") summary.numPartial++

			else if (result == "FAIL") summary.numFail++

			else summary.numError++

		}

	}

	return summary

}

function computeStatistics(resultsSummary, result, results)

{

	summary = resultsSummary[results.port_name]

	if (!summary)

		resultsSummary[results.port_name] = summary = {numPass: 0, numPartial: 0, numFail: 0, numError: 0}

	if (result == "PASS") summary.numPass++

	else if (result == "PARTIAL") summary.numPartial++

	else if (result == "FAIL") summary.numFail++

	else summary.numError++

}

function addTestSuiteListOfResults(moduleName, tests, listOfResults, isOpen)

{

	var resultsSummary =  {}

    var numPass = numPartial = numFail = numError = 0

    var innerHTML = ""

    tests.forEach(function(key, i) {

			innerHTML += "<div class='test'><div class='testname'>" + getTestLink(key, listOfResults[0]) + "</div>"

			listOfResults.forEach(function(results) {

				result = results.conformance_results[key]

                if (!result) {

                    result = "Missing"

                    console.log("WARNING: test " + key + " is missing");

                }

				innerHTML += "<div class='testresult " + result.toLowerCase() + "'>" + getTestResultLink(key, result, results) + "</div>"

				computeStatistics(resultsSummary, result, results)

			})

			innerHTML += "</div>"

       })

    var html = '<div class="module"><div class="module-summary" onclick="toggleTests(event.currentTarget)"><div class="module-name">' + moduleName + '</div>'

	listOfResults.forEach(function(results) {

		var summary = resultsSummary[results.port_name]

		html += testSuiteHist(summary.numError, summary.numFail, summary.numPartial, summary.numPass)

	})

	html += '</div><div class="' + (isOpen ? "box" : "hidden-data") + '">';

    html += innerHTML

    html += '</div></div>'

    document.getElementById("results").innerHTML += html

}

function writeSummary()

{

	var html = "<div>W3C conformance results from run-webkit-test ("

    listOfResults.forEach(function(results) {

        html += "<a href='" + getResultsDirectory(results) + "'>" + results.port_name + "</a> &nbsp;"

    })

    html += ")</div>"

	if (listOfResults.length > 1) {

		var sameTestVersion = true

		listOfResults.forEach(function(results) {

			if (listOfResults[0].tests_version != results.tests_version) {

				sameTestVersion = false

                html += "<div>Warning: W3C test version are not the same for all platforms: "

                html += listOfResults[0].tests_version + " vs " + results.tests_version + " of platform " + results.port_name

                console.log(results)

                html += "</div>"

			}

		})

		if (!sameTestVersion) {/*

			html += "<div>Warning: W3C test version are not the same for all platforms: "

            html += listOfResults[0].tests_version + " vs " + results.tests_version + " of platform " + results.port_name

            console.log(results)

            html += "</div>"*/

        }

		else if (listOfResults[0].tests_version == "master")

			html += "<div>Warning: using master as W3C test version</div>"

	}

    document.getElementById("summary").innerHTML = html

}

function displayResults()

{

    setTimeout(function()  {

	var firstLineHTML = "<div class='firstLine'><div class='firstLineCell'>Module/Test Name</div>"

	listOfResults.forEach(function(results) {

		firstLineHTML += "<div class='port'>" + results.port_name.toUpperCase() + "</div>"

	})

	firstLineHTML += "</div>"

	var secondLineHTML = "<div class='secondLine'><div class='secondLineCell'>Overall conformance</div>"

	listOfResults.forEach(function(results) {

		summary = computeGlobalConformance(results)

		secondLineHTML += testSuiteHist(summary.numError, summary.numFail, summary.numPartial, summary.numPass)

	})

	secondLineHTML += "</div>"

	document.getElementById("results").innerHTML = firstLineHTML + secondLineHTML

	var modules =  getTestsuiteModules(listOfResults[0])

	var keys = Object.keys(modules)

    keys.sort().forEach( function(key, i) {

		addTestSuiteListOfResults(key, modules[key], listOfResults, keys.length == 1)

	})

	writeSummary(listOfResults)

    }, 0)

}

listOfResults = [] 

function ADD_CONFORMANCE_RESULTS(results)

{

    results.root_directory = dirs[counter]

	listOfResults.push(results)

    setTimeout(loadScripts, 0)

}

//window.onload = displayResults

var counter

var dirs

function loadScripts(directories)

{

    if (directories) {

        dirs = directories

        counter = -1

    }

    if (counter < dirs.length - 1) {

        document.getElementById('summary').innerHTML = "Loading " + dirs[++counter] + "..."

        var head= document.getElementsByTagName('head')[0];

        var script= document.createElement('script');

        script.type= 'text/javascript';

        script.src= dirs[counter] + '/w3c_conformance_results.json';

        head.appendChild(script);

    }

    else {

        document.getElementById('summary').innerHTML = "Displaying results..."

        setTimeout(displayResults, 0)

    }

}

</script>

</head>

<body>

<div id="summary">

    Loading...

</div>

<div class="legend">

	<div class="error-results-hist legend">&nbsp;</div>

	<div>Error, Timeout or Crash</div>

	<div class="fail-results-hist legend">&nbsp;</div>

	<div> All assertions failing</div>

	<div class="partial-results-hist legend">&nbsp;</div>

	<div> Some assertions passing</div>

	<div class="pass-results-hist legend">&nbsp;</div>

	<div> All assertions passing</div>

</div>

<div id="results">

</div>

<!-- place as many .json file as needed -->

<script>

    setTimeout(function() {

        loadScripts([

            "."

        ])

    }, 0)

</script>

<!--script src="efl/w3c_conformance_results.json"></script>

<script src="gtk/w3c_conformance_results.json"></script>

<script src="mac/w3c_conformance_results.json"></script>

<script src="mac-wk2/w3c_conformance_results.json"></script-->

</body>

</html>