2015-03-30  Youenn Fablet  <youenn.fablet@crf.canon.fr>

        Add support for automated W3C conformance testing

        https://bugs.webkit.org/show_bug.cgi?id=134766

        Reviewed by NOBODY (OOPS!).

        Scripts/check-w3c-conformance is the executable to test W3C test conformance.

        Results are stored by default in a WebKitBuild sub-folder (see Port.w3c_conformance_results_directory).

        Scripts/webkitpy/w3c/conformance_checker.py is the main script.

        It uses test importer to clone and import the tests.

        It then uses run-webkit-test to run the tests and generate the -expected.txt files for testharness tests.

        From the -expected.txt files, it computes whether the test is passing, partially passing or failing.

        Results are outputted through TestResultsManager as a json file that can be loaded using w3c_conformance_results.html.

        Draft Test expectations and import rules are also generated in the same folder.

        * Scripts/check-w3c-conformance: Added.

        * Scripts/webkitpy/port/base.py:

        (Port.w3c_conformance_results_directory):

        * Scripts/webkitpy/w3c/conformance_checker.py: Added.

        (create_argument_parser):

        (ConformanceChecker):

        (ConformanceChecker.__init__):

        (ConformanceChecker._print_rwt_output_line):

        (ConformanceChecker.run_tests):

        (ConformanceChecker.init_test_importer):

        (ConformanceChecker.test_suites):

        (ConformanceChecker.check_conformance):

        (ConformanceChecker.print_summary_to_console):

        (ConformanceChecker.import_expectation_updates):

        (ConformanceChecker.test_expectation_updates):

        (ConformanceChecker.write_conformance_results):

        (TestResultsConformanceParser):

        (TestResultsConformanceParser.__init__):

        (TestResultsConformanceParser._get_test_results):

        (TestResultsConformanceParser._js_test_conformance):

        (TestResultsConformanceParser._conformance_status):

        (TestResultsConformanceParser.w3c_test_name):

        (TestResultsConformanceParser.compute_conformance_results):

        (configure_logging):

        (configure_logging.LogHandler):

        (configure_logging.LogHandler.format):

        (main):

        * Scripts/webkitpy/w3c/conformance_checker_unittest.py: Added.

        (ConformancCheckerTest):

        (ConformancCheckerTest.setUp):

        (ConformancCheckerTest.write_expected_file):

        (ConformancCheckerTest.test_w3c_name_conversion):

        (ConformancCheckerTest.test_json_result_parsing):

        (ConformancCheckerTest.test_missing_test):

        (ConformancCheckerTest.test_ref_test_timeout):

        (ConformancCheckerTest.test_ref_test_crash):

        (ConformancCheckerTest.test_ref_test_fail):

        (ConformancCheckerTest.test_js_test_pass):

        (ConformancCheckerTest.test_js_test_partial):

        (ConformancCheckerTest.test_js_test_fail):

        (ConformancCheckerTest.test_js_test_crash):

        (ConformancCheckerTest.test_js_test_timeout):

        (ConformancCheckerTest.test_import_expectations_update):

        (ConformancCheckerTest.test_test_expectations_update):

        * Scripts/webkitpy/w3c/test_downloader.py:

        (TestDownloader.all_test_paths):

#!/usr/bin/env python

# Copyright (c) 2015, Canon Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

# 3.  Neither the name of Canon Inc. nor the names of

#     its contributors may be used to endorse or promote products derived

#     from this software without specific prior written permission.

#

# THIS SOFTWARE IS PROVIDED BY CANON INC. AND ITS CONTRIBUTORS "AS IS" AND ANY

# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL CANON INC. AND ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Wrapper around webkitpy/w3c/conformance_checker.py"""

from webkitpy.common import multiprocessing_bootstrap

multiprocessing_bootstrap.run('webkitpy', 'w3c', 'conformance_checker.py')

    def w3c_conformance_results_directory(self):

        """Absolute path to the place to store the W3C conformance results."""

        return self._build_path('w3c-conformance-results')

# Copyright (c) 2015, Canon Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

# 3.  Neither the name of Canon Inc. nor the names of

#     its contributors may be used to endorse or promote products derived

#     from this software without specific prior written permission.

#

# THIS SOFTWARE IS PROVIDED BY CANON INC. AND ITS CONTRIBUTORS "AS IS" AND ANY

# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL CANON INC. AND ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""

 This script evaluates W3C tests conformance by:

    - importing W3C tests as WebKit tests

    - generating expected.txt files (using run-webkit-tests)

    - computing whether a js test is passing or not by checking PASS/FAIL assertions outputted within the expected.txt files

"""

import argparse

import json

import logging

import optparse

import sys

from webkitpy.common.host import Host

from webkitpy.common.system.executive import Executive

from webkitpy.layout_tests.layout_package import json_results_generator

from webkitpy.layout_tests.models.test_expectations import TestExpectations

from webkitpy.port import configuration_options, platform_options

from webkitpy.w3c.test_importer import TestImporter

from webkitpy.w3c.test_importer import parse_args as parse_importer_args

from webkitpy.w3c.test_downloader import TestDownloader

_log = logging.getLogger(__name__)

def create_argument_parser():

    parser = argparse.ArgumentParser()

    for opt in platform_options(use_globs=False):

        parser.add_argument(opt.get_opt_string(), action=opt.action, dest=opt.dest, const=getattr(opt, 'const'), help=getattr(opt, 'help'))

    parser.add_argument('--dump-render-tree', '-1', dest='webkit_test_runner', action='store_false', default=True,

        help='Set WebKit architecture WebKit1 or WebKit2 (default WK2).')

    parser.add_argument('--debug', dest='configuration', action='store_const', const='Debug', default='Release',

        help='Use "Debug" configuration. Default is "Release"')

    parser.add_argument('-q', dest='quiet', action='store_true', default=False,

        help='Print minimal log.')

    parser.add_argument('-v', dest='verbose', action='store_true', default=False,

        help='Print maximal log.')

    parser.add_argument('-t', '--test-path', action='append', dest='test_paths', default=[],

        help='Import and run only tests in the supplied subdirectory of the w3c_test_directory. Can be supplied multiple times to give multiple paths')

    parser.add_argument('-s', '--skip-refresh', action='store_false', default=True, dest='refresh',

        help='Disable refresh of tests (git checkout  + import). By default, tests are refreshed.')

    parser.add_argument('--no-fetch', action='store_false', dest='fetch', default=True,

        help='Do not fetch the CSS/WPT repositories. By default, repositories are fetched.')

    return parser

class ConformanceChecker(object):

    def __init__(self, port, options):

        self._port = port

        self._filesystem = port.host.filesystem

        self._options = options

        self._conformance_results = None

        self.WPT_TESTSUITE = 'web-platform-tests'

        self.layout_w3c_directory = self._filesystem.join('imported', 'w3c')

        self.results_directory = self._port.w3c_conformance_results_directory()

        self.layout_tests_result_directory = self._filesystem.join(self.results_directory, 'layout-test-results')

        self.imported_tests_directory = self._filesystem.join(self.results_directory, 'imported-tests')

        self.runned_tests_directory = self._filesystem.join(self.results_directory, 'runned-tests')

        self.init_test_importer()

    def _print_rwt_output_line(self, line):

        if not self._options.verbose:

            if line.startswith('['):

                tokens = line.split(' ')

                sys.stdout.write('\r' + tokens[0] + ' ' + tokens[1].replace(self.runned_tests_directory, '')[1:] + ' ')

                sys.stdout.flush()

            elif line == '':

                sys.stdout.write('\n')

        else:

            sys.stdout.write(line)

    def run_tests(self, tests):

        run_webkit_args = [self._port.path_to_script('run-webkit-tests'), '--no-retry-failures', '--no-show-results', '--full-results-html']

        test_list_filename = self._filesystem.join(self.runned_tests_directory, 'test_list.txt')

        self._filesystem.write_text_file(test_list_filename, '\n'.join(tests))

        run_webkit_args.append('--test-list=' + test_list_filename)

        if self._filesystem.exists(self.layout_tests_result_directory):

            self._filesystem.rmtree(self.layout_tests_result_directory)

        run_webkit_args.append('--results-directory=' + self.layout_tests_result_directory)

        run_webkit_args.append('--wptserver-doc-root=' + self._filesystem.join(self.runned_tests_directory, self.WPT_TESTSUITE))

        if self._options.configuration:

            run_webkit_args.append('--target=' + self._options.configuration)

        if self._options.platform:

            run_webkit_args.append('--platform=' + self._options.platform)

        if not self._options.webkit_test_runner:

            run_webkit_args.append('-1')

        if not self._options.verbose:

            run_webkit_args.append('-q')

        process = Executive().popen(run_webkit_args, stdout=Executive.PIPE, stderr=Executive.STDOUT)

        while True:

            line = process.stdout.readline()

            self._print_rwt_output_line(line)

            if line == '':

                break

        process.wait()

    def init_test_importer(self):

        args = ['-d', self.imported_tests_directory, '--ignore-expectations']

        for test_path in self._options.test_paths:

            args.append('-t')

            args.append(test_path)

        if not self._options.fetch:

            args.append('--no-fetch')

        if self._options.verbose:

            args.append('-v')

        elif self._options.quiet:

            logging.getLogger('webkitpy.w3c.test_importer').setLevel(logging.WARN)

            logging.getLogger('webkitpy.w3c.test_converter').setLevel(logging.WARN)

            logging.getLogger('webkitpy.w3c.test_downloader').setLevel(logging.WARN)

        options, args = parse_importer_args(args)

        self.test_importer = TestImporter(self._port.host, None, options)

    def test_suites(self):

        return self._options.test_paths if self._options.test_paths else self.test_importer.test_downloader().all_test_paths()

    def check_conformance(self):

        if self._options.refresh is True:

            _log.info('Importing tests...')

            self._filesystem.rmtree(self.imported_tests_directory)

            self.test_importer.do_import()

            # FIXME: We should install WPT modules in imported tests directory to not reinstall them for each run.

        else:

            _log.info('Skipping test import...')

        # Regenerate runned_test_directory to start in a clean state (no -expected.txt files in particular).

        self._filesystem.rmtree(self.runned_tests_directory)

        self._filesystem.copytree(self.imported_tests_directory, self.runned_tests_directory)

        _log.info('Running tests...')

        tests = self._port.tests([self._filesystem.join(self.runned_tests_directory, test_suite) for test_suite in self.test_suites()])

        self.run_tests(tests)

        _log.info('Parsing test results...')

        json_file_path = self._filesystem.join(self.layout_tests_result_directory, 'full_results.json')

        if not self._filesystem.exists(json_file_path):

            _log.error('Cannot read run-webkit-tests result file "%s". run-webkit-tests may have run into some errors. Use -v option\nAborting...' % json_file_path)

            return False

        self._conformance_results = TestResultsConformanceParser(self._port, self.runned_tests_directory).compute_conformance_results(tests, json_file_path)

        _log.info('Generating conformance results...')

        self._filesystem.maybe_make_directory(self.results_directory)

        self.write_conformance_results()

        _log.info('Generating expectation updates...')

        self._filesystem.write_text_file(self._filesystem.join(self.results_directory, 'TestExpectations'), '\n'.join(self.test_expectation_updates()))

        self._filesystem.write_text_file(self._filesystem.join(self.results_directory, 'ImportExpectations'), '\n'.join(self.import_expectation_updates()))

        return True

    def print_summary_to_console(self):

        num_pass = num_partial = num_fail = num_error = 0

        for key, value in self._conformance_results.iteritems():

            result = value['status']

            if result == 'PASS':

                num_pass += 1

            elif result == 'PARTIAL':

                num_partial += 1

            elif result == 'FAIL':

                num_fail += 1

            else:

                num_error += 1

        _log.info('Run ' + str(num_pass + num_fail + num_partial + num_error) + ' tests')

        _log.info('Pass: ' + str(num_pass))

        _log.info('Partial: ' + str(num_partial))

        _log.info('Fail: ' + str(num_fail))

        _log.info('Error: ' + str(num_error))

    def import_expectation_updates(self):

        import_expectations = []

        for test_suite in self.test_suites():

            import_expectations.append('# %s [ Pass ]' % self._filesystem.join(self.layout_w3c_directory, test_suite))

        import_expectations.append('')

        for test in self._conformance_results.keys():

            status = self._conformance_results[test]['status']

            if not(status == 'PASS' or status == 'PARTIAL'):

                import_expectations.append('%s [ Skip ]' % test)

        return import_expectations

    def test_expectation_updates(self):

        test_expectations = ['# W3C test infrastructure folders']

        for test_repository in self.test_importer.test_downloader().test_repositories:

            for path in test_repository['paths_to_import']:

                file_path = self._filesystem.join(self.runned_tests_directory, path)

                if (self._filesystem.isdir(file_path)):

                    test_expectations.append('%s [ Skip ]' % self._filesystem.join(self.layout_w3c_directory, path))

        test_expectations.append('# Partially passing tests')

        for test in self._conformance_results.keys():

            if self._conformance_results[test]['status'] == 'PARTIAL':

                test_expectations.append('%s [ Pass ]' % self._filesystem.join(self.layout_w3c_directory, test))

        return test_expectations

    def write_conformance_results(self):

        _JSON_PREFIX = 'ADD_CONFORMANCE_RESULTS('

        _JSON_SUFFIX = ');'

        test_run_info = {}

        test_run_info['port_name'] = self._port.port_name

        test_run_info['test_repositories'] = self.test_importer.test_downloader().test_repositories

        test_run_info['results_directory'] = self._filesystem.relpath(self.layout_tests_result_directory, self.results_directory)

        test_run_info['tests_directory'] = self._filesystem.relpath(self.runned_tests_directory, self.results_directory)

        results = {}

        results['conformance_results'] = self._conformance_results

        results.update(test_run_info)

        json_string = json.dumps(results, indent=4, sort_keys=True, separators=(',', ':'))

        json_string = _JSON_PREFIX + json_string + _JSON_SUFFIX

        self._filesystem.write_text_file(self._filesystem.join(self.results_directory, 'w3c_conformance_results.json'), json_string)

        results_file = self._port.path_from_webkit_base('LayoutTests', 'imported', 'w3c', 'resources', 'w3c_conformance_results.html')

        if not self._filesystem.exists(results_file):

            _log.warning('w3c_conformance_results.html template file cannot be found and copied to conformance result directory')

            return

        self._filesystem.copyfile(results_file, self._filesystem.join(self.results_directory, 'w3c_conformance_results.html'))

class TestResultsConformanceParser(object):

    def __init__(self, port, base_directory):

        self._port = port

        self._filesystem = port.host.filesystem

        self._base_directory = base_directory

    def _get_test_results(self, tests, test_dict, path):

        for key in test_dict.keys():

            _path = self._filesystem.join(path, key)

            _potential_test = test_dict[key]

            # Is it a test entry or a sub-directory

            if 'report' in _potential_test:

                # Store the test

                tests[_path] = _potential_test

            else:

                self._get_test_results(tests, _potential_test, _path)

    def _js_test_conformance(self, expected_text):

        # Test is said to PASS if at least one line is PASS and no line is FAIL

        hasFail = False

        hasPass = False

        for line in expected_text.splitlines():

            if line.startswith('PASS'):

                hasPass = True

            elif line.startswith('FAIL'):

                hasFail = True

        if hasPass:

            return 'PASS' if not hasFail else 'PARTIAL'

        if hasFail:

            return 'FAIL'

        return 'ERROR'

    def _conformance_status(self, test_name, test_results):

        if not test_name in test_results:

            return {'type': 'reftest', 'status': 'PASS'}

        test_result = test_results[test_name]

        test_type = 'jstest' if ('is_missing_text' in test_result and test_result['is_missing_text']) else 'reftest'

        reason = test_result['actual']

        if reason == 'CRASH' or reason == 'TIMEOUT':

            return {'type': test_type, 'status': reason}

        report = test_result['report']

        if report == 'MISSING':

            text = self._port.expected_text(test_name)

            if not(text is None or len(text) == 0):

                return {'type': test_type, 'status': self._js_test_conformance(text)}

        elif report == 'REGRESSION':

            return {'type': test_type, 'status': 'FAIL'}

        return {'type': test_type, 'status': 'ERROR'}

    def w3c_test_name(self, test):

        return self._filesystem.relpath(test, self._base_directory)

    def compute_conformance_results(self, tests, json_results_filepath):

        test_results = {}

        results_json = json_results_generator.load_json(self._filesystem, json_results_filepath)

        self._get_test_results(test_results, results_json['tests'], self._filesystem.sep if self._filesystem.isabs(self._base_directory) else '')

        results = {}

        for test_name in tests:

            results[self.w3c_test_name(test_name)] = self._conformance_status(test_name, test_results)

        return results

def configure_logging():

    class LogHandler(logging.StreamHandler):

        def format(self, record):

            if record.levelno > logging.INFO:

                return "%s: %s" % (record.levelname, record.getMessage())

            return record.getMessage()

    logger = logging.getLogger()

    logger.setLevel(logging.INFO)

    handler = LogHandler()

    handler.setLevel(logging.INFO)

    logger.addHandler(handler)

    return handler

def main(argv, stdout, stderr):

    configure_logging()

    host = Host()

    parser = create_argument_parser()

    options, args = parser.parse_known_args(argv)

    # FIXME: We should pass all argparse options to the port factory.

    port = host.port_factory.get(options.platform, optparse.Values({'configuration': options.configuration, 'webkit_test_runner': options.webkit_test_runner}))

    runner = ConformanceChecker(port, options)

    if runner.check_conformance():

        runner.print_summary_to_console()

        _log.info("Results exported in '" + runner.results_directory + "'")

if '__main__' == __name__:

    sys.exit(main(sys.argv[1:], sys.stdout, sys.stderr))

# Copyright (c) 2015, Canon Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

# 3.  Neither the name of Canon Inc. nor the names of

#     its contributors may be used to endorse or promote products derived

#     from this software without specific prior written permission.

#

# THIS SOFTWARE IS PROVIDED BY CANON INC. AND ITS CONTRIBUTORS "AS IS" AND ANY

# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL CANON INC. AND ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import unittest

from webkitpy.w3c.conformance_checker import TestResultsConformanceParser, ConformanceChecker, create_argument_parser

from webkitpy.port.base import Port

from webkitpy.common.system.filesystem_mock import MockFileSystem

from webkitpy.common.host_mock import MockHost

class ConformancCheckerTest(unittest.TestCase):

    def setUp(self):

        self.base_dir = '/tmp/conformance-directory/w3c-conformance-results/runned-tests'

        self.host = MockHost()

        self.fs = self.host.filesystem

        self.port = Port(self.host, 'MockPort')

        self.w3c_tests = ['test1/pass.html', 'missing.html', 'pass.html', 'fail.html', 'partial.html', 'crash.html', 'timeout.html', 'ref_pass.html', 'ref_fail.html', 'ref_crash.html', 'ref_timeout.html']

        self.runned_tests = [self.fs.join(self.base_dir, test) for test in self.w3c_tests]

        self.json_file_path = self.fs.join(self.base_dir, 'results_dir', 'results.json')

        self.fs.write_text_file(self.json_file_path,

                'ADD_RESULTS({"tests":{"":{"tmp":{"conformance-directory":{"w3c-conformance-results":{"runned-tests":{"test1":{"pass.html":{"report":"MISSING","expected":"PASS","is_missing_text":true,"actual":"MISSING"}},"pass.html":{"report":"MISSING","expected":"PASS","is_missing_text":true,"actual":"MISSING"},"fail.html":{"report":"MISSING","expected":"PASS","is_missing_text":true,"actual":"MISSING"},"partial.html":{"report":"MISSING","expected":"PASS","is_missing_text":true,"actual":"MISSING"},"crash.html":{"report":"MISSING","expected":"PASS","is_missing_text":true,"actual":"CRASH"},"timeout.html":{"report":"REGRESSION","expected":"PASS","is_missing_text":true,"actual":"TIMEOUT"}, "ref_fail.html":{"report":"REGRESSION","expected":"PASS","actual":"IMAGE"}, "ref_crash.html":{"report":"REGRESSION","expected":"PASS","actual":"CRASH"}, "ref_timeout.html":{"report":"REGRESSION","expected":"PASS","actual":"TIMEOUT"}}}}}}},"skipped":0,"num_regressions":1,"interrupted":false,"num_missing":6,"layout_tests_dir":"LayoutTests","version":3,"num_passes":0,"pixel_tests_enabled":false,"has_pretty_patch":true,"fixable":10,"num_flaky":0,"uses_expectations_file":true});')

        self.write_expected_file('test1/pass.html', 'PASS this is a passing test\n')

        self.write_expected_file('pass.html', 'CONSOLE: this is a warning line\nPASS this is a passing test\n')

        self.write_expected_file('fail.html', 'CONSOLE: this is a warning line\nFAIL this is a failing test\n')

        self.write_expected_file('partial.html', 'CONSOLE: this is a warning line\nPASS this is a passing test\nFAIL this is a failing test\n')

        self.parser = TestResultsConformanceParser(self.port, self.base_dir)

        self.conformance_results = self.parser.compute_conformance_results(self.runned_tests, self.json_file_path)

        options, args = create_argument_parser().parse_known_args(['-t .'])

        self.conformance_checker = ConformanceChecker(self.port, options)

        self.conformance_checker._conformance_results = self.conformance_results

    def write_expected_file(self, test, result):

        self.fs.write_text_file(self.port.expected_filename(self.fs.join(self.base_dir, test), '.txt'), result)

    def test_w3c_name_conversion(self):

        self.assertEquals(self.parser.w3c_test_name(self.runned_tests[0]), self.w3c_tests[0])

    def test_json_result_parsing(self):

        self.assertItemsEqual(self.conformance_results.keys(), self.w3c_tests)

    def test_missing_test(self):

        self.assertEquals(self.conformance_results['missing.html']['type'], 'reftest')

        self.assertEquals(self.conformance_results['missing.html']['status'], 'PASS')

    def test_ref_test_timeout(self):

        self.assertEquals(self.conformance_results['ref_timeout.html']['type'], 'reftest')

        self.assertEquals(self.conformance_results['ref_timeout.html']['status'], 'TIMEOUT')

    def test_ref_test_crash(self):

        self.assertEquals(self.conformance_results['ref_crash.html']['type'], 'reftest')

        self.assertEquals(self.conformance_results['ref_crash.html']['status'], 'CRASH')

    def test_ref_test_fail(self):

        self.assertEquals(self.conformance_results['ref_fail.html']['type'], 'reftest')

        self.assertEquals(self.conformance_results['ref_fail.html']['status'], 'FAIL')

    def test_js_test_pass(self):

        self.assertEquals(self.conformance_results['pass.html']['type'], 'jstest')

        self.assertEquals(self.conformance_results['pass.html']['status'], 'PASS')

        self.assertEquals(self.conformance_results['test1/pass.html']['type'], 'jstest')

        self.assertEquals(self.conformance_results['test1/pass.html']['status'], 'PASS')

    def test_js_test_partial(self):

        self.assertEquals(self.conformance_results['partial.html']['type'], 'jstest')

        self.assertEquals(self.conformance_results['partial.html']['status'], 'PARTIAL')

    def test_js_test_fail(self):

        self.assertEquals(self.conformance_results['fail.html']['type'], 'jstest')

        self.assertEquals(self.conformance_results['fail.html']['status'], 'FAIL')

    def test_js_test_crash(self):

        self.assertEquals(self.conformance_results['crash.html']['type'], 'jstest')

        self.assertEquals(self.conformance_results['crash.html']['status'], 'CRASH')

    def test_js_test_timeout(self):

        self.assertEquals(self.conformance_results['timeout.html']['type'], 'jstest')

        self.assertEquals(self.conformance_results['timeout.html']['status'], 'TIMEOUT')

    def test_import_expectations_update(self):

        expectations = '\n'.join(self.conformance_checker.import_expectation_updates())

        self.assertTrue('timeout.html [ Skip ]' in expectations)

        self.assertTrue('crash.html [ Skip ]' in expectations)

        self.assertTrue('fail.html [ Skip ]' in expectations)

        self.assertTrue('ref_crash.html [ Skip ]' in expectations)

        self.assertTrue('ref_timeout.html [ Skip ]' in expectations)

        self.assertTrue('ref_fail.html [ Skip ]' in expectations)

        self.assertFalse('missing.html' in expectations)

        self.assertFalse('partial.html' in expectations)

        self.assertFalse('pass.html' in expectations)

    def test_test_expectations_update(self):

        expectations = '\n'.join(self.conformance_checker.test_expectation_updates())

        self.assertTrue('partial.html [ Pass ]' in expectations)

    def all_test_paths(self):

        return [test_repository.name for test_repository in self.test_repositories]

2015-03-30  Youenn Fablet  <youenn.fablet@crf.canon.fr>

        Add support for automated W3C conformance testing

        https://bugs.webkit.org/show_bug.cgi?id=134766

        Reviewed by NOBODY (OOPS!).

        The template to display conformance results.

        * resources/w3c_conformance_results.html: Added.

<!DOCTYPE html>

<html>

<head>

<style>

#results {

    display: table;

    padding: 20px;

    clear: both;

}

#summary {

    margin-top: 20px;

    margin-bottom: 20px;

}

.test, .firstline {

    display: table;

}

.testname, .testresult, .firstLine > div {

    background-color: lavender;

    border-bottom:solid 1px white;

}

.firstLineCell{

    display: table-cell;

    text-align: center;

    width: 800px;

}

.secondLineCell {

    display: table-cell;

    padding-left: 50px;

    width: 750px;

}

.testname {

    display: table-cell;

    width: 900px;

    margin-left:20px;

}

.testnamelink {

    float:left;

    margin-right:10px;

}

.testresultlink  {

    float: left;

}

.testresultlink a {

    text-transform: capitalize;

}

.bugfilinglink  {

    text-align: right;

}

.port {

    display: table-cell;

    width: 250px;

    text-align: center;

}

.testresult {

    display: table-cell;

    width: 150px;

    text-align: center;

}

.testresult:not(:last-child) {

    padding-right: 100px;

}

.partial, .pass, .fail, .timeout, .error {

    text-align: center;

}

.pass a {

    color: green;

}

.partial a {

    color: yellow;

}

.fail a {

    color: orange;

}

.timeout a {

    color: red;

}

.crash a {

    color: red;

}

.error a {

    color: red;

}

.error-results-hist {

    background-color: red;

    float: left; 

}

.fail-results-hist {

    background-color: orange;

    float: left;

}

.partial-results-hist {

    background-color: yellow;

    float: left;

}

.pass-results-hist {

    background-color: green;

    float: left;

}

.module-name {

    display:table-cell;

    width: 800px;

}

.number-of-tests {

    display:inline-block;

    width: 50px;

}

.module-results {

    text-align: center;

    display:table-cell;

    width: 250px;

}

.histogram {

    display:inline-block;

    margin-left: 40px;

    margin-top: 5px;

    width:150px;

}

.module{

    background-color: lavender;

    border:solid 1px white;

    display:table;

}

.module-summary, .secondLine {

    display:table;

    background: lavender;

    border:solid 1px white;

    height: 25px;

    cursor:pointer;

}

.box{

    display: table;

}

.hidden-data {

    display: none;

}

.legend > div {

    float: left;

    margin-right: 5px;

    min-width: 20px;

}

.legend {

}

</style>

<script>

function testsDirectory(results)

{

    return (results.root_directory ? results.root_directory : results.port_name) + "/" + 

            (results.tests_directory ? results.tests_directory : "imported-tests")

}

function resultsDirectory(results)

{

    return (results.root_directory ? results.root_directory : results.port_name) + "/" + 

            (results.results_directory ? results.results_directory : "layout-test-results")

}

function githubTestURL(testname, results)

{

    for(var i = 0; i < results.test_repositories.length; i++) {

        repository = results.test_repositories[i]

        if (testname.indexOf(repository.name) == 0)

            return repository.url.replace(".git","") + "/blob/" + repository.revision + testname.replace(repository.name, "")

    }

    return "#cannot find test repository"

}

function bugzillaURL(title, comment)

{

    return "https://bugs.webkit.org/enter_bug.cgi?assigned_to=webkit-unassigned%40lists.webkit.org&product=WebKit&form_name=enter_bug&short_desc=" + title + "&comment=" + comment

}

function testFileLink(testname, results)

{

    return "<div class='testnamelink'><a href='" + testsDirectory(results) + "/" + testname + "'>" + testname + "</a></div>"

        + "<div class='githublink'>(<a href='" + githubTestURL(testname, results) + "'>github</a>)</div>" 

}

function testResultFileLink(testname, result, results)

{

    var suffix = "." + testname.split('.').pop()

    var new_suffix = "-expected.txt"

    result_status = result.status.toLowerCase().trim()

    if (result_status == "crash")

        new_suffix = "-crash-log.txt"

    if (result.type == "reftest")

        new_suffix = "-diffs.html"

    var link = "<div class='testresultlink'><a href='" + testsDirectory(results) + "/" + testname.replace(suffix, new_suffix) + "'>" + result_status + "</a></div>"

    if (result != "pass") {

        var title = encodeURIComponent("W3C " + testname + " test is " + result_status + "ing")

        var comment = encodeURIComponent(githubTestURL(testname, results) + " is " + result_status + "ing on " + results.port_name.toUpperCase() + ".")

        link += "<div class='bugfilinglink'><a href='" + bugzillaURL(title, comment) + "'>file bug</a></div>"

    }

    return link

}

function testSuiteStateHistogram(numError, numFail, numPartial, numPass)

{

    var total = numError + numPass + numPartial + numFail

    var html = '<div class="module-results">'

    html += '<div class="number-of-tests">' + numPass + '/' + total + '</div>'

    html += '<div class="histogram">'

    if (numError)

        html += '<div class="error-results-hist" style="width:' + (100 * numError / total) + '%">&nbsp;</div>'

    if (numFail)

        html += '<div class="fail-results-hist" style="width:' + (100 * numFail / total)  + '%">&nbsp;</div>'

    if (numPartial)

        html += '<div class="partial-results-hist" style="width:' + (100 * numPartial / total) + '%">&nbsp;</div>'

    if (numPass)

        html += '<div class="pass-results-hist" style="width:' + (100 * numPass / total) + '%">&nbsp;</div>'

    html += '</div>'

    return html + '</div>'

}

function testsuiteModules(results)

 {

    var modules = {}

    // First, try to only show module from URL hash

    var hash = window.location.hash.substring(1)

    var hasTest = false

    if (hash.length) {

        Object.keys(results.conformance_results).sort().forEach(function(key, i) {

            var testName = key

            if (testName.indexOf(hash) == 0) {

                hasTest = true

                var items = testName.split("/")

                var name = items[0]

                if (modules[name])

                    modules[name].push(key)

                else

                    modules[name] = [key]

            }

        })

    }

    if (hasTest)

        return modules

    // If nothing to show from URL hash, show all modules

    Object.keys(results.conformance_results).sort().forEach(function(key, i) {

        var testName = key

        var items = testName.split("/")

        var name = items[0]

        // In case of html module, we split display in sub-modules

        if (name == 'web-platform-tests' || name == 'csswg-tests' || name == 'html')

            name += '/' + items[1]

        if (modules[name])

            modules[name].push(key)

        else

            modules[name] = [key]

    })

    return modules

 }

function toggleTests(div)

{

    var box = div.nextElementSibling

    if (box)

        box.className = (box.className == 'box') ? 'hidden-data' : 'box'

}

function incrementSummary(result, summary)

{

    if (result.status == "PASS")

        summary.numPass++

    else if (result.status == "PARTIAL")

        summary.numPartial++

    else if (result.status == "FAIL")

        summary.numFail++

    else

        summary.numError++

}

function computeGlobalConformance(results)

{

    var summary = {numPass: 0, numPartial: 0, numFail: 0, numError: 0}

    for (var test in results.conformance_results) {

        if (results.conformance_results.hasOwnProperty(test))

            incrementSummary(results.conformance_results[test], summary)

    }

    return summary

}

function computeStatistics(resultsSummary, result, results)

{

    var summary = resultsSummary[results.port_name]

    if (!summary)

        resultsSummary[results.port_name] = summary = {numPass: 0, numPartial: 0, numFail: 0, numError: 0}

    incrementSummary(result, summary)

}

function addTestSuiteListOfResults(moduleName, tests, listOfResults, isOpen)

{

    var innerHTML = ""

    var resultsSummary =  {}

    tests.forEach(function(key, i) {

        innerHTML += "<div class='test'><div class='testname'>" + testFileLink(key, listOfResults[0]) + "</div>"

        listOfResults.forEach(function(results) {

            var result = results.conformance_results[key]

            if (!result) {

                result = "Missing"

                console.log("WARNING: test " + key + " is missing")

            }

            innerHTML += "<div class='testresult " + result.status.toLowerCase() + "'>" + testResultFileLink(key, result, results) + "</div>"

            computeStatistics(resultsSummary, result, results)

        })

        innerHTML += "</div>"

    })

    var html = '<div class="module"><div class="module-summary" onclick="toggleTests(event.currentTarget)"><div class="module-name">' + moduleName + '</div>'

    listOfResults.forEach(function(results) {

        var summary = resultsSummary[results.port_name]

        html += testSuiteStateHistogram(summary.numError, summary.numFail, summary.numPartial, summary.numPass)

    })

    html += '</div><div class="' + (isOpen ? "box" : "hidden-data") + '">'

    html += innerHTML

    html += '</div></div>'

    document.getElementById("results").innerHTML += html

}

function writeSummary()

{

    var html = "<div>W3C conformance results from run-webkit-test ("

    listOfResults.forEach(function(results) {

        html += "<a href='" + resultsDirectory(results) + "'>" + results.port_name + "</a> &nbsp;"

    })

    html += ")</div>"

    html += "<div> Generated <a href='TestExpectations'>test expectations</a> and <a href='ImportExpectations'>import expectations</a>.</div>"

    // FIXME: Check that all conformance results use the same test suite revisions.

    document.getElementById("summary").innerHTML = html

}

function displayResults()

{

    setTimeout(function()  {

        var firstLineHTML = "<div class='firstLine'><div class='firstLineCell'>Module/Test Name</div>"

        listOfResults.forEach(function(results) {

            firstLineHTML += "<div class='port'>" + results.port_name.toUpperCase() + "</div>"

        })

        firstLineHTML += "</div>"

        var secondLineHTML = "<div class='secondLine'><div class='secondLineCell'>Overall conformance</div>"

        listOfResults.forEach(function(results) {

            summary = computeGlobalConformance(results)

            secondLineHTML += testSuiteStateHistogram(summary.numError, summary.numFail, summary.numPartial, summary.numPass)

        })

        secondLineHTML += "</div>"

        document.getElementById("results").innerHTML = firstLineHTML + secondLineHTML

        var modules =  testsuiteModules(listOfResults[0])

        var keys = Object.keys(modules)

        keys.sort().forEach(function(key, i) {

            addTestSuiteListOfResults(key, modules[key], listOfResults, keys.length == 1)

        })

        writeSummary(listOfResults)

    }, 0)

}

listOfResults = [] 

function ADD_CONFORMANCE_RESULTS(results)

{

    results.root_directory = dirs[counter]

    listOfResults.push(results)

    setTimeout(loadScripts, 0)    

}

var counter

var dirs

function loadScripts(directories)

{

    if (directories) {

        dirs = directories

        counter = -1

    }

    if (counter < dirs.length - 1) {

        document.getElementById('summary').innerHTML = "Loading " + dirs[++counter] + "..."

        var head= document.getElementsByTagName('head')[0]

        var script= document.createElement('script')

        script.type= 'text/javascript'

        script.src= dirs[counter] + '/w3c_conformance_results.json'

        head.appendChild(script)

    }

    else {

        document.getElementById('summary').innerHTML = "Displaying results..."

        setTimeout(displayResults, 0)

    }

}

</script>

</head>

<body>

<div id="summary">

    Loading...

</div>

<div class="legend">

    <div class="error-results-hist legend">&nbsp;</div>

    <div>Error, Timeout or Crash</div>

    <div class="fail-results-hist legend">&nbsp;</div>

    <div> All assertions failing</div>

    <div class="partial-results-hist legend">&nbsp;</div>

    <div> Some assertions passing</div>

    <div class="pass-results-hist legend">&nbsp;</div>

    <div> All assertions passing</div>

</div>

<div id="results">

</div>

<script>

    setTimeout(function() {

        // Add as many folder containing w3c_conformance_results.json as needed in loadScripts array to display and compare the results.

        loadScripts(["."])

    }, 0)

</script>

</body>

</html>