#!/usr/bin/env python

# Copyright (c) 2014, Canon Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

# 3.  Neither the name of Canon Inc. nor the names of

#     its contributors may be used to endorse or promote products derived

#     from this software without specific prior written permission.

#

# THIS SOFTWARE IS PROVIDED BY CANON INC. AND ITS CONTRIBUTORS "AS IS" AND ANY

# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL CANON INC. AND ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Wrapper around webkitpy/w3c/run_w3c_conformance_tests.py"""

from webkitpy.common import multiprocessing_bootstrap

multiprocessing_bootstrap.run('webkitpy', 'w3c', 'run_w3c_conformance_tests.py')

# Copyright (c) 2014, Canon Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

# 3.  Neither the name of Canon Inc. nor the names of

#     its contributors may be used to endorse or promote products derived

#     from this software without specific prior written permission.

#

# THIS SOFTWARE IS PROVIDED BY CANON INC. AND ITS CONTRIBUTORS "AS IS" AND ANY

# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL CANON INC. AND ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""

 This script evaluates W3C tests conformance by:

    - importing W3C tests as WebKit tests (using web_platform_test_importer)

    - generating expected.txt files (using run-webkit-tests)

    - computing whether a test is passing or not by checking PASS/FAIL assertions outputted within the expected.txt files

"""

import argparse

import logging

import optparse

import os

import sys

from webkitpy.common.host import Host

from webkitpy.common.system.executive import Executive

from webkitpy.layout_tests.models.test_expectations import TestExpectations

from webkitpy.port import platform_options

from webkitpy.w3c.factory import w3c_create_argument_parser

from webkitpy.w3c.test_results_manager import TestResultsManager

from webkitpy.w3c.web_platform_test_setup import WebPlatformTestSetup

from webkitpy.w3c.web_platform_test_importer import WebPlatformTestImporter

_log = logging.getLogger(__name__)

class W3CConformanceRunner(object):

    def __init__(self, port, options):

        self.port = port

        self.filesystem = port.host.filesystem

        self.options = options

        self.web_platform_test_importer = WebPlatformTestImporter(port, options)

        self.tests = self.web_platform_test_importer.tests()

        self._results_manager = None

    def get_conformance_results_directory(self):

        return self.filesystem.join(self.port.results_directory(), "..", "w3c-conformance-results")

    def layout_tests_result_directory(self):

        return self.filesystem.join(self.get_conformance_results_directory(), "layout-test-results")

    def test_suites(self):

        return self.web_platform_test_importer.test_suites()

    def run_test_suites(self, parameters):

        #TODO: clean this to have a better path computation

        up = os.path.dirname

        script_dir = up(up(up(os.path.abspath(__file__))))

        tmp_args = [self.filesystem.join(script_dir, 'run-webkit-tests')] + parameters + ['--results-directory=' + self.layout_tests_result_directory()]

        tmp_args.append('--' + self.options.platform)

        if self.options.webkit_test_runner is True:

            tmp_args.append('-2')

        #compute list of tests to run

        for test_suite in self.test_suites():

            path = self.filesystem.join('LayoutTests',  self.options.__dict__['importdir'], test_suite)

            tmp_args.append(path)

        Executive().run_command(tmp_args, error_handler=Executive.ignore_error, return_stderr=(not self.options.__dict__['w3c_verbose']), decode_output=False)

        self._results_manager = TestResultsManager(self.port, self.options.platform, self.options.__dict__['importdir'], self.layout_tests_result_directory(), self.tests)

    def run_tests_to_generate_test_baseline(self):

        self.run_test_suites(['-q', '--no-retry-failures', '--no-show-results', '--skipped=ignore'])

    def remove_test_expected_files(self):

        for test in self.tests:

            filename = self.port.expected_filename(test, '.txt', False)

            if not(filename is None):

                file_path = self.filesystem.join(self.port.layout_tests_dir(), filename)

                self.filesystem.remove(file_path)

    def results_manager(self):

        return self._results_manager

    def run_tests(self):

        if self.options.__dict__['w3c_list'] is True:

            return False

        # import tests

        _log.info('Importing tests...')

        self.web_platform_test_importer.import_tests()

        # setup the necessary configuration files

        wpt_setup = WebPlatformTestSetup(self.port.host, self.options)

        wpt_setup.copy_webkit_testharnessreport()

        wpt_setup.copy_wpt_json_config()

        self.remove_test_expected_files()

        _log.info('Running tests...')

        self.run_tests_to_generate_test_baseline()

        # generate conformance results

        self.filesystem.maybe_make_directory(self.get_conformance_results_directory())

        self.results_manager().write_conformance_results(self.get_conformance_results_directory())

        return True

    def print_results_to_console(self):

        summary = self.results_manager().summary()

        _log.info('Run ' + str(summary['total']) + ' tests')

        _log.info('Pass: ' + str(summary['num_pass']))

        _log.info('Partial: ' + str(summary['num_partial']))

        _log.info('Fail: ' + str(summary['num_fail']))

        _log.info('Error: ' + str(summary['num_error']))

    def print_potential_test_expectations_update(self):

        test_expectations = TestExpectations(self.port).model()

        conformance = self.results_manager().conformance_results()

        # TODO: Finalize this check to get meaningful information

        for test in self.tests:

            if not test_expectations.has_test(test):

                if not conformance[test] == 'PASS':

                    _log.info('test ' + test + ' is not passing')

            elif conformance[test] == 'PASS':

                _log.info('test ' + test + ' is now passing')

def configure_logging():

    class LogHandler(logging.StreamHandler):

        def format(self, record):

            if record.levelno > logging.INFO:

                return "%s: %s" % (record.levelname, record.getMessage())

            return record.getMessage()

    logger = logging.getLogger()

    logger.setLevel(logging.INFO)

    handler = LogHandler()

    handler.setLevel(logging.INFO)

    logger.addHandler(handler)

    return handler

def main(argv, stdout, stderr):

    configure_logging()

    host = Host()

    parser = w3c_create_argument_parser(host, host.filesystem.basename(__file__))

    options, args = parser.parse_known_args(argv)

    port = host.port_factory.get(options.platform, optparse.Values({'webkit_test_runner': options.webkit_test_runner}))

    runner = W3CConformanceRunner(port, options)

    if runner.run_tests():

        runner.print_results_to_console()

        runner.print_potential_test_expectations_update()

        runner.remove_test_expected_files()

        _log.info("Results exported in '" + runner.get_conformance_results_directory() + "'")

if '__main__' == __name__:

    sys.exit(main(sys.argv[1:], sys.stdout, sys.stderr))

# Copyright (c) 2014, Canon Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

# 3.  Neither the name of Canon Inc. nor the names of

#     its contributors may be used to endorse or promote products derived

#     from this software without specific prior written permission.

#

# THIS SOFTWARE IS PROVIDED BY CANON INC. AND ITS CONTRIBUTORS "AS IS" AND ANY

# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL CANON INC. AND ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""

 This script manage JSON and results.html files for W3C test results.

"""

import collections

import fnmatch

import io

import json

import os

import tempfile

from webkitpy.common.host import Host

from webkitpy.layout_tests.layout_package import json_results_generator

class TestResultsManager(object):

    REASONS = ['TIMEOUT', 'CRASH', 'FAILURE']

    EXTENSIONS = ['.html', '.htm', '.xhtml', '.svg', '.xml', '.xht']

    _JSON_PREFIX = 'ADD_CONFORMANCE_RESULTS('

    _JSON_SUFFIX = ');'

    def __init__(self, port, platform, base_dir, result_dir, tests):

        self._port = port

        self._filesystem = port.host.filesystem

        self._results_directory = result_dir

        self._platform = platform

        self._tests = tests

        results_json_path = self._filesystem.join(self._results_directory, 'full_results.json')

        results_json = json_results_generator.load_json(self._filesystem, results_json_path)

        # Root directory of W3C imported test

        self._base_dir = base_dir

        # TODO: can we simplify this

        self._json_test_dict = None

        if results_json['tests']:

            path = []

            node = self._filesystem.split(self._base_dir)

            while node[0]:

                path.append(node[1])

                node = self._filesystem.split(node[0])

            path.append(node[1])

            path.reverse()

            self._json_test_dict = results_json['tests']

            for key in path:

                self._json_test_dict = self._json_test_dict[key]

    def _get_test_results(self, tests, test_dict=None, path=None):

        if test_dict is None:

            test_dict = self._json_test_dict

            if test_dict is None:

                return

        if path is None:

            path = self._base_dir

        for key in test_dict.keys():

            _path = self._filesystem.join(path, key)

            (filename, ext) = os.path.splitext(key)

            if ext in self.EXTENSIONS:

                tests[_path] = test_dict[key]

            else:

                # Process subdirectories

                self._get_test_results(tests, test_dict[key], _path)

    def test_results(self):

        test_results = {}

        self._get_test_results(test_results)

        return test_results

    def _get_conformance_from_expected_text(self, test_name, text, test_results):

        if text is None or len(text) == 0:

            # empty expected file, probably due to CRASH or TIMEOUT. Let's find why in json results.

            if test_name in test_results and 'actual' in test_results[test_name]:

                reason = test_results[test_name]['actual']

                if reason == 'CRASH' or reason == 'TIMEOUT':

                    return reason

            return 'ERROR'

        # Let's check whether some lines start with PASS or FAIL

        # Test is said to PASS if at least one line is PASS an no line is FAIL

        hasFail = False

        hasPass = False

        buf = io.StringIO(unicode(text, "utf-8"))

        for line in buf.readlines():

            if line.startswith('PASS'):

                hasPass = True

            elif line.startswith('FAIL'):

                hasFail = True

        if hasPass:

            return 'PARTIAL' if hasFail else 'PASS'

        if hasFail:

            return 'FAIL'

        return 'ERROR'

    def conformance_results(self):

        status = {}

        test_results = self.test_results()

        for test_name in self._tests:

            status[test_name] = self._get_conformance_from_expected_text(test_name, self._port.expected_text(test_name), test_results)

        return status

    def summary(self):

        num_pass = num_partial = num_fail = num_error = 0

        status = self.conformance_results()

        for test_name in self._tests:

            result = status[test_name]

            if result == 'PASS':

                num_pass += 1

            elif result == 'PARTIAL':

                num_partial += 1

            elif result == 'FAIL':

                num_fail += 1

            else:

                num_error += 1

        summary = {}

        summary['total'] = num_pass + num_fail + num_partial + num_error

        summary['num_pass'] = num_pass

        summary['num_fail'] = num_fail

        summary['num_partial'] = num_partial

        summary['num_error'] = num_error

        return summary

    def write_conformance_results(self, path):

        status = self.conformance_results()

        results = {}

        results['conformance_results'] = status

        results['port_name'] = self._port.port_name

        # copy conformance results as json data

        json_string = json.dumps(results, separators=(',', ':'))

        json_string = self._JSON_PREFIX + json_string + self._JSON_SUFFIX

        self._filesystem.write_text_file(self._filesystem.join(path, 'w3c_conformance_results.json'), json_string)

        # copy html template

        results_file = self._port.path_from_webkit_base('LayoutTests', 'imported', 'w3c', 'resources', 'w3c_conformance_results.html')

        # Note that the results.html template file won't exist when we're using a MockFileSystem during unit tests,

        # so make sure it exists before we try to copy it.

        if self._filesystem.exists(results_file):

            self._filesystem.copyfile(results_file, self._filesystem.join(path, 'w3c_conformance_results.html'))

#!/usr/bin/env python

# Copyright (c) 2014, Canon Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

# 3.  Neither the name of Canon Inc. nor the names of

#     its contributors may be used to endorse or promote products derived

#     from this software without specific prior written permission.

#

# THIS SOFTWARE IS PROVIDED BY CANON INC. AND ITS CONTRIBUTORS "AS IS" AND ANY

# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL CANON INC. AND ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""

 This script allows setting up the web platform test parameters such as:

    - The config.json file used by the WPT http server

    - The testharnessreport.js used for web platform tests (copy of webkit testharnessreport.js).

"""

import logging

from webkitpy.common.host import Host

from webkitpy.common.webkit_finder import WebKitFinder

_log = logging.getLogger(__name__)

class WebPlatformTestSetup(object):

    def __init__(self, host, options):

        self.filesystem = host.filesystem

        webkit_finder = WebKitFinder(self.filesystem)

        self._webkit_root = webkit_finder.webkit_base()

        self._wpt_root = self.filesystem.join(self._webkit_root, 'LayoutTests', options.__dict__['importdir'])

        self._layout_root = self.filesystem.join(self._webkit_root, "LayoutTests")

    def restore_w3c_testharnessreport(self):

        _log.debug('Using W3C WPT testharnessreport.js')

        testharnessreport_wpt_filename = self.filesystem.join(self._wpt_root, "resources", "testharnessreport-wpt.js")

        if self.filesystem.isfile(testharnessreport_wpt_filename):

            self.filesystem.copyfile(testharnessreport_wpt_filename, self.filesystem.join(self._wpt_root, "resources", "testharnessreport.js"))

            self.filesystem.remove(testharnessreport_wpt_filename)

    def copy_webkit_testharnessreport(self):

        _log.debug('Using WebKit testharnessreport.js')

        testharnessreport_wk_filename = self.filesystem.join(self._layout_root, "resources", "testharnessreport.js")

        if self.filesystem.isfile(testharnessreport_wk_filename):

            self.filesystem.copyfile(self.filesystem.join(self._wpt_root, "resources", "testharnessreport.js"), self.filesystem.join(self._wpt_root, "resources", "testharnessreport-wpt.js"))

            self.filesystem.copyfile(self.filesystem.join(self._layout_root, "resources", "testharnessreport.js"), self.filesystem.join(self._wpt_root, "resources", "testharnessreport.js"))

    def copy_wpt_json_config(self):

        _log.debug('Copying json web platform server config')

        config_wpt_filename = self.filesystem.join(self._layout_root, "imported", "w3c", "config.json")

        if self.filesystem.isfile(config_wpt_filename):

            self.filesystem.copyfile(config_wpt_filename, self.filesystem.join(self._wpt_root, "config.json"))

{"host": "localhost",

 "ports":{"http":[8800, 8801],

          "https":[],

          "ws":[49001]},

 "check_subdomains": false,

 "log_level":"debug",

 "bind_hostname": false}

<!DOCTYPE html>

<html>

<head>

<style>

#results {

	display: table;

	clear: both;

	padding: 20px;

}

#summary {

	padding: 20px;

}

.test {

	display: table-row;

	background-color: lavender;

}

.testname, .testresult {

	border-bottom: 1px solid white;

	border-top: 1px solid white;

	padding-right: 10px;

	display: table-cell;

}

.testresult a {

	text-transform: capitalize;

}

.pass a {

	color: green;

}

.partial a {

	color: yellow;

}

.fail a {

	color: orange;

}

.timeout a {

	color: red;

}

.crash a {

	color: red;

}

.error a {

	color: red;

}

.error-results-hist {

    background-color: red;

    float: left;

}

.fail-results-hist {

    background-color: orange;

    float: left;

}

.partial-results-hist {

    background-color: yellow;

    float: left;

}

.pass-results-hist {

    background-color: green;

    float: left;

}

</style>

<script>

function pretty_testname(testname)

{

	return testname.replace("imported/w3c/wpt/","")

}

function get_test_link(testname)

{

	return "<a href='" + "../../../LayoutTests/" + testname + "'>" + pretty_testname(testname) + "</a>"

}

function get_test_result_link(testname, result)

{

	suffix = "." + testname.split('.').pop();

	return "<a href='" + "layout-test-results/" + testname.replace(suffix,"-actual.txt") + "'>" + result.toLowerCase() + "</a>"

}

function testSuiteHist(numError, numFail, numPartial, numPass)

{

    var total = numError + numPass + numPartial + numFail;

    html = '<div style="float:left;margin-right:10px"> Conformance summary </div> <div style="float:left;width:150px;">';

    if (numError)

        html += '<div class="error-results-hist" style="width:' + (100 * numError / total) + '%">&nbsp;</div>';

    if (numFail)

        html += '<div class="fail-results-hist" style="width:' + (100 * numFail / total)  + '%">&nbsp;</div>';

    if (numPartial)

        html += '<div class="partial-results-hist" style="width:' + (100 * numPartial / total) + '%">&nbsp;</div>';

    if (numPass)

        html += '<div class="pass-results-hist" style="width:' + (100 * numPass / total) + '%">&nbsp;</div>';

    html += '</div>';

    return html;

}

function ADD_CONFORMANCE_RESULTS(results)

{

	var numPass = numPartial = numFail = numError = 0

	var innerHTML = "<div class='test'><div class='testname'>Test Name</div><div class='testresult'>" + results.port_name.toUpperCase() + "</div></div>"

	Object.keys(results.conformance_results)

		.sort()

		.forEach(function(key, i) {

			result = results.conformance_results[key]

			innerHTML += "<div class='test'><div class='testname'>" + get_test_link(key) + "</div><div class='testresult " + result.toLowerCase() + "'>" + get_test_result_link(key, result) + "</div></div>"

			if (result == "PASS") numPass++

			else if (result == "PARTIAL") numPartial++

			else if (result == "FAIL") numFail++

			else numError++

       });

	document.getElementById("summary").innerHTML = testSuiteHist(numError, numFail, numPartial, numPass)

	document.getElementById("results").innerHTML = innerHTML

}

</script>

</head>

<body>

<div>

W3C conformance results from run-webkit-test (<a href="layout-test-results/results.html">run-webkit-test results</a>)

</div>

<div id="summary">

</div>

<div id="results">

</div>

<script src="w3c_conformance_results.json"></script>

</body>

</html>