Attachment 360337 Details for Bug 193904 – Patch

[patch] Patch

wd-run-subtest.diff (text/plain), 9.13 KB, created by Carlos Garcia Campos on 2019-01-28 06:35:39 PST

(hide)

Description:

Filename:

MIME Type:

Creator: Carlos Garcia Campos

Created: 2019-01-28 06:35:39 PST

Size: 9.13 KB

patch

obsolete

>diff --git a/Tools/ChangeLog b/Tools/ChangeLog
>index bdb520ae94b..59b19249726 100644
>--- a/Tools/ChangeLog
>+++ b/Tools/ChangeLog
>@@ -1,3 +1,27 @@
>+2019-01-28  Carlos Garcia Campos  <cgarcia@igalia.com>
>+
>+        WebDriver: add support for running subtests
>+        https://bugs.webkit.org/show_bug.cgi?id=193904
>+
>+        Reviewed by NOBODY (OOPS!).
>+
>+        pytest already supports it by passing test.py::subtest, so we can do the same.
>+
>+        * Scripts/webkitpy/webdriver_tests/pytest_runner.py:
>+        (get_item_name): Made this global.
>+        (CollectRecorder.__init__): Receive the parameter to ignore.
>+        (CollectRecorder.pytest_collectreport): Collect also the subtests.
>+        (TestExpectationsMarker.pytest_collection_modifyitems): Use get_item_name().
>+        (collect): Add parameter to ignore.
>+        * Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py:
>+        (WebDriverSeleniumExecutor.collect): Pass the driver name as parameter to ignore.
>+        * Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py:
>+        (WebDriverTestRunnerSelenium.collect_tests): Handle subtest name in test path.
>+        (WebDriverTestRunnerSelenium.run): Ditto.
>+        * Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py:
>+        (WebDriverTestRunnerW3C.collect_tests): Ditto.
>+        (WebDriverTestRunnerW3C.run): Ditto.
>+
> 2019-01-25  Carlos Garcia Campos  <cgarcia@igalia.com>
> 
>         [GTK][WPE] Add web extensions API to whitelist access to a security origin
>diff --git a/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py b/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py
>index 7cf05049c4f..9dc44c314c3 100644
>--- a/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py
>+++ b/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py
>@@ -50,14 +50,32 @@ class TemporaryDirectory(object):
>                 raise
> 
> 
>+def get_item_name(item, ignore_param):
>+    if ignore_param is None:
>+        return item.name
>+
>+    single_param = '[%s]' % ignore_param
>+    if item.name.endswith(single_param):
>+        return item.name[:-len(single_param)]
>+
>+    param = '[%s-' % ignore_param
>+    if param in item.name:
>+        return item.name.replace('%s-' % ignore_param, '')
>+
>+    return item.name
>+
>+
> class CollectRecorder(object):
> 
>-    def __init__(self):
>-        self.tests = []
>+    def __init__(self, ignore_param):
>+        self._ignore_param = ignore_param
>+        self.tests = {}
> 
>     def pytest_collectreport(self, report):
>         if report.nodeid:
>-            self.tests.append(report.nodeid)
>+            self.tests.setdefault(report.nodeid, [])
>+            for subtest in report.result:
>+                self.tests[report.nodeid].append(get_item_name(subtest, self._ignore_param))
> 
> 
> class HarnessResultRecorder(object):
>@@ -136,24 +154,10 @@ class TestExpectationsMarker(object):
>         self._ignore_param = ignore_param
>         self._base_dir = WebKitFinder(FileSystem()).path_from_webkit_base('WebDriverTests')
> 
>-    def _item_name(self, item):
>-        if self._ignore_param is None:
>-            return item.name
>-
>-        single_param = '[%s]' % self._ignore_param
>-        if item.name.endswith(single_param):
>-            return item.name[:-len(single_param)]
>-
>-        param = '[%s-' % self._ignore_param
>-        if param in item.name:
>-            return item.name.replace('%s-' % self._ignore_param, '')
>-
>-        return item.name
>-
>     def pytest_collection_modifyitems(self, session, config, items):
>         for item in items:
>             test = os.path.relpath(str(item.fspath), self._base_dir)
>-            item_name = self._item_name(item)
>+            item_name = get_item_name(item, self._ignore_param)
>             if self._expectations.is_slow(test, item_name):
>                 item.add_marker(pytest.mark.timeout(self._timeout * 5))
>             expected = self._expectations.get_expectation(test, item_name)[0]
>@@ -165,8 +169,8 @@ class TestExpectationsMarker(object):
>                 item.add_marker(pytest.mark.skip)
> 
> 
>-def collect(directory, args):
>-    collect_recorder = CollectRecorder()
>+def collect(directory, args, ignore_param=None):
>+    collect_recorder = CollectRecorder(ignore_param)
>     stdout = sys.stdout
>     with open(os.devnull, 'wb') as devnull:
>         sys.stdout = devnull
>diff --git a/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py b/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py
>index cc758625b8d..1d157045245 100644
>--- a/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py
>+++ b/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py
>@@ -57,7 +57,7 @@ class WebDriverSeleniumExecutor(object):
>             do_delayed_imports()
> 
>     def collect(self, directory):
>-        return pytest_runner.collect(directory, self._args)
>+        return pytest_runner.collect(directory, self._args, self._driver_name)
> 
>     def run(self, test, timeout, expectations):
>         return pytest_runner.run(test, self._args, timeout, self._env, expectations, self._driver_name)
>diff --git a/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py b/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py
>index 48f11124a14..3de9e3b2ac8 100644
>--- a/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py
>+++ b/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py
>@@ -51,18 +51,29 @@ class WebDriverTestRunnerSelenium(object):
>         executor = WebDriverSeleniumExecutor(self._driver, self._env)
>         # Collected tests are relative to test directory.
>         base_dir = os.path.join(self._tests_dir, os.path.dirname(relative_tests_dir))
>-        collected_tests = [os.path.join(base_dir, test) for test in executor.collect(os.path.join(self._tests_dir, relative_tests_dir))]
>+        collected_tests = {}
>+        for test, subtests in executor.collect(os.path.join(self._tests_dir, relative_tests_dir)).iteritems():
>+            collected_tests[os.path.join(base_dir, test)] = subtests
>         selenium_tests = []
>         if not tests:
>             tests = [relative_tests_dir]
>         for test in tests:
>+            subtest = None
>+            if '::' in test:
>+                test, subtest = test.split('::')
>+
>             if not test.startswith(relative_tests_dir):
>                 continue
>+
>             test_path = os.path.join(self._tests_dir, test)
>             if os.path.isdir(test_path):
>                 selenium_tests.extend([test for test in collected_tests if test.startswith(test_path) and test not in skipped])
>             elif test_path in collected_tests and test_path not in skipped:
>-                selenium_tests.append(test_path)
>+                if subtest is not None:
>+                    if subtest in collected_tests[test_path]:
>+                        selenium_tests.append(test_path + '::' + subtest)
>+                else:
>+                    selenium_tests.append(test_path)
>         return selenium_tests
> 
>     def run(self, tests=[]):
>@@ -72,7 +83,7 @@ class WebDriverTestRunnerSelenium(object):
>         executor = WebDriverSeleniumExecutor(self._driver, self._env)
>         timeout = self._port.get_option('timeout')
>         for test in tests:
>-            test_name = os.path.relpath(test, self._tests_dir)
>+            test_name = os.path.relpath(test.split('::')[0], self._tests_dir)
>             harness_result, test_results = executor.run(test, timeout, self._expectations)
>             result = WebDriverTestResult(test_name, *harness_result)
>             if harness_result[0] == 'OK':
>diff --git a/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py b/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py
>index 0e0e871713a..f94c7c89bc4 100644
>--- a/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py
>+++ b/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py
>@@ -52,13 +52,21 @@ class WebDriverTestRunnerW3C(object):
>         if not tests:
>             tests = [relative_tests_dir]
>         for test in tests:
>+            subtest = None
>+            if '::' in test:
>+                test, subtest = test.split('::')
>+
>             if not test.startswith(relative_tests_dir):
>                 continue
>+
>             test_path = os.path.join(self._tests_dir, test)
>             if os.path.isdir(test_path):
>                 w3c_tests.extend(self._scan_directory(test_path, skipped))
>             elif self._is_test(test_path) and test_path not in skipped:
>-                w3c_tests.append(test_path)
>+                if subtest is not None:
>+                    w3c_tests.append(test_path + '::' + subtest)
>+                else:
>+                    w3c_tests.append(test_path)
>         return w3c_tests
> 
>     def _is_test(self, test):
>@@ -91,7 +99,7 @@ class WebDriverTestRunnerW3C(object):
>         need_restart = False
>         try:
>             for test in tests:
>-                test_name = os.path.relpath(test, self._tests_dir)
>+                test_name = os.path.relpath(test.split('::')[0], self._tests_dir)
>                 harness_result, test_results = executor.run(test)
>                 result = WebDriverTestResult(test_name, *harness_result)
>                 if harness_result[0] == 'OK':

Flags:

mcatanzaro: review+

Actions: View | Formatted Diff | Diff

Attachments on bug 193904: 360337