WebKit Bugzilla
Attachment 360337 Details for
Bug 193904
: WebDriver: add support for running subtests
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Requests
|
Help
|
New Account
|
Log In
Remember
[x]
|
Forgot Password
Login:
[x]
[patch]
Patch
wd-run-subtest.diff (text/plain), 9.13 KB, created by
Carlos Garcia Campos
on 2019-01-28 06:35:39 PST
(
hide
)
Description:
Patch
Filename:
MIME Type:
Creator:
Carlos Garcia Campos
Created:
2019-01-28 06:35:39 PST
Size:
9.13 KB
patch
obsolete
>diff --git a/Tools/ChangeLog b/Tools/ChangeLog >index bdb520ae94b..59b19249726 100644 >--- a/Tools/ChangeLog >+++ b/Tools/ChangeLog >@@ -1,3 +1,27 @@ >+2019-01-28 Carlos Garcia Campos <cgarcia@igalia.com> >+ >+ WebDriver: add support for running subtests >+ https://bugs.webkit.org/show_bug.cgi?id=193904 >+ >+ Reviewed by NOBODY (OOPS!). >+ >+ pytest already supports it by passing test.py::subtest, so we can do the same. >+ >+ * Scripts/webkitpy/webdriver_tests/pytest_runner.py: >+ (get_item_name): Made this global. >+ (CollectRecorder.__init__): Receive the parameter to ignore. >+ (CollectRecorder.pytest_collectreport): Collect also the subtests. >+ (TestExpectationsMarker.pytest_collection_modifyitems): Use get_item_name(). >+ (collect): Add parameter to ignore. >+ * Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py: >+ (WebDriverSeleniumExecutor.collect): Pass the driver name as parameter to ignore. >+ * Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py: >+ (WebDriverTestRunnerSelenium.collect_tests): Handle subtest name in test path. >+ (WebDriverTestRunnerSelenium.run): Ditto. >+ * Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py: >+ (WebDriverTestRunnerW3C.collect_tests): Ditto. >+ (WebDriverTestRunnerW3C.run): Ditto. >+ > 2019-01-25 Carlos Garcia Campos <cgarcia@igalia.com> > > [GTK][WPE] Add web extensions API to whitelist access to a security origin >diff --git a/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py b/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py >index 7cf05049c4f..9dc44c314c3 100644 >--- a/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py >+++ b/Tools/Scripts/webkitpy/webdriver_tests/pytest_runner.py >@@ -50,14 +50,32 @@ class TemporaryDirectory(object): > raise > > >+def get_item_name(item, ignore_param): >+ if ignore_param is None: >+ return item.name >+ >+ single_param = '[%s]' % ignore_param >+ if item.name.endswith(single_param): >+ return item.name[:-len(single_param)] >+ >+ param = '[%s-' % ignore_param >+ if param in item.name: >+ return item.name.replace('%s-' % ignore_param, '') >+ >+ return item.name >+ >+ > class CollectRecorder(object): > >- def __init__(self): >- self.tests = [] >+ def __init__(self, ignore_param): >+ self._ignore_param = ignore_param >+ self.tests = {} > > def pytest_collectreport(self, report): > if report.nodeid: >- self.tests.append(report.nodeid) >+ self.tests.setdefault(report.nodeid, []) >+ for subtest in report.result: >+ self.tests[report.nodeid].append(get_item_name(subtest, self._ignore_param)) > > > class HarnessResultRecorder(object): >@@ -136,24 +154,10 @@ class TestExpectationsMarker(object): > self._ignore_param = ignore_param > self._base_dir = WebKitFinder(FileSystem()).path_from_webkit_base('WebDriverTests') > >- def _item_name(self, item): >- if self._ignore_param is None: >- return item.name >- >- single_param = '[%s]' % self._ignore_param >- if item.name.endswith(single_param): >- return item.name[:-len(single_param)] >- >- param = '[%s-' % self._ignore_param >- if param in item.name: >- return item.name.replace('%s-' % self._ignore_param, '') >- >- return item.name >- > def pytest_collection_modifyitems(self, session, config, items): > for item in items: > test = os.path.relpath(str(item.fspath), self._base_dir) >- item_name = self._item_name(item) >+ item_name = get_item_name(item, self._ignore_param) > if self._expectations.is_slow(test, item_name): > item.add_marker(pytest.mark.timeout(self._timeout * 5)) > expected = self._expectations.get_expectation(test, item_name)[0] >@@ -165,8 +169,8 @@ class TestExpectationsMarker(object): > item.add_marker(pytest.mark.skip) > > >-def collect(directory, args): >- collect_recorder = CollectRecorder() >+def collect(directory, args, ignore_param=None): >+ collect_recorder = CollectRecorder(ignore_param) > stdout = sys.stdout > with open(os.devnull, 'wb') as devnull: > sys.stdout = devnull >diff --git a/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py b/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py >index cc758625b8d..1d157045245 100644 >--- a/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py >+++ b/Tools/Scripts/webkitpy/webdriver_tests/webdriver_selenium_executor.py >@@ -57,7 +57,7 @@ class WebDriverSeleniumExecutor(object): > do_delayed_imports() > > def collect(self, directory): >- return pytest_runner.collect(directory, self._args) >+ return pytest_runner.collect(directory, self._args, self._driver_name) > > def run(self, test, timeout, expectations): > return pytest_runner.run(test, self._args, timeout, self._env, expectations, self._driver_name) >diff --git a/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py b/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py >index 48f11124a14..3de9e3b2ac8 100644 >--- a/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py >+++ b/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_selenium.py >@@ -51,18 +51,29 @@ class WebDriverTestRunnerSelenium(object): > executor = WebDriverSeleniumExecutor(self._driver, self._env) > # Collected tests are relative to test directory. > base_dir = os.path.join(self._tests_dir, os.path.dirname(relative_tests_dir)) >- collected_tests = [os.path.join(base_dir, test) for test in executor.collect(os.path.join(self._tests_dir, relative_tests_dir))] >+ collected_tests = {} >+ for test, subtests in executor.collect(os.path.join(self._tests_dir, relative_tests_dir)).iteritems(): >+ collected_tests[os.path.join(base_dir, test)] = subtests > selenium_tests = [] > if not tests: > tests = [relative_tests_dir] > for test in tests: >+ subtest = None >+ if '::' in test: >+ test, subtest = test.split('::') >+ > if not test.startswith(relative_tests_dir): > continue >+ > test_path = os.path.join(self._tests_dir, test) > if os.path.isdir(test_path): > selenium_tests.extend([test for test in collected_tests if test.startswith(test_path) and test not in skipped]) > elif test_path in collected_tests and test_path not in skipped: >- selenium_tests.append(test_path) >+ if subtest is not None: >+ if subtest in collected_tests[test_path]: >+ selenium_tests.append(test_path + '::' + subtest) >+ else: >+ selenium_tests.append(test_path) > return selenium_tests > > def run(self, tests=[]): >@@ -72,7 +83,7 @@ class WebDriverTestRunnerSelenium(object): > executor = WebDriverSeleniumExecutor(self._driver, self._env) > timeout = self._port.get_option('timeout') > for test in tests: >- test_name = os.path.relpath(test, self._tests_dir) >+ test_name = os.path.relpath(test.split('::')[0], self._tests_dir) > harness_result, test_results = executor.run(test, timeout, self._expectations) > result = WebDriverTestResult(test_name, *harness_result) > if harness_result[0] == 'OK': >diff --git a/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py b/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py >index 0e0e871713a..f94c7c89bc4 100644 >--- a/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py >+++ b/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py >@@ -52,13 +52,21 @@ class WebDriverTestRunnerW3C(object): > if not tests: > tests = [relative_tests_dir] > for test in tests: >+ subtest = None >+ if '::' in test: >+ test, subtest = test.split('::') >+ > if not test.startswith(relative_tests_dir): > continue >+ > test_path = os.path.join(self._tests_dir, test) > if os.path.isdir(test_path): > w3c_tests.extend(self._scan_directory(test_path, skipped)) > elif self._is_test(test_path) and test_path not in skipped: >- w3c_tests.append(test_path) >+ if subtest is not None: >+ w3c_tests.append(test_path + '::' + subtest) >+ else: >+ w3c_tests.append(test_path) > return w3c_tests > > def _is_test(self, test): >@@ -91,7 +99,7 @@ class WebDriverTestRunnerW3C(object): > need_restart = False > try: > for test in tests: >- test_name = os.path.relpath(test, self._tests_dir) >+ test_name = os.path.relpath(test.split('::')[0], self._tests_dir) > harness_result, test_results = executor.run(test) > result = WebDriverTestResult(test_name, *harness_result) > if harness_result[0] == 'OK':
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Flags:
mcatanzaro
:
review+
Actions:
View
|
Formatted Diff
|
Diff
Attachments on
bug 193904
: 360337