WebKit Bugzilla
Attachment 347960 Details for
Bug 188869
: API tests should output json results
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Requests
|
Help
|
New Account
|
Log In
Remember
[x]
|
Forgot Password
Login:
[x]
[patch]
Patch for landing
bug-188869-20180823151224.patch (text/plain), 5.50 KB, created by
Jonathan Bedard
on 2018-08-23 15:12:25 PDT
(
hide
)
Description:
Patch for landing
Filename:
MIME Type:
Creator:
Jonathan Bedard
Created:
2018-08-23 15:12:25 PDT
Size:
5.50 KB
patch
obsolete
>Index: Tools/ChangeLog >=================================================================== >--- Tools/ChangeLog (revision 235249) >+++ Tools/ChangeLog (working copy) >@@ -1,3 +1,26 @@ >+2018-08-23 Jonathan Bedard <jbedard@apple.com> >+ >+ API tests should output json results >+ https://bugs.webkit.org/show_bug.cgi?id=188869 >+ <rdar://problem/43615652> >+ >+ Reviewed by Aakash Jain. >+ >+ JSON output for API tests is of the form: >+ { >+ "Failed": [{"name": <test name>, "output": <test log>}], >+ "Timedout": [...], >+ "Skipped": [...], >+ "Crashed": [...] >+ } >+ Tests which are successful are not displayed in the json output. >+ >+ * Scripts/webkitpy/api_tests/manager.py: >+ (Manager.run): Print test results to provided file as a json dictionary. >+ * Scripts/webkitpy/api_tests/run_api_tests.py: >+ (run): Pass json option. >+ (parse_args): Add --json-output flag. >+ > 2018-08-23 Myles C. Maxfield <mmaxfield@apple.com> > > [WSL] Ternary expressions appear to be unimplemented >Index: Tools/Scripts/webkitpy/api_tests/manager.py >=================================================================== >--- Tools/Scripts/webkitpy/api_tests/manager.py (revision 235249) >+++ Tools/Scripts/webkitpy/api_tests/manager.py (working copy) >@@ -20,6 +20,7 @@ > # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS > # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > >+import json > import logging > import os > >@@ -148,7 +149,12 @@ class Manager(object): > return self._port.path_to_api_test_binaries().keys() > return binaries or self._port.path_to_api_test_binaries().keys() > >- def run(self, args): >+ def run(self, args, json_output=None): >+ if json_output: >+ json_output = self.host.filesystem.abspath(json_output) >+ if not self.host.filesystem.isdir(self.host.filesystem.dirname(json_output)) or self.host.filesystem.isdir(json_output): >+ raise RuntimeError('Cannot write to {}'.format(json_output)) >+ > self._stream.write_update('Checking build ...') > if not self._port.check_api_test_build(self._binaries_for_arguments(args)): > _log.error('Build check failed') >@@ -184,9 +190,18 @@ class Manager(object): > disabled = len(runner.result_map_by_status(runner.STATUS_DISABLED)) > _log.info('Ran {} tests of {} with {} successful'.format(len(runner.results) - disabled, len(test_names), len(successful))) > >- self._stream.writeln('------------------------------') >+ result_dictionary = { >+ 'Skipped': [], >+ 'Failed': [], >+ 'Crashed': [], >+ 'Timedout': [], >+ } >+ >+ self._stream.writeln('-' * 30) > if len(successful) + disabled == len(test_names): > self._stream.writeln('All tests successfully passed!') >+ if json_output: >+ self.host.filesystem.write_text_file(json_output, json.dumps(result_dictionary, indent=4)) > return Manager.SUCCESS > > self._stream.writeln('Test suite failed') >@@ -196,6 +211,7 @@ class Manager(object): > for test in test_names: > if test not in runner.results: > skipped.append(test) >+ result_dictionary['Skipped'].append({'name': test, 'output': None}) > if skipped: > self._stream.writeln('Skipped {} tests'.format(len(skipped))) > self._stream.writeln('') >@@ -207,4 +223,17 @@ class Manager(object): > self._print_tests_result_with_status(runner.STATUS_CRASHED, runner) > self._print_tests_result_with_status(runner.STATUS_TIMEOUT, runner) > >+ for test, result in runner.results.iteritems(): >+ status_to_string = { >+ runner.STATUS_FAILED: 'Failed', >+ runner.STATUS_CRASHED: 'Crashed', >+ runner.STATUS_TIMEOUT: 'Timedout', >+ }.get(result[0]) >+ if not status_to_string: >+ continue >+ result_dictionary[status_to_string].append({'name': test, 'output': result[1]}) >+ >+ if json_output: >+ self.host.filesystem.write_text_file(json_output, json.dumps(result_dictionary, indent=4)) >+ > return Manager.FAILED_TESTS >Index: Tools/Scripts/webkitpy/api_tests/run_api_tests.py >=================================================================== >--- Tools/Scripts/webkitpy/api_tests/run_api_tests.py (revision 235249) >+++ Tools/Scripts/webkitpy/api_tests/run_api_tests.py (working copy) >@@ -73,7 +73,7 @@ def run(port, options, args, logging_str > stream = MeteredStream(logging_stream, options.verbose, logger=logger, number_of_columns=port.host.platform.terminal_width(), print_timestamps=options.timestamps) > manager = Manager(port, options, stream) > >- result = manager.run(args) >+ result = manager.run(args, json_output=options.json_output) > _log.debug("Testing completed, Exit status: %d" % result) > return result > finally: >@@ -92,6 +92,8 @@ def parse_args(args): > help='Enable verbose printing'), > optparse.make_option('--timestamps', action='store_true', default=False, > help='Print timestamps for each logged line'), >+ optparse.make_option('--json-output', action='store', default=None, >+ help='Save test results as JSON to file'), > ])) > > option_group_definitions.append(('WebKit Options', [
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Formatted Diff
|
Diff
Attachments on
bug 188869
:
347876
|
347879
|
347952
|
347956
| 347960