WebKit Bugzilla
New
Browse
Search+
Log In
×
Sign in with GitHub
or
Remember my login
Create Account
·
Forgot Password
Forgotten password account recovery
[patch]
Patch
bug-162814-20190404230936.patch (text/plain), 140.90 KB, created by
Devin Rousso
on 2019-04-04 23:09:37 PDT
(
hide
)
Description:
Patch
Filename:
MIME Type:
Creator:
Devin Rousso
Created:
2019-04-04 23:09:37 PDT
Size:
140.90 KB
patch
obsolete
>diff --git a/Source/WebInspectorUI/ChangeLog b/Source/WebInspectorUI/ChangeLog >index 6809f8dab28ddb8f9777e0283e37e13058e00342..df53c863bd0db06b1a441067118815309a5a0a67 100644 >--- a/Source/WebInspectorUI/ChangeLog >+++ b/Source/WebInspectorUI/ChangeLog >@@ -1,3 +1,30 @@ >+2019-04-04 Devin Rousso <drousso@apple.com> >+ >+ Web Inspector: TestSuite test cases should have their own timeout to ensure tests fail with output instead of timeout by test runner >+ https://bugs.webkit.org/show_bug.cgi?id=162814 >+ <rdar://problem/28574102> >+ >+ Reviewed by NOBODY (OOPS!). >+ >+ A 10s timer is started for every test case added to an async suite. The timer is cleared >+ when the test finishes, but if the timer fires, the test is forcibly ended with an error. >+ >+ This timer can be configured by setting a `timeout` value when adding the test case. Values >+ are expected to be in milliseconds. The value `-1` will prevent a timer from being set. >+ >+ This change also relaxes the expectation that any individual test case failure will stop the >+ rest of the suite from running. Since timers are set per test case, it is possible to >+ recover from a "bad" test case to still run the remaining test cases. >+ >+ NOTE: there may be unexpected behaviour if a test times out, as the timer doesn't actually >+ stop the execution of the test, so it may still run and log information, which may appear >+ "out of nowhere" in the middle of other tests. >+ >+ * UserInterface/Test/TestSuite.js: >+ (TestSuite.prototype.get passCount): >+ (AsyncTestSuite.prototype.runTestCases): >+ (SyncTestSuite.prototype.runTestCases): >+ > 2019-04-03 Devin Rousso <drousso@apple.com> > > Web Inspector: Single click on links in non-read-only TextEditors should not follow links >diff --git a/Source/WebInspectorUI/UserInterface/Test/TestSuite.js b/Source/WebInspectorUI/UserInterface/Test/TestSuite.js >index 3e17fdf42a1af87e9ee39513f6f74f974863e90e..5d29d6743fd8c1549496a27e315ffe4a577a62b0 100644 >--- a/Source/WebInspectorUI/UserInterface/Test/TestSuite.js >+++ b/Source/WebInspectorUI/UserInterface/Test/TestSuite.js >@@ -54,7 +54,7 @@ TestSuite = class TestSuite > > get passCount() > { >- return this.runCount - this.failCount; >+ return this.runCount - (this.failCount - this.skipCount); > } > > get skipCount() >@@ -135,45 +135,67 @@ AsyncTestSuite = class AsyncTestSuite extends TestSuite > > // Avoid adding newlines if nothing was logged. > let priorLogCount = this._harness.logCount; >- let result = this.testcases.reduce((chain, testcase, i) => { >- if (testcase.setup) { >- chain = chain.then(() => { >+ >+ return Promise.resolve().then(() => Promise.chain(this.testcases.map((testcase, i) => () => new Promise(async (resolve, reject) => { >+ if (i > 0 && priorLogCount + 1 < this._harness.logCount) >+ this._harness.log(""); >+ priorLogCount = this._harness.logCount; >+ >+ let hasTimeout = testcase.timeout !== -1; >+ let timeoutId = undefined; >+ if (hasTimeout) { >+ let delay = testcase.timeout || 10000; >+ timeoutId = setTimeout(() => { >+ if (!timeoutId) >+ return; >+ >+ timeoutId = undefined; >+ >+ this.failCount++; >+ this._harness.log(`!! TIMEOUT: took longer than ${delay}ms`); >+ >+ resolve(); >+ }, delay); >+ } >+ >+ try { >+ if (testcase.setup) { > this._harness.log("-- Running test setup."); > if (testcase.setup[Symbol.toStringTag] === "AsyncFunction") >- return testcase.setup(); >- return new Promise(testcase.setup); >- }); >- } >+ await testcase.setup(); >+ else >+ await new Promise(testcase.setup); >+ } > >- chain = chain.then(() => { >- if (i > 0 && priorLogCount + 1 < this._harness.logCount) >- this._harness.log(""); >- >- priorLogCount = this._harness.logCount; >- this._harness.log(`-- Running test case: ${testcase.name}`); > this.runCount++; >- if (testcase.test[Symbol.toStringTag] === "AsyncFunction") >- return testcase.test(); >- return new Promise(testcase.test); >- }); > >- if (testcase.teardown) { >- chain = chain.then(() => { >+ this._harness.log(`-- Running test case: ${testcase.name}`); >+ if (testcase.test[Symbol.toStringTag] === "AsyncFunction") >+ await testcase.test(); >+ else >+ await new Promise(testcase.test); >+ >+ if (testcase.teardown) { > this._harness.log("-- Running test teardown."); > if (testcase.teardown[Symbol.toStringTag] === "AsyncFunction") >- return testcase.teardown(); >- return new Promise(testcase.teardown); >- }); >+ await testcase.teardown(); >+ else >+ await new Promise(testcase.teardown); >+ } >+ } catch (e) { >+ this.failCount++; >+ this.logThrownObject(e); > } >- return chain; >- }, Promise.resolve()); > >- return result.catch((e) => { >- this.failCount++; >- this.logThrownObject(e); >+ if (!hasTimeout || timeoutId) { >+ clearTimeout(timeoutId); >+ timeoutId = undefined; > >- throw e; // Reject this promise by re-throwing the error. >- }); >+ resolve(); >+ } >+ }))) >+ // Clear result value. >+ .then(() => {})); > } > }; > >@@ -208,53 +230,45 @@ SyncTestSuite = class SyncTestSuite extends TestSuite > let priorLogCount = this._harness.logCount; > for (let i = 0; i < this.testcases.length; i++) { > let testcase = this.testcases[i]; >+ > if (i > 0 && priorLogCount + 1 < this._harness.logCount) > this._harness.log(""); >- > priorLogCount = this._harness.logCount; > >- // Run the setup action, if one was provided. >- if (testcase.setup) { >- this._harness.log("-- Running test setup."); >- try { >- let result = testcase.setup.call(null); >- if (result === false) { >- this._harness.log("!! SETUP FAILED"); >- return false; >- } >- } catch (e) { >- this.logThrownObject(e); >- return false; >- } >- } >- >- this._harness.log("-- Running test case: " + testcase.name); >- this.runCount++; > try { >- let result = testcase.test.call(null); >- if (result === false) { >+ // Run the setup action, if one was provided. >+ if (testcase.setup) { >+ this._harness.log("-- Running test setup."); >+ let setupResult = testcase.setup(); >+ if (setupResult === false) { >+ this._harness.log("!! SETUP FAILED"); >+ this.failCount++; >+ continue; >+ } >+ } >+ >+ this.runCount++; >+ >+ this._harness.log(`-- Running test case: ${testcase.name}`); >+ let testResult = testcase.test(); >+ if (testResult === false) { > this.failCount++; >- return false; >+ continue; >+ } >+ >+ // Run the teardown action, if one was provided. >+ if (testcase.teardown) { >+ this._harness.log("-- Running test teardown."); >+ let teardownResult = testcase.teardown(); >+ if (teardownResult === false) { >+ this._harness.log("!! TEARDOWN FAILED"); >+ this.failCount++; >+ continue; >+ } > } > } catch (e) { > this.failCount++; > this.logThrownObject(e); >- return false; >- } >- >- // Run the teardown action, if one was provided. >- if (testcase.teardown) { >- this._harness.log("-- Running test teardown."); >- try { >- let result = testcase.teardown.call(null); >- if (result === false) { >- this._harness.log("!! TEARDOWN FAILED"); >- return false; >- } >- } catch (e) { >- this.logThrownObject(e); >- return false; >- } > } > } > >diff --git a/LayoutTests/ChangeLog b/LayoutTests/ChangeLog >index 460ac15f2603986a67d9be648b879bba25faff67..1da4bc26b0dec49db3d8df369449cf39829645f5 100644 >--- a/LayoutTests/ChangeLog >+++ b/LayoutTests/ChangeLog >@@ -1,3 +1,35 @@ >+2019-04-04 Devin Rousso <drousso@apple.com> >+ >+ Web Inspector: TestSuite test cases should have their own timeout to ensure tests fail with output instead of timeout by test runner >+ https://bugs.webkit.org/show_bug.cgi?id=162814 >+ <rdar://problem/28574102> >+ >+ Reviewed by NOBODY (OOPS!). >+ >+ * inspector/unit-tests/async-test-suite.html: >+ * inspector/unit-tests/async-test-suite-expected.txt: >+ * inspector/unit-tests/sync-test-suite.html: >+ * inspector/unit-tests/sync-test-suite-expected.txt: >+ >+ * http/tests/inspector/network/set-resource-caching-disabled-disk-cache-expected.txt: >+ * inspector/canvas/recording-2d.html: >+ * inspector/canvas/recording-webgl-snapshots.html: >+ * inspector/canvas/recording-webgl.html: >+ * inspector/canvas/resources/shaderProgram-utilities.js: >+ (TestPage.registerInitializer.whenProgramAdded): Added. >+ (TestPage.registerInitializer.whenProgramRemoved): Added. >+ (TestPage.registerInitializer.window.initializeTestSuite): >+ (TestPage.registerInitializer.window.addSimpleTestCase): >+ (TestPage.registerInitializer.window.addParentCanvasRemovedTestCase): >+ (TestPage.registerInitializer.awaitProgramAdded): Added. >+ (TestPage.registerInitializer.awaitProgramRemoved): Added. >+ * inspector/console/command-line-api-expected.txt: >+ * inspector/console/heap-snapshot.html: >+ * inspector/debugger/async-stack-trace-truncate-expected.txt: >+ * inspector/debugger/pause-for-internal-scripts-expected.txt: >+ * inspector/formatting/resources/utilities.js: >+ (TestPage.registerInitializer.window.addFormattingTests): >+ > 2019-04-03 Timothy Hatcher <timothy@apple.com> > > Update AutoFill field icons to be SVG instead of PNG images. >diff --git a/LayoutTests/http/tests/inspector/network/set-resource-caching-disabled-disk-cache-expected.txt b/LayoutTests/http/tests/inspector/network/set-resource-caching-disabled-disk-cache-expected.txt >index 1e87a5c120798dcc4ec8704c69fcf05197db5aed..174f1a13efaee664c6f95c37eba34b5192793d0d 100644 >--- a/LayoutTests/http/tests/inspector/network/set-resource-caching-disabled-disk-cache-expected.txt >+++ b/LayoutTests/http/tests/inspector/network/set-resource-caching-disabled-disk-cache-expected.txt >@@ -6,8 +6,8 @@ Test for `Network.setResourceCachingDisabled` for disk cache loads. > -- Running test case: PossibleNetworkLoad > PASS: Resource should be created. > PASS: Resource should receive a Response. >--- Running test setup. > >+-- Running test setup. > -- Running test case: SetResourceCachingDisabled.DiskCache > PASS: Resource should be created. > PASS: Resource should receive a Response. >diff --git a/LayoutTests/inspector/canvas/recording-2d.html b/LayoutTests/inspector/canvas/recording-2d.html >index df20f0bf95c694c2653a5c50d0065b419bb900c6..a412029af66ea224705bcf99f6e02e5da3727b78 100644 >--- a/LayoutTests/inspector/canvas/recording-2d.html >+++ b/LayoutTests/inspector/canvas/recording-2d.html >@@ -444,6 +444,7 @@ function test() { > test(resolve, reject) { > startRecording(WI.Canvas.ContextType.Canvas2D, resolve, reject, {frameCount: 1}); > }, >+ timeout: -1, > }); > > suite.addTestCase({ >@@ -452,6 +453,7 @@ function test() { > test(resolve, reject) { > startRecording(WI.Canvas.ContextType.Canvas2D, resolve, reject); > }, >+ timeout: -1, > }); > > suite.addTestCase({ >@@ -460,6 +462,7 @@ function test() { > test(resolve, reject) { > startRecording(WI.Canvas.ContextType.Canvas2D, resolve, reject, {memoryLimit: 10}); > }, >+ timeout: -1, > }); > > suite.addTestCase({ >diff --git a/LayoutTests/inspector/canvas/recording-webgl-snapshots.html b/LayoutTests/inspector/canvas/recording-webgl-snapshots.html >index 6afc61ec305fa1d8971365497498b91d751ebd5b..b70c7c59ee6a0b03f5ac19ef8c01b1f6568016b9 100644 >--- a/LayoutTests/inspector/canvas/recording-webgl-snapshots.html >+++ b/LayoutTests/inspector/canvas/recording-webgl-snapshots.html >@@ -102,6 +102,7 @@ function test() { > test(resolve, reject) { > startRecording(WI.Canvas.ContextType.WebGL, resolve, reject, {frameCount: 1, checkForContentChange: true}); > }, >+ timeout: -1, > }); > > suite.runTestCasesAndFinish(); >diff --git a/LayoutTests/inspector/canvas/recording-webgl.html b/LayoutTests/inspector/canvas/recording-webgl.html >index 7672d76489406116738e1ff0ee2b68845cbad3d4..9e8109502c44904a03b1d8cb5e0ff82e1c5622b7 100644 >--- a/LayoutTests/inspector/canvas/recording-webgl.html >+++ b/LayoutTests/inspector/canvas/recording-webgl.html >@@ -526,6 +526,7 @@ function test() { > test(resolve, reject) { > startRecording(WI.Canvas.ContextType.WebGL, resolve, reject, {frameCount: 1}); > }, >+ timeout: -1, > }); > > suite.addTestCase({ >@@ -534,6 +535,7 @@ function test() { > test(resolve, reject) { > startRecording(WI.Canvas.ContextType.WebGL, resolve, reject); > }, >+ timeout: -1, > }); > > suite.addTestCase({ >@@ -542,6 +544,7 @@ function test() { > test(resolve, reject) { > startRecording(WI.Canvas.ContextType.WebGL, resolve, reject, {memoryLimit: 10}); > }, >+ timeout: -1, > }); > > suite.addTestCase({ >diff --git a/LayoutTests/inspector/canvas/resources/shaderProgram-utilities.js b/LayoutTests/inspector/canvas/resources/shaderProgram-utilities.js >index 74ea3ea74d983358633f81d72c6b2f9db0d7f231..5275c441aafc73b9107945b3cd73f9ca67e1e65e 100644 >--- a/LayoutTests/inspector/canvas/resources/shaderProgram-utilities.js >+++ b/LayoutTests/inspector/canvas/resources/shaderProgram-utilities.js >@@ -46,25 +46,23 @@ function deleteContext() { > TestPage.registerInitializer(() => { > let suite = null; > >- function awaitProgramAdded() { >+ function whenProgramAdded(callback) { > InspectorTest.assert(WI.canvasManager.canvases.length === 1, "There should only be one canvas."); >- return WI.canvasManager.canvases[0].shaderProgramCollection.awaitEvent(WI.Collection.Event.ItemAdded) >- .then((event) => { >+ WI.canvasManager.canvases[0].shaderProgramCollection.singleFireEventListener(WI.Collection.Event.ItemAdded, (event) => { > let program = event.data.item; > InspectorTest.expectThat(program instanceof WI.ShaderProgram, "Added ShaderProgram."); > InspectorTest.expectThat(program.canvas instanceof WI.Canvas, "ShaderProgram should have a parent Canvas."); >- return program; >+ callback(program); > }); > } > >- function awaitProgramRemoved() { >+ function whenProgramRemoved(callback) { > InspectorTest.assert(WI.canvasManager.canvases.length === 1, "There should only be one canvas."); >- return WI.canvasManager.canvases[0].shaderProgramCollection.awaitEvent(WI.Collection.Event.ItemRemoved) >- .then((event) => { >+ WI.canvasManager.canvases[0].shaderProgramCollection.singleFireEventListener(WI.Collection.Event.ItemRemoved, (event) => { > let program = event.data.item; > InspectorTest.expectThat(program instanceof WI.ShaderProgram, "Removed ShaderProgram."); > InspectorTest.expectThat(program.canvas instanceof WI.Canvas, "ShaderProgram should have a parent Canvas."); >- return program; >+ callback(program); > }); > } > >@@ -77,7 +75,9 @@ TestPage.registerInitializer(() => { > test(resolve, reject) { > // This can't use `awaitEvent` since the promise resolution happens on the next tick. > WI.canvasManager.singleFireEventListener(WI.CanvasManager.Event.CanvasAdded, (event) => { >- awaitProgramAdded().then(resolve, reject); >+ whenProgramAdded((program) => { >+ resolve(); >+ }); > }); > > InspectorTest.reloadPage(); >@@ -92,13 +92,11 @@ TestPage.registerInitializer(() => { > name: `${suite.name}.ShaderProgramAdded`, > description: "Check that added/removed events are sent.", > test(resolve, reject) { >- awaitProgramAdded() >- .then((addedProgram) => { >- awaitProgramRemoved() >- .then((removedProgram) => { >+ whenProgramAdded((addedProgram) => { >+ whenProgramRemoved((removedProgram) => { > InspectorTest.expectEqual(removedProgram, addedProgram, "Removed the previously added ShaderProgram."); >- }) >- .then(resolve, reject); >+ resolve(); >+ }); > > InspectorTest.evaluateInPage(`deleteProgram()`); > }); >@@ -113,16 +111,16 @@ TestPage.registerInitializer(() => { > name: `${suite.name}.ParentCanvasRemoved`, > description: "Check that the ShaderProgram is removed before it's parent Canvas.", > test(resolve, reject) { >- Promise.race([ >- awaitProgramRemoved() >- .then(() => { >- InspectorTest.pass("Removed ShaderProgram before Canvas."); >- resolve(); >- }), >- WI.canvasManager.awaitEvent(WI.CanvasManager.Event.CanvasRemoved) >- .then(reject) >- ]) >- .catch(() => { InspectorTest.fail("Removed Canvas before ShaderProgram."); }); >+ let canvasRemoved = false; >+ >+ WI.canvasManager.singleFireEventListener(WI.CanvasManager.Event.CanvasRemoved, (event) => { >+ canvasRemoved = true; >+ }); >+ >+ whenProgramRemoved((program) => { >+ InspectorTest.expectFalse(canvasRemoved, "Removed ShaderProgram before Canvas."); >+ resolve(); >+ }); > > InspectorTest.evaluateInPage(`createProgram("${contextType}")`); > InspectorTest.evaluateInPage(`deleteContext()`); >diff --git a/LayoutTests/inspector/console/command-line-api-expected.txt b/LayoutTests/inspector/console/command-line-api-expected.txt >index ed414cd4e4dbb7dc7b4450d5fda5233933254988..d3365b5efc02234038237f762f15f0c673518fb9 100644 >--- a/LayoutTests/inspector/console/command-line-api-expected.txt >+++ b/LayoutTests/inspector/console/command-line-api-expected.txt >@@ -10,13 +10,13 @@ Output: 0,1 > -- Running test case: EvaluateArrayValues > Input: values([3,4]) > Output: 3,4 >--- Running test setup. > >+-- Running test setup. > -- Running test case: EvaluateDollarZero > Input: $0 > Output: [object HTMLParagraphElement] >--- Running test setup. > >+-- Running test setup. > -- Running test case: EvaluateInvalidSelector > Input: $('foo') > CONSOLE: The console function $() has changed from $=getElementById(id) to $=querySelector(selector). You might try $("#%s") >diff --git a/LayoutTests/inspector/console/heapSnapshot.html b/LayoutTests/inspector/console/heapSnapshot.html >index 92be69d2fc2f6c2e4879d82605c80b6820cf59c5..c20e6063eebd767fed6a29a4f69312bafaaf672c 100644 >--- a/LayoutTests/inspector/console/heapSnapshot.html >+++ b/LayoutTests/inspector/console/heapSnapshot.html >@@ -27,7 +27,8 @@ function test() > }); > > ProtocolTest.evaluateInPage("triggerHeapSnapshotNoTitle()"); >- } >+ }, >+ timeout: -1, > }); > > suite.addTestCase({ >@@ -42,7 +43,8 @@ function test() > }); > > ProtocolTest.evaluateInPage("triggerHeapSnapshotWithTitle()"); >- } >+ }, >+ timeout: -1, > }); > > suite.runTestCasesAndFinish(); >diff --git a/LayoutTests/inspector/debugger/async-stack-trace-truncate-expected.txt b/LayoutTests/inspector/debugger/async-stack-trace-truncate-expected.txt >index 771c8b8ee3b27a5316b8844bb795140ac8d0785c..032f1b2e796dd6b01f0055085205700d9f97b8b6 100644 >--- a/LayoutTests/inspector/debugger/async-stack-trace-truncate-expected.txt >+++ b/LayoutTests/inspector/debugger/async-stack-trace-truncate-expected.txt >@@ -10,9 +10,9 @@ CALL STACK: > 0: [F] handleAnimationFrame > PASS: Async stack trace should be null. > -- Running test teardown. >+ > -- Running test setup. > Set maximum stack trace depth = 10. >- > -- Running test case: AsyncStackTrace.CheckTruncated > PAUSED > CALL STACK: >@@ -31,9 +31,9 @@ ASYNC CALL STACK: > (remaining call frames truncated) > PASS: Async stack trace should be truncated. > -- Running test teardown. >+ > -- Running test setup. > Set maximum stack trace depth = 10. >- > -- Running test case: AsyncStackTrace.CheckNotTruncated > PAUSED > CALL STACK: >diff --git a/LayoutTests/inspector/debugger/pause-for-internal-scripts-expected.txt b/LayoutTests/inspector/debugger/pause-for-internal-scripts-expected.txt >index 92eee8b1e5e04a4335097931495c46b7a11cefd1..53c63e7afad319e5174e976f84fc93e4b7717876 100644 >--- a/LayoutTests/inspector/debugger/pause-for-internal-scripts-expected.txt >+++ b/LayoutTests/inspector/debugger/pause-for-internal-scripts-expected.txt >@@ -65,8 +65,8 @@ PAUSE AT entryConsoleLog:12:2 > ACTION: resume > RESUMED > PASS: Should have used all steps. >--- Running test setup. > >+-- Running test setup. > -- Running test case: Debugger.setPauseForInternalScripts.Enabled > EXPRESSION: setTimeout(entryConsoleLog) > STEPS: over, in, in, in, resume >diff --git a/LayoutTests/inspector/formatting/resources/utilities.js b/LayoutTests/inspector/formatting/resources/utilities.js >index 644cebcec2ed08ad03ac6b5c7d49e2c2bdc51f01..9b4e326ec043e5cfbaec2f304859bfd9f2ba7de5 100644 >--- a/LayoutTests/inspector/formatting/resources/utilities.js >+++ b/LayoutTests/inspector/formatting/resources/utilities.js >@@ -43,7 +43,7 @@ TestPage.registerInitializer(function() { > > window.addFormattingTests = function(suite, mode, tests) { > let testPageURL = WI.networkManager.mainFrame.mainResource.url; >- let testPageResourcesURL = testPageURL.substring(0, testPageURL.lastIndexOf("/")); >+ let testPageResourcesURL = testPageURL.substring(0, testPageURL.lastIndexOf("/")); > > for (let test of tests) { > let testName = test.substring(test.lastIndexOf("/") + 1); >@@ -52,7 +52,8 @@ TestPage.registerInitializer(function() { > name: suite.name + "." + testName, > test(resolve, reject) { > runFormattingTest(mode, testName, testURL).then(resolve).catch(reject); >- } >+ }, >+ timeout: -1, > }); > } > }; >diff --git a/LayoutTests/inspector/unit-tests/async-test-suite-expected.txt b/LayoutTests/inspector/unit-tests/async-test-suite-expected.txt >index 8b9beb0a89b5ba0a659fa27d4d858827e9771f4b..9be6e8fa055eaada568e243f6549b76157c3ec24 100644 >--- a/LayoutTests/inspector/unit-tests/async-test-suite-expected.txt >+++ b/LayoutTests/inspector/unit-tests/async-test-suite-expected.txt >@@ -1,152 +1,441 @@ >-PASS: instantiating AsyncTestSuite requires name argument. >-PASS: instantiating AsyncTestSuite requires string name argument. >-PASS: instantiating AsyncTestSuite requires non-whitespace name argument. >-PASS: instantiating AsyncTestSuite requires test harness argument. >-PASS: should not be able to add empty test case. >-PASS: should not be able to add non-object test case. >-PASS: test case should require string name. >-PASS: test case should require non-whitespace name. >-PASS: test case should require test function. >-PASS: should not be able to specify non-Function `setup` parameter. >-PASS: should not be able to specify non-Function `setup` parameter. >-PASS: should not be able to specify non-Function `setup` parameter. >-PASS: should not be able to specify non-Function `teardown` parameter. >-PASS: should not be able to specify non-Function `teardown` parameter. >-PASS: should not be able to specify non-Function `teardown` parameter. >+PASS: Should produce an exception. >+Error: Must pass the test's harness as the first argument. >+PASS: Should produce an exception. >+Error: Must pass the test's harness as the first argument. >+PASS: Should produce an exception. >+Error: Must pass the test's harness as the first argument. >+PASS: Should produce an exception. >+Error: Must pass the test's harness as the first argument. >+PASS: Should produce an exception. >+Error: Tried to add non-object test case. >+PASS: Should produce an exception. >+Error: Tried to add non-object test case. >+PASS: Should produce an exception. >+Error: Tried to add test case without a name. >+PASS: Should produce an exception. >+Error: Tried to add test case without a name. >+PASS: Should produce an exception. >+Error: Tried to add test case without `test` function. >+PASS: Should produce an exception. >+Error: Tried to add test case with invalid `setup` parameter (must be a function). >+PASS: Should produce an exception. >+Error: Tried to add test case with invalid `setup` parameter (must be a function). >+PASS: Should produce an exception. >+Error: Tried to add test case with invalid `setup` parameter (must be a function). >+PASS: Should produce an exception. >+Error: Tried to add test case with invalid `teardown` parameter (must be a function). >+PASS: Should produce an exception. >+Error: Tried to add test case with invalid `teardown` parameter (must be a function). >+PASS: Should produce an exception. >+Error: Tried to add test case with invalid `teardown` parameter (must be a function). > PASS: should not be able to run empty test suite. > > == Running test suite: AsyncTestSuite.RunTwiceSuite >-PASS: should not be able to run a test suite twice. >+PASS: Should produce an exception. >+Error: Tried to call runTestCases() more than once. > -- Running test case: DummyTest0 >+PASS: DummyTest0 should only run once. > >-== Running test suite: AsyncTestSuite.SequentialExecution >-PASS: AsyncTestSuite.RunTestCases() should return a Promise. >--- Running test case: DummyTest1 >--- Running test case: DummyTest2 >--- Running test case: DummyTest3 >--- Running test case: FailingTest4 >-!! EXCEPTION: [object Object] >+== Running test suite: AsyncTestSuite.PromiseFunctionSuccess >+-- Running test case: PromiseFunctionSuccess >+ >+PASS: Promise from AsyncTestSuite.PromiseFunctionSuccess.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.PromiseFunctionSuccess should have executed 1 tests. >+PASS: AsyncTestSuite.PromiseFunctionSuccess should have passed 1 tests. >+PASS: AsyncTestSuite.PromiseFunctionSuccess should have failed 0 tests. >+PASS: AsyncTestSuite.PromiseFunctionSuccess should have skipped 0 tests. >+ >+== Running test suite: PromiseTestSuite.PromiseFunctionException >+-- Running test case: PromiseFunctionException >+Throwing... >+!! EXCEPTION: PromiseFunctionException throw > Stack Trace: (suppressed) >-PASS: Promise from sequentialExecutionSuite.runTestCases() should reject when a test case fails. >-PASS: Promise from sequentialExecutionSuite.runTestCases() should reject without altering its result value. >-PASS: sequentialExecutionSuite should have executed four tests. >-PASS: sequentialExecutionSuite should have passed three tests. >-PASS: sequentialExecutionSuite should have failed 1 test. >-PASS: sequentialExecutionSuite should have skipped zero tests. > >-== Running test suite: AsyncTestSuite.AbortOnFailure >--- Running test case: PassingTest5 >--- Running test case: FailingTest6 >-!! EXCEPTION: {"token":666} >+PASS: Promise from PromiseTestSuite.PromiseFunctionException.runTestCases() should resolve even if a test case fails. >+PASS: PromiseTestSuite.PromiseFunctionException should have executed 1 tests. >+PASS: PromiseTestSuite.PromiseFunctionException should have passed 0 tests. >+PASS: PromiseTestSuite.PromiseFunctionException should have failed 1 tests. >+PASS: PromiseTestSuite.PromiseFunctionException should have skipped 0 tests. >+ >+== Running test suite: PromiseTestSuite.PromiseFunctionFailure >+-- Running test case: PromiseFunctionFailure >+Rejecting... >+!! EXCEPTION: PromiseFunctionFailure reject > Stack Trace: (suppressed) >-PASS: Promise from abortOnFailureSuite.runTestCases() should reject when a test case fails. >-PASS: Promise from abortOnFailureSuite.runTestCases() should reject without altering its result value. >-PASS: abortOnFailureSuite should have executed two tests. >-PASS: abortOnFailureSuite should have passed one test. >-PASS: abortOnFailureSuite should have failed one test. >-PASS: abortOnFailureSuite should have skipped one test. > >-== Running test suite: AsyncTestSuite.SetupAndTeardown >+PASS: Promise from PromiseTestSuite.PromiseFunctionFailure.runTestCases() should resolve even if a test case fails. >+PASS: PromiseTestSuite.PromiseFunctionFailure should have executed 1 tests. >+PASS: PromiseTestSuite.PromiseFunctionFailure should have passed 0 tests. >+PASS: PromiseTestSuite.PromiseFunctionFailure should have failed 1 tests. >+PASS: PromiseTestSuite.PromiseFunctionFailure should have skipped 0 tests. >+ >+== Running test suite: AsyncTestSuite.PromiseSequentialExecution >+-- Running test case: 1 (Pass) >+-- Running test case: 2 (Pass) >+-- Running test case: 3 (Pass) >+-- Running test case: 4 (Pass) >+ >+PASS: Promise from AsyncTestSuite.PromiseSequentialExecution.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.PromiseSequentialExecution should have executed 4 tests. >+PASS: AsyncTestSuite.PromiseSequentialExecution should have passed 4 tests. >+PASS: AsyncTestSuite.PromiseSequentialExecution should have failed 0 tests. >+PASS: AsyncTestSuite.PromiseSequentialExecution should have skipped 0 tests. >+ >+== Running test suite: AsyncTestSuite.PromiseContinueOnFailure >+-- Running test case: 1 (Pass) >+-- Running test case: 2 (Fail) >+Throwing... >+!! EXCEPTION: {"x":"PromiseContinueOnFailure throw"} >+Stack Trace: (suppressed) >+ >+-- Running test case: 3 (Pass) >+-- Running test case: 4 (Fail) >+Rejecting... >+!! EXCEPTION: {"x":"PromiseContinueOnFailure reject"} >+Stack Trace: (suppressed) >+ >+PASS: Promise from AsyncTestSuite.PromiseContinueOnFailure.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.PromiseContinueOnFailure should have executed 4 tests. >+PASS: AsyncTestSuite.PromiseContinueOnFailure should have passed 2 tests. >+PASS: AsyncTestSuite.PromiseContinueOnFailure should have failed 2 tests. >+PASS: AsyncTestSuite.PromiseContinueOnFailure should have skipped 0 tests. >+ >+== Running test suite: AsyncTestSuite.PromiseSetupAndTeardown > -- Running test setup. > -- Running test case: TestWithSetupAndTeardown >-PASS: Test should see side effects of running setup() action. > -- Running test teardown. >-PASS: Teardown should see side effects of running setup() action. > > -- Running test case: TestRunningAfterTeardown >-PASS: Test should see side effects of previous test's teardown() action. >-PASS: Promise from setupAndTeardownTestSuite.runTestCases() should resolve. > >-== Running test suite: AsyncTestSuite.SetupException >+PASS: Promise from AsyncTestSuite.PromiseSetupAndTeardown.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.PromiseSetupAndTeardown should have executed 2 tests. >+PASS: AsyncTestSuite.PromiseSetupAndTeardown should have passed 2 tests. >+PASS: AsyncTestSuite.PromiseSetupAndTeardown should have failed 0 tests. >+PASS: AsyncTestSuite.PromiseSetupAndTeardown should have skipped 0 tests. >+ >+== Running test suite: AsyncTestSuite.PromiseSetupException > -- Running test setup. >-!! EXCEPTION: >+Throwing... >+!! EXCEPTION: PromiseSetupException throw > Stack Trace: (suppressed) >-PASS: Promise from setupExceptionTestSuite.runTestCases() should reject. > >-== Running test suite: AsyncTestSuite.SetupFailure > -- Running test setup. >-!! EXCEPTION: undefined >-Stack Trace: (suppressed) >-PASS: Promise from setupFailureTestSuite.runTestCases() should reject. >+PASS: Setup action should still execute if previous test's setup action threw an exception. >+-- Running test case: TestAfterSetupException >+PASS: Test should still execute if previous test's setup action threw an exception. >+-- Running test teardown. >+PASS: Teardown action should still execute if previous test's setup action threw an exception. > >-== Running test suite: AsyncTestSuite.TeardownException >+PASS: Promise from AsyncTestSuite.PromiseSetupException.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.PromiseSetupException should have executed 1 tests. >+PASS: AsyncTestSuite.PromiseSetupException should have passed 1 tests. >+PASS: AsyncTestSuite.PromiseSetupException should have failed 1 tests. >+PASS: AsyncTestSuite.PromiseSetupException should have skipped 1 tests. >+ >+== Running test suite: AsyncTestSuite.PromiseSetupFailure >+-- Running test setup. >+Rejecting... >+!! EXCEPTION: PromiseSetupFailure reject >+Stack Trace: (suppressed) >+ >+-- Running test setup. >+PASS: Setup action should still execute if previous test's setup action failed. >+-- Running test case: TestAfterSetupException >+PASS: Test should still execute if previous test's setup action failed. >+-- Running test teardown. >+PASS: Teardown action should still execute if previous test's setup action failed. >+ >+PASS: Promise from AsyncTestSuite.PromiseSetupFailure.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.PromiseSetupFailure should have executed 1 tests. >+PASS: AsyncTestSuite.PromiseSetupFailure should have passed 1 tests. >+PASS: AsyncTestSuite.PromiseSetupFailure should have failed 1 tests. >+PASS: AsyncTestSuite.PromiseSetupFailure should have skipped 1 tests. >+ >+== Running test suite: AsyncTestSuite.PromiseTeardownException > -- Running test case: TestWithExceptionDuringTeardown > -- Running test teardown. >-!! EXCEPTION: >+Throwing... >+!! EXCEPTION: PromiseTeardownException throw > Stack Trace: (suppressed) >-PASS: Promise from teardownExceptionTestSuite.runTestCases() should reject. > >-== Running test suite: AsyncTestSuite.TeardownFailure >+-- Running test setup. >+PASS: Setup action should still execute if previous test's teardown action threw an exception. >+-- Running test case: TestAfterTeardownException >+PASS: Test should still execute if previous test's teardown action threw an exception. >+-- Running test teardown. >+PASS: Teardown action should still execute if previous test's teardown action threw an exception. >+ >+PASS: Promise from AsyncTestSuite.PromiseTeardownException.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.PromiseTeardownException should have executed 2 tests. >+PASS: AsyncTestSuite.PromiseTeardownException should have passed 1 tests. >+PASS: AsyncTestSuite.PromiseTeardownException should have failed 1 tests. >+PASS: AsyncTestSuite.PromiseTeardownException should have skipped 0 tests. >+ >+== Running test suite: AsyncTestSuite.PromiseTeardownFailure > -- Running test case: TestWithExceptionDuringTeardown > -- Running test teardown. >-!! EXCEPTION: undefined >+Rejecting... >+!! EXCEPTION: PromiseTeardownFailure reject > Stack Trace: (suppressed) >-PASS: Promise from teardownFailureTestSuite.runTestCases() should reject. >+ >+-- Running test setup. >+PASS: Setup action should still execute if previous test's teardown action failed. >+-- Running test case: TestAfterTeardownException >+PASS: Test should still execute if previous test's teardown action failed. >+-- Running test teardown. >+PASS: Teardown action should still execute if previous test's teardown action failed. >+ >+PASS: Promise from AsyncTestSuite.PromiseTeardownFailure.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.PromiseTeardownFailure should have executed 2 tests. >+PASS: AsyncTestSuite.PromiseTeardownFailure should have passed 1 tests. >+PASS: AsyncTestSuite.PromiseTeardownFailure should have failed 1 tests. >+PASS: AsyncTestSuite.PromiseTeardownFailure should have skipped 0 tests. >+ >+== Running test suite: AsyncTestSuite.PromiseTimeout >+-- Running test case: PromiseTestWithTimeout >+Timeout... >+!! TIMEOUT: took longer than 5ms >+ >+-- Running test setup. >+PASS: Setup action should still execute if previous test timed out. >+-- Running test case: PromiseTestAfterTimeout >+PASS: Test should still execute if previous test timed out. >+-- Running test teardown. >+PASS: Teardown action should still execute if previous test timed out. >+ >+PASS: Promise from AsyncTestSuite.PromiseTimeout.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.PromiseTimeout should have executed 2 tests. >+PASS: AsyncTestSuite.PromiseTimeout should have passed 1 tests. >+PASS: AsyncTestSuite.PromiseTimeout should have failed 1 tests. >+PASS: AsyncTestSuite.PromiseTimeout should have skipped 0 tests. > > == Running test suite: AsyncTestSuite.AsyncFunctionSuccess > -- Running test case: AsyncFunctionSuccess >-PASS: Promise from asyncFunctionSuccessTestSuite.runTestCases() should succeed. >-PASS: Promise did evaluate the async test function. >-PASS: Resolved value should be 42. > >-== Running test suite: AsyncTestSuite.AsyncFunctionExplicitFailure >--- Running test case: AsyncFunctionFailure >-!! EXCEPTION: AsyncFunctionFailure Exception Message >+PASS: Promise from AsyncTestSuite.AsyncFunctionSuccess.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.AsyncFunctionSuccess should have executed 1 tests. >+PASS: AsyncTestSuite.AsyncFunctionSuccess should have passed 1 tests. >+PASS: AsyncTestSuite.AsyncFunctionSuccess should have failed 0 tests. >+PASS: AsyncTestSuite.AsyncFunctionSuccess should have skipped 0 tests. >+ >+== Running test suite: AsyncTestSuite.AsyncFunctionExplicitException >+-- Running test case: AsyncFunctionExplicitException >+Throwing... >+!! EXCEPTION: AsyncFunctionExplicitException throw > Stack Trace: (suppressed) >-PASS: Promise from asyncFunctionExplicitFailureTestSuite.runTestCases() should reject. >-PASS: Promise did evaluate the async test function. >-PASS: Rejected value should be thrown exception. > >-== Running test suite: AsyncTestSuite.AsyncFunctionRuntimeFailure >--- Running test case: AsyncFunctionFailure >+PASS: Promise from AsyncTestSuite.AsyncFunctionExplicitException.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.AsyncFunctionExplicitException should have executed 1 tests. >+PASS: AsyncTestSuite.AsyncFunctionExplicitException should have passed 0 tests. >+PASS: AsyncTestSuite.AsyncFunctionExplicitException should have failed 1 tests. >+PASS: AsyncTestSuite.AsyncFunctionExplicitException should have skipped 0 tests. >+ >+== Running test suite: AsyncTestSuite.AsyncFunctionRuntimeException >+-- Running test case: AsyncFunctionRuntimeException >+Throwing... > !! EXCEPTION: undefined is not an object (evaluating '({}).x.x') > Stack Trace: (suppressed) >-PASS: Promise from asyncFunctionRuntimeFailureTestSuite.runTestCases() should reject. >-PASS: Promise did evaluate the async test function. >-PASS: Rejected value should be a runtime exception. > >-== Running test suite: AsyncTestSuite.AsyncSetupAndAsyncTeardown >+PASS: Promise from AsyncTestSuite.AsyncFunctionRuntimeException.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.AsyncFunctionRuntimeException should have executed 1 tests. >+PASS: AsyncTestSuite.AsyncFunctionRuntimeException should have passed 0 tests. >+PASS: AsyncTestSuite.AsyncFunctionRuntimeException should have failed 1 tests. >+PASS: AsyncTestSuite.AsyncFunctionRuntimeException should have skipped 0 tests. >+ >+== Running test suite: AsyncTestSuite.AsyncFunctionFailure >+-- Running test case: AsyncFunctionException >+Rejecting... >+!! EXCEPTION: AsyncFunctionFailure reject >+Stack Trace: (suppressed) >+ >+PASS: Promise from AsyncTestSuite.AsyncFunctionFailure.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.AsyncFunctionFailure should have executed 1 tests. >+PASS: AsyncTestSuite.AsyncFunctionFailure should have passed 0 tests. >+PASS: AsyncTestSuite.AsyncFunctionFailure should have failed 1 tests. >+PASS: AsyncTestSuite.AsyncFunctionFailure should have skipped 0 tests. >+ >+== Running test suite: AsyncTestSuite.AsyncSequentialExecution >+-- Running test case: 1 (Pass) >+-- Running test case: 2 (Pass) >+-- Running test case: 3 (Pass) >+-- Running test case: 4 (Pass) >+ >+PASS: Promise from AsyncTestSuite.AsyncSequentialExecution.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.AsyncSequentialExecution should have executed 4 tests. >+PASS: AsyncTestSuite.AsyncSequentialExecution should have passed 4 tests. >+PASS: AsyncTestSuite.AsyncSequentialExecution should have failed 0 tests. >+PASS: AsyncTestSuite.AsyncSequentialExecution should have skipped 0 tests. >+ >+== Running test suite: AsyncTestSuite.AsyncContinueOnFailure >+-- Running test case: 1 (Pass) >+-- Running test case: 2 (Fail) >+Throwing... >+!! EXCEPTION: {"x":"AsyncContinueOnFailure throw"} >+Stack Trace: (suppressed) >+ >+-- Running test case: 3 (Pass) >+-- Running test case: 4 (Fail) >+Throwing... >+!! EXCEPTION: undefined is not an object (evaluating '({}).x.x') >+Stack Trace: (suppressed) >+ >+-- Running test case: 5 (Pass) >+-- Running test case: 6 (Fail) >+Rejecting... >+!! EXCEPTION: AsyncContinueOnFailure reject >+Stack Trace: (suppressed) >+ >+PASS: Promise from AsyncTestSuite.AsyncContinueOnFailure.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.AsyncContinueOnFailure should have executed 6 tests. >+PASS: AsyncTestSuite.AsyncContinueOnFailure should have passed 3 tests. >+PASS: AsyncTestSuite.AsyncContinueOnFailure should have failed 3 tests. >+PASS: AsyncTestSuite.AsyncContinueOnFailure should have skipped 0 tests. >+ >+== Running test suite: AsyncTestSuite.AsyncSetupAndTeardown > -- Running test setup. >--- Running test case: TestWithSetupAndTeardown >-PASS: Test should see side effects of running setup() action. >+-- Running test case: AsyncTestWithSetupAndTeardown > -- Running test teardown. >-PASS: Teardown should see side effects of running setup() action. > >--- Running test case: TestRunningAfterTeardown >-PASS: Test should see side effects of previous test's teardown() action. >-PASS: Promise from asyncSetupAndAsyncTeardownTestSuite.runTestCases() should resolve. >+-- Running test case: AsyncTestRunningAfterTeardown > >-== Running test suite: AsyncTestSuite.AsyncSetupExplicitFailure >--- Running test case: AsyncFunctionFailure >-!! EXCEPTION: AsyncFunctionFailure Exception Message >-Stack Trace: (suppressed) >-PASS: Promise from asyncSetupExplicitFailureTestSuite.runTestCases() should reject. >-PASS: Promise did evaluate the async setup function. >-PASS: Rejected value should be thrown exception. >+PASS: Promise from AsyncTestSuite.AsyncSetupAndTeardown.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.AsyncSetupAndTeardown should have executed 2 tests. >+PASS: AsyncTestSuite.AsyncSetupAndTeardown should have passed 2 tests. >+PASS: AsyncTestSuite.AsyncSetupAndTeardown should have failed 0 tests. >+PASS: AsyncTestSuite.AsyncSetupAndTeardown should have skipped 0 tests. > >-== Running test suite: AsyncTestSuite.AsyncSetupRuntimeFailure >+== Running test suite: AsyncTestSuite.AsyncSetupExplicitException > -- Running test setup. >+Throwing... >+!! EXCEPTION: AsyncSetupExplicitException throw >+Stack Trace: (suppressed) >+ >+-- Running test setup. >+PASS: Setup action should still execute if previous test's setup action threw an exception. >+-- Running test case: AsyncTestAfterSetupExplicitException >+PASS: Test should still execute if previous test's setup action threw an exception. >+-- Running test teardown. >+PASS: Teardown action should still execute if previous test's setup action threw an exception. >+ >+PASS: Promise from AsyncTestSuite.AsyncSetupExplicitException.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.AsyncSetupExplicitException should have executed 1 tests. >+PASS: AsyncTestSuite.AsyncSetupExplicitException should have passed 1 tests. >+PASS: AsyncTestSuite.AsyncSetupExplicitException should have failed 1 tests. >+PASS: AsyncTestSuite.AsyncSetupExplicitException should have skipped 1 tests. >+ >+== Running test suite: AsyncTestSuite.AsyncSetupRuntimeException >+-- Running test setup. >+Throwing... > !! EXCEPTION: undefined is not an object (evaluating '({}).x.x') > Stack Trace: (suppressed) >-PASS: Promise from asyncSetupRuntimeFailureTestSuite.runTestCases() should reject. >-PASS: Promise did evaluate the async setup function. >-PASS: Rejected value should be a runtime exception. > >-== Running test suite: AsyncTestSuite.AsyncTeardownExplicitFailure >--- Running test case: AsyncFunctionFailure >+-- Running test setup. >+PASS: Setup action should still execute if previous test's setup action threw an exception. >+-- Running test case: AsyncTestAfterSetupRuntimeException >+PASS: Test should still execute if previous test's setup action threw an exception. > -- Running test teardown. >-!! EXCEPTION: AsyncFunctionFailure Exception Message >+PASS: Teardown action should still execute if previous test's setup action threw an exception. >+ >+PASS: Promise from AsyncTestSuite.AsyncSetupRuntimeException.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.AsyncSetupRuntimeException should have executed 1 tests. >+PASS: AsyncTestSuite.AsyncSetupRuntimeException should have passed 1 tests. >+PASS: AsyncTestSuite.AsyncSetupRuntimeException should have failed 1 tests. >+PASS: AsyncTestSuite.AsyncSetupRuntimeException should have skipped 1 tests. >+ >+== Running test suite: AsyncTestSuite.AsyncSetupFailure >+-- Running test setup. >+Rejecting... >+!! EXCEPTION: AsyncSetupFailure reject > Stack Trace: (suppressed) >-PASS: Promise from asyncTeardownExplicitFailureTestSuite.runTestCases() should reject. >-PASS: Promise did evaluate the async teardown function. >-PASS: Rejected value should be thrown exception. > >-== Running test suite: AsyncTestSuite.AsyncTeardownRuntimeFailure >--- Running test case: AsyncFunctionFailure >+-- Running test setup. >+PASS: Setup action should still execute if previous test's setup action failed. >+-- Running test case: AsyncTestAfterSetupFailure >+PASS: Test should still execute if previous test's setup action failed. > -- Running test teardown. >+PASS: Setup action should still execute if previous test's setup action failed. >+ >+PASS: Promise from AsyncTestSuite.AsyncSetupFailure.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.AsyncSetupFailure should have executed 1 tests. >+PASS: AsyncTestSuite.AsyncSetupFailure should have passed 1 tests. >+PASS: AsyncTestSuite.AsyncSetupFailure should have failed 1 tests. >+PASS: AsyncTestSuite.AsyncSetupFailure should have skipped 1 tests. >+ >+== Running test suite: AsyncTestSuite.AsyncTeardownExplicitException >+-- Running test case: AsyncTestWithExplicitExceptionDuringTeardown >+-- Running test teardown. >+Throwing... >+!! EXCEPTION: AsyncTeardownExplicitException throw >+Stack Trace: (suppressed) >+ >+-- Running test setup. >+PASS: Setup action should still execute if previous test's teardown action threw an exception. >+-- Running test case: AsyncTestAfterTeardownExplicitException >+PASS: Test should still execute if previous test's teardown action threw an exception. >+-- Running test teardown. >+PASS: Teardown action should still execute if previous test's teardown action threw an exception. >+ >+PASS: Promise from AsyncTestSuite.AsyncTeardownExplicitException.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.AsyncTeardownExplicitException should have executed 2 tests. >+PASS: AsyncTestSuite.AsyncTeardownExplicitException should have passed 1 tests. >+PASS: AsyncTestSuite.AsyncTeardownExplicitException should have failed 1 tests. >+PASS: AsyncTestSuite.AsyncTeardownExplicitException should have skipped 0 tests. >+ >+== Running test suite: AsyncTestSuite.AsyncTeardownRuntimeException >+-- Running test case: AsyncTestWithRuntimeExceptionDuringTeardown >+-- Running test teardown. >+Throwing... > !! EXCEPTION: undefined is not an object (evaluating '({}).x.x') > Stack Trace: (suppressed) >-PASS: Promise from asyncTeardownRuntimeFailureTestSuite.runTestCases() should reject. >-PASS: Promise did evaluate the async teardown function. >-PASS: Rejected value should be a runtime exception. >+ >+-- Running test setup. >+PASS: Setup action should still execute if previous test's teardown action threw an exception. >+-- Running test case: AsyncTestAfterTeardownRuntimeException >+PASS: Test should still execute if previous test's teardown action threw an exception. >+-- Running test teardown. >+PASS: Teardown action should still execute if previous test's teardown action threw an exception. >+ >+PASS: Promise from AsyncTestSuite.AsyncTeardownRuntimeException.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.AsyncTeardownRuntimeException should have executed 2 tests. >+PASS: AsyncTestSuite.AsyncTeardownRuntimeException should have passed 1 tests. >+PASS: AsyncTestSuite.AsyncTeardownRuntimeException should have failed 1 tests. >+PASS: AsyncTestSuite.AsyncTeardownRuntimeException should have skipped 0 tests. >+ >+== Running test suite: AsyncTestSuite.AsyncTeardownFailure >+-- Running test case: AsyncTestWithFailureDuringTeardown >+-- Running test teardown. >+Rejecting... >+!! EXCEPTION: AsyncTeardownFailure reject >+Stack Trace: (suppressed) >+ >+-- Running test setup. >+PASS: Setup action should still execute if previous test's teardown action failed. >+-- Running test case: AsyncTestAfterTeardownFailure >+PASS: Test should still execute if previous test's teardown action failed. >+-- Running test teardown. >+PASS: Teardown action should still execute if previous test's teardown action failed. >+ >+PASS: Promise from AsyncTestSuite.AsyncTeardownFailure.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.AsyncTeardownFailure should have executed 2 tests. >+PASS: AsyncTestSuite.AsyncTeardownFailure should have passed 1 tests. >+PASS: AsyncTestSuite.AsyncTeardownFailure should have failed 1 tests. >+PASS: AsyncTestSuite.AsyncTeardownFailure should have skipped 0 tests. >+ >+== Running test suite: AsyncTestSuite.AsyncTimeout >+-- Running test case: AsyncTestWithTimeout >+Timeout... >+!! TIMEOUT: took longer than 5ms >+ >+-- Running test setup. >+PASS: Setup action should still execute if previous test timed out. >+-- Running test case: AsyncTestAfterTimeout >+PASS: Test should still execute if previous test timed out. >+-- Running test teardown. >+PASS: Teardown action should still execute if previous test timed out. >+ >+PASS: Promise from AsyncTestSuite.AsyncTimeout.runTestCases() should resolve even if a test case fails. >+PASS: AsyncTestSuite.AsyncTimeout should have executed 2 tests. >+PASS: AsyncTestSuite.AsyncTimeout should have passed 1 tests. >+PASS: AsyncTestSuite.AsyncTimeout should have failed 1 tests. >+PASS: AsyncTestSuite.AsyncTimeout should have skipped 0 tests. > >diff --git a/LayoutTests/inspector/unit-tests/async-test-suite.html b/LayoutTests/inspector/unit-tests/async-test-suite.html >index 7794d1043734f9b806a4f41226d436f1fb1267ae..5a03db7005b57592b850bde93f8592ef34e5c5a2 100644 >--- a/LayoutTests/inspector/unit-tests/async-test-suite.html >+++ b/LayoutTests/inspector/unit-tests/async-test-suite.html >@@ -1,4 +1,4 @@ >-<!doctype html> >+<!DOCTYPE html> > <html> > <head> > <script src="../../http/tests/inspector/resources/protocol-test.js"></script> >@@ -7,134 +7,97 @@ function test() > { > ProtocolTest.suppressStackTraces = true; > >- try { >- let result = new AsyncTestSuite(this); >- ProtocolTest.fail("instantiating AsyncTestSuite requires name argument."); >- } catch (e) { >- ProtocolTest.pass("instantiating AsyncTestSuite requires name argument."); >- } >+ ProtocolTest.expectException(() => { >+ new AsyncTestSuite(this); >+ }); > >- try { >- let result = new AsyncTestSuite(this, {}); >- ProtocolTest.fail("instantiating AsyncTestSuite requires string name argument."); >- } catch (e) { >- ProtocolTest.pass("instantiating AsyncTestSuite requires string name argument."); >- } >+ ProtocolTest.expectException(() => { >+ new AsyncTestSuite(this, {}); >+ }); > >- try { >- let result = new AsyncTestSuite(this, " "); >- ProtocolTest.fail("instantiating AsyncTestSuite requires non-whitespace name argument."); >- } catch (e) { >- ProtocolTest.pass("instantiating AsyncTestSuite requires non-whitespace name argument."); >- } >+ ProtocolTest.expectException(() => { >+ new AsyncTestSuite(this, " "); >+ }); > >- try { >- let result = new AsyncTestSuite("something", {}); >- ProtocolTest.fail("instantiating AsyncTestSuite requires test harness argument."); >- } catch (e) { >- ProtocolTest.pass("instantiating AsyncTestSuite requires test harness argument."); >- } >+ ProtocolTest.expectException(() => { >+ new AsyncTestSuite("something", {}); >+ }); > > let badArgsSuite = ProtocolTest.createAsyncSuite("dummy"); >- try { >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase(); >- ProtocolTest.fail("should not be able to add empty test case."); >- } catch (e) { >- ProtocolTest.pass("should not be able to add empty test case."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase("string"); >- ProtocolTest.fail("should not be able to add non-object test case."); >- } catch (e) { >- ProtocolTest.pass("should not be able to add non-object test case."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: {}, >- test() {}, >+ test() { >+ }, > }); >- ProtocolTest.fail("test case should require string name."); >- } catch (e) { >- ProtocolTest.pass("test case should require string name."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: " ", >- test() {}, >+ test() { >+ }, > }); >- ProtocolTest.fail("test case should require non-whitespace name."); >- } catch (e) { >- ProtocolTest.pass("test case should require non-whitespace name."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: "foo", > test: null, > }); >- ProtocolTest.fail("test case should require test function."); >- } catch (e) { >- ProtocolTest.pass("test case should require test function."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: "foo", >- test() {}, >- setup: "astd" >+ test() { >+ }, >+ setup: "astd", > }); >- ProtocolTest.fail("should not be able to specify non-Function `setup` parameter."); >- } catch (e) { >- ProtocolTest.pass("should not be able to specify non-Function `setup` parameter."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: "foo", >- test() {}, >- setup: 123 >+ test() { >+ }, >+ setup: 123, > }); >- ProtocolTest.fail("should not be able to specify non-Function `setup` parameter."); >- } catch (e) { >- ProtocolTest.pass("should not be able to specify non-Function `setup` parameter."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: "foo", >- test() {}, >- setup: {} >+ test() { >+ }, >+ setup: {}, > }); >- ProtocolTest.fail("should not be able to specify non-Function `setup` parameter."); >- } catch (e) { >- ProtocolTest.pass("should not be able to specify non-Function `setup` parameter."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: "foo", >- test() {}, >- teardown: "astd" >+ test() { >+ }, >+ teardown: "astd", > }); >- ProtocolTest.fail("should not be able to specify non-Function `teardown` parameter."); >- } catch (e) { >- ProtocolTest.pass("should not be able to specify non-Function `teardown` parameter."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: "foo", >- test() {}, >- teardown: 123 >+ test() { >+ }, >+ teardown: 123, > }); >- ProtocolTest.fail("should not be able to specify non-Function `teardown` parameter."); >- } catch (e) { >- ProtocolTest.pass("should not be able to specify non-Function `teardown` parameter."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: "foo", >- test() {}, >- teardown: {} >+ test() { >+ }, >+ teardown: {}, > }); >- ProtocolTest.fail("should not be able to specify non-Function `teardown` parameter."); >- } catch (e) { >- ProtocolTest.pass("should not be able to specify non-Function `teardown` parameter."); >- } >+ }); > > let runEmptySuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.RunEmptySuite"); > try { >@@ -144,459 +107,765 @@ function test() > ProtocolTest.pass("should not be able to run empty test suite."); > } > >+ let runTwiceSuiteRunCount = 0; > let runTwiceSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.RunTwiceSuite"); > runTwiceSuite.addTestCase({ > name: "DummyTest0", >- description: "Check that a suite can't run more than once.", >- test(resolve, reject) { resolve(); } >+ test(resolve, reject) { >+ ProtocolTest.expectEqual(++runTwiceSuiteRunCount, 1, "DummyTest0 should only run once."); >+ resolve(); >+ }, > }); > > let result = runTwiceSuite.runTestCases(); >- try { >+ ProtocolTest.expectException(() => { > // Test cases won't run in this event loop; this call should still throw. > // Later tests are chained to this suite to avoid nondeterminism. > runTwiceSuite.runTestCases(); >- ProtocolTest.fail("should not be able to run a test suite twice."); >- } catch (e) { >- ProtocolTest.pass("should not be able to run a test suite twice."); >+ }); >+ >+ function checkResult(suite, expectedCounts) { >+ result = result.then(() => suite.runTestCases()); >+ >+ let message = `Promise from ${suite.name}.runTestCases() should resolve even if a test case fails.`; >+ >+ result = result.then(function resolved() { >+ ProtocolTest.log(""); >+ ProtocolTest.pass(message); >+ >+ ProtocolTest.expectEqual(expectedCounts.runCount, suite.runCount, `${suite.name} should have executed ${expectedCounts.runCount} tests.`); >+ ProtocolTest.expectEqual(expectedCounts.passCount, suite.passCount, `${suite.name} should have passed ${expectedCounts.passCount} tests.`); >+ ProtocolTest.expectEqual(expectedCounts.failCount, suite.failCount, `${suite.name} should have failed ${expectedCounts.failCount} tests.`); >+ ProtocolTest.expectEqual(expectedCounts.skipCount, suite.skipCount, `${suite.name} should have skipped ${expectedCounts.skipCount} tests.`); >+ }, function rejected(e) { >+ ProtocolTest.log(""); >+ ProtocolTest.fail(message); >+ >+ ProtocolTest.log(e.message); >+ }); > } > >- let rejectToken = {"token": 666}; >- let thrownError = new Error(rejectToken); >+ // Promise test functions. > >- let sequentialExecutionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.SequentialExecution"); >- sequentialExecutionSuite.addTestCase({ >- name: "DummyTest1", >- description: "Check test case execution order.", >- test(resolve, reject) { resolve(); } >+ let promiseFunctionSuccessSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseFunctionSuccess"); >+ promiseFunctionSuccessSuite.addTestCase({ >+ name: "PromiseFunctionSuccess", >+ test(resolve, reject) { >+ resolve(); >+ }, > }); >- sequentialExecutionSuite.addTestCase({ >- name: "DummyTest2", >- description: "Check test case execution order.", >- test(resolve, reject) { resolve(); } >- }); >- sequentialExecutionSuite.addTestCase({ >- name: "DummyTest3", >- description: "Check test case execution order.", >- test(resolve, reject) { resolve(); } >- }); >- sequentialExecutionSuite.addTestCase({ >- name: "FailingTest4", >- description: "Check that test fails by throwing an Error instance.", >- test(resolve, reject) { throw thrownError; } >+ checkResult(promiseFunctionSuccessSuite, { >+ runCount: 1, >+ passCount: 1, >+ failCount: 0, >+ skipCount: 0, > }); > >- let abortOnFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AbortOnFailure"); >- abortOnFailureSuite.addTestCase({ >- name: "PassingTest5", >- description: "This test is a dummy.", >- test(resolve, reject) { resolve(); } >+ let promiseFunctionExceptionSuite = ProtocolTest.createAsyncSuite("PromiseTestSuite.PromiseFunctionException"); >+ promiseFunctionExceptionSuite.addTestCase({ >+ name: "PromiseFunctionException", >+ test(resolve, reject) { >+ ProtocolTest.log("Throwing..."); >+ throw "PromiseFunctionException throw"; >+ }, > }); >- abortOnFailureSuite.addTestCase({ >- name: "FailingTest6", >- description: "This test should fail by explicitly calling the `reject` callback.", >- test(resolve, reject) { reject(rejectToken); } >- }); >- abortOnFailureSuite.addTestCase({ >- name: "PassingTest7", >- description: "This test should not executed when the preceding test fails.", >- test(resolve, reject) { resolve(); } >+ checkResult(promiseFunctionExceptionSuite, { >+ runCount: 1, >+ passCount: 0, >+ failCount: 1, >+ skipCount: 0, > }); > >+ let promiseFunctionFailureSuite = ProtocolTest.createAsyncSuite("PromiseTestSuite.PromiseFunctionFailure"); >+ promiseFunctionFailureSuite.addTestCase({ >+ name: "PromiseFunctionFailure", >+ test(resolve, reject) { >+ ProtocolTest.log("Rejecting..."); >+ reject("PromiseFunctionFailure reject"); >+ }, >+ }); >+ checkResult(promiseFunctionFailureSuite, { >+ runCount: 1, >+ passCount: 0, >+ failCount: 1, >+ skipCount: 0, >+ }); >+ >+ let promiseSequentialExecutionPhase = 0; >+ let promiseSequentialExecutionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseSequentialExecution"); >+ promiseSequentialExecutionSuite.addTestCase({ >+ name: "1 (Pass)", >+ test(resolve, reject) { >+ ProtocolTest.assert(promiseSequentialExecutionPhase === 0); >+ promiseSequentialExecutionPhase = 1; >+ resolve(); >+ }, >+ }); >+ promiseSequentialExecutionSuite.addTestCase({ >+ name: "2 (Pass)", >+ test(resolve, reject) { >+ ProtocolTest.assert(promiseSequentialExecutionPhase === 1); >+ promiseSequentialExecutionPhase = 2; >+ resolve(); >+ }, >+ }); >+ promiseSequentialExecutionSuite.addTestCase({ >+ name: "3 (Pass)", >+ test(resolve, reject) { >+ ProtocolTest.assert(promiseSequentialExecutionPhase === 2); >+ promiseSequentialExecutionPhase = 3; >+ resolve(); >+ }, >+ }); >+ promiseSequentialExecutionSuite.addTestCase({ >+ name: "4 (Pass)", >+ test(resolve, reject) { >+ ProtocolTest.assert(promiseSequentialExecutionPhase === 3); >+ promiseSequentialExecutionPhase = 4; >+ resolve(); >+ }, >+ }); >+ checkResult(promiseSequentialExecutionSuite, { >+ runCount: 4, >+ passCount: 4, >+ failCount: 0, >+ skipCount: 0, >+ }); > result = result.then(() => { >- let promise = sequentialExecutionSuite.runTestCases(); >- ProtocolTest.expectThat(result instanceof Promise, "AsyncTestSuite.RunTestCases() should return a Promise."); >- return promise; >- }); >- result = result.then(function resolved() { >- ProtocolTest.fail("Promise from sequentialExecutionSuite.runTestCases() should reject when a test case fails."); >- return Promise.resolve(); // Continue this test. >- }, function rejected(e) { >- ProtocolTest.pass("Promise from sequentialExecutionSuite.runTestCases() should reject when a test case fails."); >- ProtocolTest.expectThat(e === thrownError, "Promise from sequentialExecutionSuite.runTestCases() should reject without altering its result value."); >- >- ProtocolTest.expectThat(sequentialExecutionSuite.runCount === 4, "sequentialExecutionSuite should have executed four tests."); >- ProtocolTest.expectThat(sequentialExecutionSuite.passCount === 3, "sequentialExecutionSuite should have passed three tests."); >- ProtocolTest.expectThat(sequentialExecutionSuite.failCount === 1, "sequentialExecutionSuite should have failed 1 test."); >- ProtocolTest.expectThat(sequentialExecutionSuite.skipCount === 0, "sequentialExecutionSuite should have skipped zero tests."); >- return Promise.resolve(); // Continue this test. >+ ProtocolTest.assert(promiseSequentialExecutionPhase === 4); > }); > >- result = result.then(() => { >- return abortOnFailureSuite.runTestCases(); >- }).then(function resolved() { >- ProtocolTest.fail("Promise from abortOnFailureSuite.runTestCases() should reject when a test case fails."); >- return Promise.resolve(); // Continue this test. >- }, function rejected(e) { >- ProtocolTest.pass("Promise from abortOnFailureSuite.runTestCases() should reject when a test case fails."); >- ProtocolTest.expectThat(e === rejectToken, "Promise from abortOnFailureSuite.runTestCases() should reject without altering its result value."); >- ProtocolTest.expectThat(abortOnFailureSuite.runCount === 2, "abortOnFailureSuite should have executed two tests."); >- ProtocolTest.expectThat(abortOnFailureSuite.passCount === 1, "abortOnFailureSuite should have passed one test."); >- ProtocolTest.expectThat(abortOnFailureSuite.failCount === 1, "abortOnFailureSuite should have failed one test."); >- ProtocolTest.expectThat(abortOnFailureSuite.skipCount === 1, "abortOnFailureSuite should have skipped one test."); >- >- return Promise.resolve(); // Continue this test. >+ let promiseContinueOnFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseContinueOnFailure"); >+ promiseContinueOnFailureSuite.addTestCase({ >+ name: "1 (Pass)", >+ test(resolve, reject) { >+ resolve(); >+ }, >+ }); >+ promiseContinueOnFailureSuite.addTestCase({ >+ name: "2 (Fail)", >+ test(resolve, reject) { >+ ProtocolTest.log("Throwing..."); >+ throw {x: "PromiseContinueOnFailure throw"}; >+ }, >+ }); >+ promiseContinueOnFailureSuite.addTestCase({ >+ name: "3 (Pass)", >+ test(resolve, reject) { >+ resolve(); >+ }, >+ }); >+ promiseContinueOnFailureSuite.addTestCase({ >+ name: "4 (Fail)", >+ test(resolve, reject) { >+ ProtocolTest.log("Rejecting..."); >+ reject({x: "PromiseContinueOnFailure reject"}); >+ }, >+ }); >+ checkResult(promiseContinueOnFailureSuite, { >+ runCount: 4, >+ passCount: 2, >+ failCount: 2, >+ skipCount: 0, > }); > >- var setupAndTeardownSymbol = Symbol("async-suite-setup-and-teardown-token"); >- window[setupAndTeardownSymbol] = 0; >- >- let setupAndTeardownTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.SetupAndTeardown"); >- setupAndTeardownTestSuite.addTestCase({ >+ let promiseSetupAndTeardownPhase = 0; >+ let promiseSetupAndTeardownSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseSetupAndTeardown"); >+ promiseSetupAndTeardownSuite.addTestCase({ > name: "TestWithSetupAndTeardown", >- description: "Check execution order for setup and teardown actions.", >- setup: (resolve, reject) => { >- window[setupAndTeardownSymbol] = 1; >+ setup(resolve, reject) { >+ ProtocolTest.assert(promiseSetupAndTeardownPhase === 0); >+ promiseSetupAndTeardownPhase = 1; > resolve(); > }, > test(resolve, reject) { >- ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 1, "Test should see side effects of running setup() action."); >- window[setupAndTeardownSymbol] = 2; >+ ProtocolTest.assert(promiseSetupAndTeardownPhase === 1); >+ promiseSetupAndTeardownPhase = 2; > resolve(); > }, >- teardown: (resolve, reject) => { >- ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 2, "Teardown should see side effects of running setup() action."); >- window[setupAndTeardownSymbol] = 3; >+ teardown(resolve, reject) { >+ ProtocolTest.assert(promiseSetupAndTeardownPhase === 2); >+ promiseSetupAndTeardownPhase = 3; > resolve(); >- } >+ }, > }); >- setupAndTeardownTestSuite.addTestCase({ >+ promiseSetupAndTeardownSuite.addTestCase({ > name: "TestRunningAfterTeardown", >- description: "Check execution order for test after a teardown action.", > test(resolve, reject) { >- ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 3, "Test should see side effects of previous test's teardown() action."); >+ ProtocolTest.assert(promiseSetupAndTeardownPhase === 3); > resolve(); > }, > }); >- >- result = result.then(() => { >- return setupAndTeardownTestSuite.runTestCases(); >- }).then(function resolved() { >- ProtocolTest.pass("Promise from setupAndTeardownTestSuite.runTestCases() should resolve."); >- return Promise.resolve(); // Continue this test. >- }, function rejected(e) { >- ProtocolTest.fail("Promise from setupAndTeardownTestSuite.runTestCases() should resolve."); >- return Promise.resolve(); // Continue this test. >+ checkResult(promiseSetupAndTeardownSuite, { >+ runCount: 2, >+ passCount: 2, >+ failCount: 0, >+ skipCount: 0, > }); > >- let setupExceptionTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.SetupException"); >- setupExceptionTestSuite.addTestCase({ >+ let promiseSetupExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseSetupException"); >+ promiseSetupExceptionSuite.addTestCase({ > name: "TestWithExceptionDuringSetup", >- description: "Check execution order for setup action that throws an exception.", >- setup: (resolve, reject) => { throw new Error() }, >+ setup(resolve, reject) { >+ ProtocolTest.log("Throwing..."); >+ throw "PromiseSetupException throw"; >+ }, > test(resolve, reject) { >- ProtocolTest.assert(false, "Test should not execute if its setup action threw an exception."); >+ ProtocolTest.fail("Test should not execute if its setup action threw an exception."); > reject(); > }, >- teardown: (resolve, reject) => { >- ProtocolTest.assert(false, "Teardown action should not execute if its setup action threw an exception."); >+ teardown(resolve, reject) { >+ ProtocolTest.fail("Teardown action should not execute if its setup action threw an exception."); > reject(); >- } >+ }, >+ }); >+ promiseSetupExceptionSuite.addTestCase({ >+ name: "TestAfterSetupException", >+ setup(resolve, reject) { >+ ProtocolTest.pass("Setup action should still execute if previous test's setup action threw an exception."); >+ resolve(); >+ }, >+ test(resolve, reject) { >+ ProtocolTest.pass("Test should still execute if previous test's setup action threw an exception."); >+ resolve(); >+ }, >+ teardown(resolve, reject) { >+ ProtocolTest.pass("Teardown action should still execute if previous test's setup action threw an exception."); >+ resolve(); >+ }, >+ }); >+ checkResult(promiseSetupExceptionSuite, { >+ runCount: 1, >+ passCount: 1, >+ failCount: 1, >+ skipCount: 1, > }); > >- result = result.then(() => { >- return setupExceptionTestSuite.runTestCases(); >- }).then(function resolved() { >- ProtocolTest.fail("Promise from setupExceptionTestSuite.runTestCases() should reject."); >- return Promise.resolve(); // Continue this test. >- }, function rejected(e) { >- ProtocolTest.pass("Promise from setupExceptionTestSuite.runTestCases() should reject."); >- return Promise.resolve(); // Continue this test. >- }); >- >- let setupFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.SetupFailure"); >- setupFailureTestSuite.addTestCase({ >+ let promiseSetupFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseSetupFailure"); >+ promiseSetupFailureSuite.addTestCase({ > name: "TestWithFailureDuringSetup", >- description: "Check execution order for setup action that has failed.", >- setup: (resolve, reject) => { reject(); }, >+ setup(resolve, reject) { >+ ProtocolTest.log("Rejecting..."); >+ reject("PromiseSetupFailure reject"); >+ }, > test(resolve, reject) { >- ProtocolTest.assert(false, "Test should not execute if its setup action failed.") >+ ProtocolTest.fail("Test should not execute if its setup action failed."); > reject(); > }, >- teardown: (resolve, reject) => { >- ProtocolTest.assert(false, "Teardown action should not execute if its setup action failed.") >+ teardown(resolve, reject) { >+ ProtocolTest.fail("Teardown action should not execute if its setup action failed."); > reject(); >- } >+ }, >+ }); >+ promiseSetupFailureSuite.addTestCase({ >+ name: "TestAfterSetupException", >+ setup(resolve, reject) { >+ ProtocolTest.pass("Setup action should still execute if previous test's setup action failed."); >+ resolve(); >+ }, >+ test(resolve, reject) { >+ ProtocolTest.pass("Test should still execute if previous test's setup action failed."); >+ resolve(); >+ }, >+ teardown(resolve, reject) { >+ ProtocolTest.pass("Teardown action should still execute if previous test's setup action failed."); >+ resolve(); >+ }, >+ }); >+ checkResult(promiseSetupFailureSuite, { >+ runCount: 1, >+ passCount: 1, >+ failCount: 1, >+ skipCount: 1, > }); > >- result = result.then(() => { >- return setupFailureTestSuite.runTestCases(); >- }).then(function resolved() { >- ProtocolTest.fail("Promise from setupFailureTestSuite.runTestCases() should reject."); >- return Promise.resolve(); // Continue this test. >- }, function rejected(e) { >- ProtocolTest.pass("Promise from setupFailureTestSuite.runTestCases() should reject."); >- return Promise.resolve(); // Continue this test. >- }); >- >- let teardownExceptionTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.TeardownException"); >- teardownExceptionTestSuite.addTestCase({ >+ let promiseTeardownExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseTeardownException"); >+ promiseTeardownExceptionSuite.addTestCase({ > name: "TestWithExceptionDuringTeardown", >- description: "Check execution order for teardown action that throws an exception.", >- test(resolve, reject) { resolve(); }, >- teardown: (resolve, reject) => { throw new Error() } >+ test(resolve, reject) { >+ resolve(); >+ }, >+ teardown(resolve, reject) { >+ ProtocolTest.log("Throwing..."); >+ throw "PromiseTeardownException throw"; >+ }, > }); >- teardownExceptionTestSuite.addTestCase({ >+ promiseTeardownExceptionSuite.addTestCase({ > name: "TestAfterTeardownException", >- descrption: "Check execution order for test after previous test's teardown throws an exception.", >- setup: (resolve, reject) => { >- ProtocolTest.assert(false, "Setup action should not execute if previous test's teardown action threw an exception."); >- reject(); >+ setup(resolve, reject) { >+ ProtocolTest.pass("Setup action should still execute if previous test's teardown action threw an exception."); >+ resolve(); > }, > test(resolve, reject) { >- ProtocolTest.assert(false, "Test should not execute if previous test's teardown action threw an exception."); >- reject(); >- } >+ ProtocolTest.pass("Test should still execute if previous test's teardown action threw an exception."); >+ resolve(); >+ }, >+ teardown(resolve, reject) { >+ ProtocolTest.pass("Teardown action should still execute if previous test's teardown action threw an exception."); >+ resolve(); >+ }, >+ }); >+ checkResult(promiseTeardownExceptionSuite, { >+ runCount: 2, >+ passCount: 1, >+ failCount: 1, >+ skipCount: 0, > }); > >- result = result.then(() => { >- return teardownExceptionTestSuite.runTestCases(); >- }).then(function resolved() { >- ProtocolTest.fail("Promise from teardownExceptionTestSuite.runTestCases() should reject."); >- return Promise.resolve(); // Continue this test. >- }, function rejected(e) { >- ProtocolTest.pass("Promise from teardownExceptionTestSuite.runTestCases() should reject."); >- return Promise.resolve(); // Continue this test. >- }); >- >- let teardownFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.TeardownFailure"); >- teardownFailureTestSuite.addTestCase({ >+ let promiseTeardownFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseTeardownFailure"); >+ promiseTeardownFailureSuite.addTestCase({ > name: "TestWithExceptionDuringTeardown", >- description: "Check execution order for teardown action that has failed.", >- test(resolve, reject) { resolve(); }, >- teardown: (resolve, reject) => { reject(); }, >+ test(resolve, reject) { >+ resolve(); >+ }, >+ teardown(resolve, reject) { >+ ProtocolTest.log("Rejecting..."); >+ reject("PromiseTeardownFailure reject"); >+ }, > }); >- teardownFailureTestSuite.addTestCase({ >+ promiseTeardownFailureSuite.addTestCase({ > name: "TestAfterTeardownException", >- description: "Check execution order for test after previous test's teardown throws an exception", >- setup: (resolve, reject) => { >- ProtocolTest.assert(false, "Setup action should not execute if previous test's teardown action failed."); >- reject(); >+ setup(resolve, reject) { >+ ProtocolTest.pass("Setup action should still execute if previous test's teardown action failed."); >+ resolve(); > }, > test(resolve, reject) { >- ProtocolTest.assert(false, "Test should not execute if previous test's teardown action failed."); >- reject(); >- } >+ ProtocolTest.pass("Test should still execute if previous test's teardown action failed."); >+ resolve(); >+ }, >+ teardown(resolve, reject) { >+ ProtocolTest.pass("Teardown action should still execute if previous test's teardown action failed."); >+ resolve(); >+ }, >+ }); >+ checkResult(promiseTeardownFailureSuite, { >+ runCount: 2, >+ passCount: 1, >+ failCount: 1, >+ skipCount: 0, > }); > >- result = result.then(() => { >- return teardownFailureTestSuite.runTestCases(); >- }).then(function resolved() { >- ProtocolTest.fail("Promise from teardownFailureTestSuite.runTestCases() should reject."); >- return Promise.resolve(); // Continue this test. >- }, function rejected(e) { >- ProtocolTest.pass("Promise from teardownFailureTestSuite.runTestCases() should reject."); >- return Promise.resolve(); // Continue this test. >+ let promiseTimeoutSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseTimeout"); >+ promiseTimeoutSuite.addTestCase({ >+ name: "PromiseTestWithTimeout", >+ test(resolve, reject) { >+ ProtocolTest.log("Timeout..."); >+ setTimeout(() => { >+ resolve(); >+ }, 50); >+ }, >+ timeout: 5, >+ }); >+ promiseTimeoutSuite.addTestCase({ >+ name: "PromiseTestAfterTimeout", >+ setup(resolve, reject) { >+ ProtocolTest.pass("Setup action should still execute if previous test timed out."); >+ resolve(); >+ }, >+ test(resolve, reject) { >+ ProtocolTest.pass("Test should still execute if previous test timed out."); >+ resolve(); >+ }, >+ teardown(resolve, reject) { >+ ProtocolTest.pass("Teardown action should still execute if previous test timed out."); >+ resolve(); >+ }, >+ }); >+ checkResult(promiseTimeoutSuite, { >+ runCount: 2, >+ passCount: 1, >+ failCount: 1, >+ skipCount: 0, > }); > > // Async test functions. >- let asyncFunctionSuccessTestSuiteDidEvaluate = false; >- let asyncFunctionSuccessTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionSuccess"); >- asyncFunctionSuccessTestSuite.addTestCase({ >+ >+ let asyncFunctionSuccessSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionSuccess"); >+ asyncFunctionSuccessSuite.addTestCase({ > name: "AsyncFunctionSuccess", >- description: "Check that an async suite with async test functions can succeed", > async test() { >- asyncFunctionSuccessTestSuiteDidEvaluate = true; >- return 42; >- } >+ }, >+ }); >+ checkResult(asyncFunctionSuccessSuite, { >+ runCount: 1, >+ passCount: 1, >+ failCount: 0, >+ skipCount: 0, > }); > >- result = result.then(() => { >- return asyncFunctionSuccessTestSuite.runTestCases(); >- }).then(function resolve(x) { >- ProtocolTest.pass("Promise from asyncFunctionSuccessTestSuite.runTestCases() should succeed."); >- ProtocolTest.expectThat(asyncFunctionSuccessTestSuiteDidEvaluate, "Promise did evaluate the async test function."); >- ProtocolTest.expectEqual(x, 42, "Resolved value should be 42."); >- return Promise.resolve(); // Continue this test. >- }, function rejected(e) { >- ProtocolTest.fail("Promise from asyncFunctionSuccessTestSuite.runTestCases() should succeed."); >- return Promise.resolve(); // Continue this test. >- }); >- >- let asyncFunctionExplicitFailureTestSuiteDidEvaluate = false; >- let asyncFunctionExplicitFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionExplicitFailure"); >- asyncFunctionExplicitFailureTestSuite.addTestCase({ >- name: "AsyncFunctionFailure", >- description: "Check that an async suite with async test functions that throws will reject", >+ let asyncFunctionExplicitExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionExplicitException"); >+ asyncFunctionExplicitExceptionSuite.addTestCase({ >+ name: "AsyncFunctionExplicitException", > async test() { >- asyncFunctionExplicitFailureTestSuiteDidEvaluate = true; >- throw "AsyncFunctionFailure Exception Message"; >- } >+ ProtocolTest.log("Throwing..."); >+ throw "AsyncFunctionExplicitException throw"; >+ }, >+ }); >+ checkResult(asyncFunctionExplicitExceptionSuite, { >+ runCount: 1, >+ passCount: 0, >+ failCount: 1, >+ skipCount: 0, > }); > >- result = result.then(() => { >- return asyncFunctionExplicitFailureTestSuite.runTestCases(); >- }).then(function resolve() { >- ProtocolTest.fail("Promise from asyncFunctionExplicitFailureTestSuite.runTestCases() should reject."); >- return Promise.resolve(); // Continue this test. >- }, function rejected(e) { >- ProtocolTest.pass("Promise from asyncFunctionExplicitFailureTestSuite.runTestCases() should reject."); >- ProtocolTest.expectThat(asyncFunctionExplicitFailureTestSuiteDidEvaluate, "Promise did evaluate the async test function."); >- ProtocolTest.expectEqual(e, "AsyncFunctionFailure Exception Message", "Rejected value should be thrown exception."); >- return Promise.resolve(); // Continue this test. >- }); >- >- let asyncFunctionRuntimeFailureTestSuiteDidEvaluate = false; >- let asyncFunctionRuntimeFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionRuntimeFailure"); >- asyncFunctionRuntimeFailureTestSuite.addTestCase({ >- name: "AsyncFunctionFailure", >- description: "Check that an async suite with async test functions that throws with a runtime error will reject", >+ let asyncFunctionRuntimeExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionRuntimeException"); >+ asyncFunctionRuntimeExceptionSuite.addTestCase({ >+ name: "AsyncFunctionRuntimeException", > async test() { >- asyncFunctionRuntimeFailureTestSuiteDidEvaluate = true; >+ ProtocolTest.log("Throwing..."); > ({}).x.x.x; >- } >+ }, >+ }); >+ checkResult(asyncFunctionRuntimeExceptionSuite, { >+ runCount: 1, >+ passCount: 0, >+ failCount: 1, >+ skipCount: 0, > }); > >+ let asyncFunctionFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionFailure"); >+ asyncFunctionFailureSuite.addTestCase({ >+ name: "AsyncFunctionException", >+ async test() { >+ ProtocolTest.log("Rejecting..."); >+ return Promise.reject("AsyncFunctionFailure reject"); >+ }, >+ }); >+ checkResult(asyncFunctionFailureSuite, { >+ runCount: 1, >+ passCount: 0, >+ failCount: 1, >+ skipCount: 0, >+ }); >+ >+ let asyncSequentialExecutionPhase = 0; >+ let asyncSequentialExecutionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSequentialExecution"); >+ asyncSequentialExecutionSuite.addTestCase({ >+ name: "1 (Pass)", >+ async test() { >+ ProtocolTest.assert(asyncSequentialExecutionPhase === 0); >+ asyncSequentialExecutionPhase = 1; >+ }, >+ }); >+ asyncSequentialExecutionSuite.addTestCase({ >+ name: "2 (Pass)", >+ async test() { >+ ProtocolTest.assert(asyncSequentialExecutionPhase === 1); >+ asyncSequentialExecutionPhase = 2; >+ }, >+ }); >+ asyncSequentialExecutionSuite.addTestCase({ >+ name: "3 (Pass)", >+ async test() { >+ ProtocolTest.assert(asyncSequentialExecutionPhase === 2); >+ asyncSequentialExecutionPhase = 3; >+ }, >+ }); >+ asyncSequentialExecutionSuite.addTestCase({ >+ name: "4 (Pass)", >+ async test() { >+ ProtocolTest.assert(asyncSequentialExecutionPhase === 3); >+ asyncSequentialExecutionPhase = 4; >+ }, >+ }); >+ checkResult(asyncSequentialExecutionSuite, { >+ runCount: 4, >+ passCount: 4, >+ failCount: 0, >+ skipCount: 0, >+ }); > result = result.then(() => { >- return asyncFunctionRuntimeFailureTestSuite.runTestCases(); >- }).then(function resolve() { >- ProtocolTest.fail("Promise from asyncFunctionRuntimeFailureTestSuite.runTestCases() should reject."); >- return Promise.resolve(); // Continue this test. >- }, function rejected(e) { >- ProtocolTest.pass("Promise from asyncFunctionRuntimeFailureTestSuite.runTestCases() should reject."); >- ProtocolTest.expectThat(asyncFunctionRuntimeFailureTestSuiteDidEvaluate, "Promise did evaluate the async test function."); >- ProtocolTest.expectThat(e instanceof TypeError, "Rejected value should be a runtime exception."); >- return Promise.resolve(); // Continue this test. >+ ProtocolTest.assert(asyncSequentialExecutionPhase === 4); > }); > >- // Async setup() and teardown() success test cases. >- const asyncSetupAndTeardownSymbol = Symbol("async-suite-async-setup-and-teardown-token"); >- window[asyncSetupAndTeardownSymbol] = 0; >+ let asyncContinueOnFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncContinueOnFailure"); >+ asyncContinueOnFailureSuite.addTestCase({ >+ name: "1 (Pass)", >+ async test() { >+ }, >+ }); >+ asyncContinueOnFailureSuite.addTestCase({ >+ name: "2 (Fail)", >+ async test() { >+ ProtocolTest.log("Throwing..."); >+ throw {x: "AsyncContinueOnFailure throw"}; >+ }, >+ }); >+ asyncContinueOnFailureSuite.addTestCase({ >+ name: "3 (Pass)", >+ async test() { >+ }, >+ }); >+ asyncContinueOnFailureSuite.addTestCase({ >+ name: "4 (Fail)", >+ async test() { >+ ProtocolTest.log("Throwing..."); >+ ({}).x.x.x; >+ }, >+ }); >+ asyncContinueOnFailureSuite.addTestCase({ >+ name: "5 (Pass)", >+ async test() { >+ }, >+ }); >+ asyncContinueOnFailureSuite.addTestCase({ >+ name: "6 (Fail)", >+ async test() { >+ ProtocolTest.log("Rejecting..."); >+ return Promise.reject("AsyncContinueOnFailure reject"); >+ }, >+ }); >+ checkResult(asyncContinueOnFailureSuite, { >+ runCount: 6, >+ passCount: 3, >+ failCount: 3, >+ skipCount: 0, >+ }); > >- let asyncSetupAndAsyncTeardownTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupAndAsyncTeardown"); >- asyncSetupAndAsyncTeardownTestSuite.addTestCase({ >- name: "TestWithSetupAndTeardown", >- description: "Check execution order for setup and teardown actions.", >+ let asyncSetupAndTeardownPhase = 0; >+ let asyncSetupAndTeardownSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupAndTeardown"); >+ asyncSetupAndTeardownSuite.addTestCase({ >+ name: "AsyncTestWithSetupAndTeardown", > async setup() { >- window[asyncSetupAndTeardownSymbol] = 1; >+ ProtocolTest.assert(asyncSetupAndTeardownPhase === 0); >+ asyncSetupAndTeardownPhase = 1; > }, > async test() { >- ProtocolTest.expectThat(window[asyncSetupAndTeardownSymbol] === 1, "Test should see side effects of running setup() action."); >- window[asyncSetupAndTeardownSymbol] = 2; >+ ProtocolTest.assert(asyncSetupAndTeardownPhase === 1); >+ asyncSetupAndTeardownPhase = 2; > }, > async teardown() { >- ProtocolTest.expectThat(window[asyncSetupAndTeardownSymbol] === 2, "Teardown should see side effects of running setup() action."); >- window[asyncSetupAndTeardownSymbol] = 3; >- } >- }); >- asyncSetupAndAsyncTeardownTestSuite.addTestCase({ >- name: "TestRunningAfterTeardown", >- description: "Check execution order for test after a teardown action.", >- test(resolve, reject) { >- ProtocolTest.expectThat(window[asyncSetupAndTeardownSymbol] === 3, "Test should see side effects of previous test's teardown() action."); >- resolve(); >+ ProtocolTest.assert(asyncSetupAndTeardownPhase === 2); >+ asyncSetupAndTeardownPhase = 3; > }, > }); >- >- result = result.then(() => { >- return asyncSetupAndAsyncTeardownTestSuite.runTestCases(); >- }).then(function resolved() { >- ProtocolTest.pass("Promise from asyncSetupAndAsyncTeardownTestSuite.runTestCases() should resolve."); >- return Promise.resolve(); // Continue this test. >- }, function rejected(e) { >- ProtocolTest.fail("Promise from asyncSetupAndAsyncTeardownTestSuite.runTestCases() should resolve."); >- return Promise.resolve(); // Continue this test. >- }); >- >- // Async setup() failure test cases. >- let asyncSetupExplicitFailureTestSuiteDidEvaluate = false; >- let asyncSetupExplicitFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupExplicitFailure"); >- asyncSetupExplicitFailureTestSuite.addTestCase({ >- name: "AsyncFunctionFailure", >- description: "Check that an async suite with async test functions that throws will reject", >+ asyncSetupAndTeardownSuite.addTestCase({ >+ name: "AsyncTestRunningAfterTeardown", > async test() { >- asyncSetupExplicitFailureTestSuiteDidEvaluate = true; >- throw "AsyncFunctionFailure Exception Message"; >- } >+ ProtocolTest.assert(asyncSetupAndTeardownPhase === 3); >+ }, >+ }); >+ checkResult(asyncSetupAndTeardownSuite, { >+ runCount: 2, >+ passCount: 2, >+ failCount: 0, >+ skipCount: 0, > }); > >- result = result.then(() => { >- return asyncSetupExplicitFailureTestSuite.runTestCases(); >- }).then(function resolve() { >- ProtocolTest.fail("Promise from asyncSetupExplicitFailureTestSuite.runTestCases() should reject."); >- return Promise.resolve(); // Continue this test. >- }, function rejected(e) { >- ProtocolTest.pass("Promise from asyncSetupExplicitFailureTestSuite.runTestCases() should reject."); >- ProtocolTest.expectThat(asyncSetupExplicitFailureTestSuiteDidEvaluate, "Promise did evaluate the async setup function."); >- ProtocolTest.expectEqual(e, "AsyncFunctionFailure Exception Message", "Rejected value should be thrown exception."); >- return Promise.resolve(); // Continue this test. >- }); >- >- let asyncSetupRuntimeFailureTestSuiteDidEvaluate = false; >- let asyncSetupRuntimeFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupRuntimeFailure"); >- asyncSetupRuntimeFailureTestSuite.addTestCase({ >- name: "AsyncFunctionFailure", >- description: "Check that an async suite with an async setup function that throws with a runtime error will reject", >+ let asyncSetupExplicitExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupExplicitException"); >+ asyncSetupExplicitExceptionSuite.addTestCase({ >+ name: "AsyncTestWithExplicitExceptionDuringSetup", > async setup() { >- asyncSetupRuntimeFailureTestSuiteDidEvaluate = true; >+ ProtocolTest.log("Throwing..."); >+ throw "AsyncSetupExplicitException throw"; >+ }, >+ async test() { >+ ProtocolTest.fail("Test should not execute if its setup action threw an exception."); >+ }, >+ async teardown() { >+ ProtocolTest.fail("Teardown action should not execute if its setup action threw an exception."); >+ }, >+ }); >+ asyncSetupExplicitExceptionSuite.addTestCase({ >+ name: "AsyncTestAfterSetupExplicitException", >+ async setup() { >+ ProtocolTest.pass("Setup action should still execute if previous test's setup action threw an exception."); >+ }, >+ async test() { >+ ProtocolTest.pass("Test should still execute if previous test's setup action threw an exception."); >+ }, >+ async teardown() { >+ ProtocolTest.pass("Teardown action should still execute if previous test's setup action threw an exception."); >+ }, >+ }); >+ checkResult(asyncSetupExplicitExceptionSuite, { >+ runCount: 1, >+ passCount: 1, >+ failCount: 1, >+ skipCount: 1, >+ }); >+ >+ let asyncSetupRuntimeExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupRuntimeException"); >+ asyncSetupRuntimeExceptionSuite.addTestCase({ >+ name: "AsyncTestWithRuntimeExceptionDuringSetup", >+ async setup() { >+ ProtocolTest.log("Throwing..."); > ({}).x.x.x; > }, >- async test() { return true; }, >- }); >- >- result = result.then(() => { >- return asyncSetupRuntimeFailureTestSuite.runTestCases(); >- }).then(function resolve() { >- ProtocolTest.fail("Promise from asyncSetupRuntimeFailureTestSuite.runTestCases() should reject."); >- return Promise.resolve(); // Continue this test. >- }, function rejected(e) { >- ProtocolTest.pass("Promise from asyncSetupRuntimeFailureTestSuite.runTestCases() should reject."); >- ProtocolTest.expectThat(asyncSetupRuntimeFailureTestSuiteDidEvaluate, "Promise did evaluate the async setup function."); >- ProtocolTest.expectThat(e instanceof TypeError, "Rejected value should be a runtime exception."); >- return Promise.resolve(); // Continue this test. >- }); >- >- // Async teardown() failure test cases. >- let asyncTeardownExplicitFailureTestSuiteDidEvaluate = false; >- let asyncTeardownExplicitFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTeardownExplicitFailure"); >- asyncTeardownExplicitFailureTestSuite.addTestCase({ >- name: "AsyncFunctionFailure", >- description: "Check that an async suite with async test functions that throws will reject", >- async test() { return true; }, >+ async test() { >+ ProtocolTest.fail("Test should not execute if its setup action threw an exception."); >+ }, > async teardown() { >- asyncTeardownExplicitFailureTestSuiteDidEvaluate = true; >- throw "AsyncFunctionFailure Exception Message"; >+ ProtocolTest.fail("Teardown action should not execute if its setup action threw an exception."); > }, > }); >- >- result = result.then(() => { >- return asyncTeardownExplicitFailureTestSuite.runTestCases(); >- }).then(function resolve() { >- ProtocolTest.fail("Promise from asyncTeardownExplicitFailureTestSuite.runTestCases() should reject."); >- return Promise.resolve(); // Continue this test. >- }, function rejected(e) { >- ProtocolTest.pass("Promise from asyncTeardownExplicitFailureTestSuite.runTestCases() should reject."); >- ProtocolTest.expectThat(asyncTeardownExplicitFailureTestSuiteDidEvaluate, "Promise did evaluate the async teardown function."); >- ProtocolTest.expectEqual(e, "AsyncFunctionFailure Exception Message", "Rejected value should be thrown exception."); >- return Promise.resolve(); // Continue this test. >+ asyncSetupRuntimeExceptionSuite.addTestCase({ >+ name: "AsyncTestAfterSetupRuntimeException", >+ async setup() { >+ ProtocolTest.pass("Setup action should still execute if previous test's setup action threw an exception."); >+ }, >+ async test() { >+ ProtocolTest.pass("Test should still execute if previous test's setup action threw an exception."); >+ }, >+ async teardown() { >+ ProtocolTest.pass("Teardown action should still execute if previous test's setup action threw an exception."); >+ }, >+ }); >+ checkResult(asyncSetupRuntimeExceptionSuite, { >+ runCount: 1, >+ passCount: 1, >+ failCount: 1, >+ skipCount: 1, > }); > >- let asyncTeardownRuntimeFailureTestSuiteDidEvaluate = false; >- let asyncTeardownRuntimeFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTeardownRuntimeFailure"); >- asyncTeardownRuntimeFailureTestSuite.addTestCase({ >- name: "AsyncFunctionFailure", >- description: "Check that an async suite with an async teardown function that throws with a runtime error will reject", >- async test() { return true; }, >+ let asyncSetupFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupFailure"); >+ asyncSetupFailureSuite.addTestCase({ >+ name: "AsyncTestWithFailureDuringSetup", >+ async setup() { >+ ProtocolTest.log("Rejecting..."); >+ return Promise.reject("AsyncSetupFailure reject"); >+ }, >+ async test() { >+ }, >+ }); >+ asyncSetupFailureSuite.addTestCase({ >+ name: "AsyncTestAfterSetupFailure", >+ async setup() { >+ ProtocolTest.pass("Setup action should still execute if previous test's setup action failed."); >+ }, >+ async test() { >+ ProtocolTest.pass("Test should still execute if previous test's setup action failed."); >+ }, > async teardown() { >- asyncTeardownRuntimeFailureTestSuiteDidEvaluate = true; >+ ProtocolTest.pass("Setup action should still execute if previous test's setup action failed."); >+ }, >+ }); >+ checkResult(asyncSetupFailureSuite, { >+ runCount: 1, >+ passCount: 1, >+ failCount: 1, >+ skipCount: 1, >+ }); >+ >+ let asyncTeardownExplicitExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTeardownExplicitException"); >+ asyncTeardownExplicitExceptionSuite.addTestCase({ >+ name: "AsyncTestWithExplicitExceptionDuringTeardown", >+ async test() { >+ }, >+ async teardown() { >+ ProtocolTest.log("Throwing..."); >+ throw "AsyncTeardownExplicitException throw"; >+ }, >+ }); >+ asyncTeardownExplicitExceptionSuite.addTestCase({ >+ name: "AsyncTestAfterTeardownExplicitException", >+ async setup() { >+ ProtocolTest.pass("Setup action should still execute if previous test's teardown action threw an exception."); >+ }, >+ async test() { >+ ProtocolTest.pass("Test should still execute if previous test's teardown action threw an exception."); >+ }, >+ async teardown() { >+ ProtocolTest.pass("Teardown action should still execute if previous test's teardown action threw an exception."); >+ }, >+ }); >+ checkResult(asyncTeardownExplicitExceptionSuite, { >+ runCount: 2, >+ passCount: 1, >+ failCount: 1, >+ skipCount: 0, >+ }); >+ >+ let asyncTeardownRuntimeExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTeardownRuntimeException"); >+ asyncTeardownRuntimeExceptionSuite.addTestCase({ >+ name: "AsyncTestWithRuntimeExceptionDuringTeardown", >+ async test() { >+ }, >+ async teardown() { >+ ProtocolTest.log("Throwing..."); > ({}).x.x.x; > }, > }); >+ asyncTeardownRuntimeExceptionSuite.addTestCase({ >+ name: "AsyncTestAfterTeardownRuntimeException", >+ async setup() { >+ ProtocolTest.pass("Setup action should still execute if previous test's teardown action threw an exception."); >+ }, >+ async test() { >+ ProtocolTest.pass("Test should still execute if previous test's teardown action threw an exception."); >+ }, >+ async teardown() { >+ ProtocolTest.pass("Teardown action should still execute if previous test's teardown action threw an exception."); >+ }, >+ }); >+ checkResult(asyncTeardownRuntimeExceptionSuite, { >+ runCount: 2, >+ passCount: 1, >+ failCount: 1, >+ skipCount: 0, >+ }); > >- result = result.then(() => { >- return asyncTeardownRuntimeFailureTestSuite.runTestCases(); >- }).then(function resolve() { >- ProtocolTest.fail("Promise from asyncTeardownRuntimeFailureTestSuite.runTestCases() should reject."); >- return Promise.resolve(); // Continue this test. >- }, function rejected(e) { >- ProtocolTest.pass("Promise from asyncTeardownRuntimeFailureTestSuite.runTestCases() should reject."); >- ProtocolTest.expectThat(asyncTeardownRuntimeFailureTestSuiteDidEvaluate, "Promise did evaluate the async teardown function."); >- ProtocolTest.expectThat(e instanceof TypeError, "Rejected value should be a runtime exception."); >- return Promise.resolve(); // Continue this test. >+ let asyncTeardownFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTeardownFailure"); >+ asyncTeardownFailureSuite.addTestCase({ >+ name: "AsyncTestWithFailureDuringTeardown", >+ async test() { >+ }, >+ async teardown() { >+ ProtocolTest.log("Rejecting..."); >+ return Promise.reject("AsyncTeardownFailure reject"); >+ }, >+ }); >+ asyncTeardownFailureSuite.addTestCase({ >+ name: "AsyncTestAfterTeardownFailure", >+ async setup() { >+ ProtocolTest.pass("Setup action should still execute if previous test's teardown action failed."); >+ }, >+ async test() { >+ ProtocolTest.pass("Test should still execute if previous test's teardown action failed."); >+ }, >+ async teardown() { >+ ProtocolTest.pass("Teardown action should still execute if previous test's teardown action failed."); >+ }, >+ }); >+ checkResult(asyncTeardownFailureSuite, { >+ runCount: 2, >+ passCount: 1, >+ failCount: 1, >+ skipCount: 0, >+ }); >+ >+ let asyncTimeoutSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTimeout"); >+ asyncTimeoutSuite.addTestCase({ >+ name: "AsyncTestWithTimeout", >+ async test() { >+ ProtocolTest.log("Timeout..."); >+ await Promise.delay(50); >+ }, >+ timeout: 5, >+ }); >+ asyncTimeoutSuite.addTestCase({ >+ name: "AsyncTestAfterTimeout", >+ async setup() { >+ ProtocolTest.pass("Setup action should still execute if previous test timed out."); >+ }, >+ async test() { >+ ProtocolTest.pass("Test should still execute if previous test timed out."); >+ }, >+ async teardown() { >+ ProtocolTest.pass("Teardown action should still execute if previous test timed out."); >+ }, >+ }); >+ checkResult(asyncTimeoutSuite, { >+ runCount: 2, >+ passCount: 1, >+ failCount: 1, >+ skipCount: 0, > }); > > // This will finish the test whether the chain was resolved or rejected. >- result = result.then(() => { ProtocolTest.completeTest(); }); >+ result = result.then(() => { >+ ProtocolTest.completeTest(); >+ }); > } > </script> > </head> >diff --git a/LayoutTests/inspector/unit-tests/sync-test-suite-expected.txt b/LayoutTests/inspector/unit-tests/sync-test-suite-expected.txt >index c27198bba51c77050bf3c918ea401306b4a8681c..fcb758aafc8c23bdbff910eba8867f4c6fc0b5cb 100644 >--- a/LayoutTests/inspector/unit-tests/sync-test-suite-expected.txt >+++ b/LayoutTests/inspector/unit-tests/sync-test-suite-expected.txt >@@ -1,77 +1,156 @@ >-PASS: instantiating SyncTestSuite requires name argument. >-PASS: instantiating SyncTestSuite requires string name argument. >-PASS: instantiating SyncTestSuite requires non-whitespace name argument. >-PASS: instantiating SyncTestSuite requires test harness argument. >-PASS: should not be able to add empty test case. >-PASS: should not be able to add non-object test case. >-PASS: test case should require string name. >-PASS: test case should require non-whitespace name. >-PASS: test case should require test function. >-PASS: should not be able to specify non-Function `setup` parameter. >-PASS: should not be able to specify non-Function `setup` parameter. >-PASS: should not be able to specify non-Function `setup` parameter. >-PASS: should not be able to specify non-Function `teardown` parameter. >-PASS: should not be able to specify non-Function `teardown` parameter. >-PASS: should not be able to specify non-Function `teardown` parameter. >-PASS: should not be able to specify async `test` parameter. >-PASS: should not be able to specify async `setup` parameter. >-PASS: should not be able to specify async `teardown` parameter. >-PASS: should not be able to run empty test suite. >+PASS: Should produce an exception. >+Error: Must pass the test's harness as the first argument. >+PASS: Should produce an exception. >+Error: Must pass the test's harness as the first argument. >+PASS: Should produce an exception. >+Error: Must pass the test's harness as the first argument. >+PASS: Should produce an exception. >+Error: Must pass the test's harness as the first argument. >+PASS: Should produce an exception. >+TypeError: undefined is not an object (evaluating 'testcase.setup') >+PASS: Should produce an exception. >+Error: Tried to add non-object test case. >+PASS: Should produce an exception. >+Error: Tried to add test case without a name. >+PASS: Should produce an exception. >+Error: Tried to add test case without a name. >+PASS: Should produce an exception. >+Error: Tried to add test case without `test` function. >+PASS: Should produce an exception. >+Error: Tried to add test case with invalid `setup` parameter (must be a function). >+PASS: Should produce an exception. >+Error: Tried to add test case with invalid `setup` parameter (must be a function). >+PASS: Should produce an exception. >+Error: Tried to add test case with invalid `setup` parameter (must be a function). >+PASS: Should produce an exception. >+Error: Tried to add test case with invalid `teardown` parameter (must be a function). >+PASS: Should produce an exception. >+Error: Tried to add test case with invalid `teardown` parameter (must be a function). >+PASS: Should produce an exception. >+Error: Tried to add test case with invalid `teardown` parameter (must be a function). >+PASS: Should produce an exception. >+Error: Tried to pass a test case with an async `setup`, `test`, or `teardown` function, but this is a synchronous test suite. >+PASS: Should produce an exception. >+Error: Tried to pass a test case with an async `setup`, `test`, or `teardown` function, but this is a synchronous test suite. >+PASS: Should produce an exception. >+Error: Tried to pass a test case with an async `setup`, `test`, or `teardown` function, but this is a synchronous test suite. >+PASS: Should produce an exception. >+Error: Tried to call runTestCases() for suite with no test cases > > == Running test suite: SyncTestSuite.RunTwiceSuite > -- Running test case: DummyTest0 > PASS: Return value of runTwiceSuite.runTestCases() should be true when all tests pass. >-PASS: should not be able to run a test suite twice. >+PASS: Should produce an exception. >+Error: Tried to call runTestCases() more than once. > > == Running test suite: SyncTestSuite.SequentialExecution >--- Running test case: DummyTest1 >--- Running test case: DummyTest2 >--- Running test case: DummyTest3 >--- Running test case: FailingTest4 >-!! EXCEPTION: [object Object] >-Stack Trace: (suppressed) >-PASS: Return value of sequentialExecutionSuite.runTestCases() should be false when a test case fails. >-PASS: sequentialExecutionSuite should have executed four tests. >-PASS: sequentialExecutionSuite should have passed three tests. >-PASS: sequentialExecutionSuite should have failed 1 test. >-PASS: sequentialExecutionSuite should have skipped zero tests. >+-- Running test case: 1 (Pass) >+-- Running test case: 2 (Pass) >+-- Running test case: 3 (Pass) >+-- Running test case: 4 (Pass) >+PASS: Return value of SyncTestSuite.SequentialExecution.runTestCases() should be true even if a test case fails. >+PASS: SyncTestSuite.SequentialExecution should have executed 4 tests. >+PASS: SyncTestSuite.SequentialExecution should have passed 4 tests. >+PASS: SyncTestSuite.SequentialExecution should have failed 0 tests. >+PASS: SyncTestSuite.SequentialExecution should have skipped 0 tests. > >-== Running test suite: SyncTestSuite.AbortOnFailure >--- Running test case: PassingTest5 >--- Running test case: FailingTest6 >-PASS: Return value of abortOnFailureSuite.runTestCases() should be false when a test case fails. >-PASS: abortOnFailureSuite should have executed two tests. >-PASS: abortOnFailureSuite should have passed one test. >-PASS: abortOnFailureSuite should have failed one test. >-PASS: abortOnFailureSuite should have skipped one test. >+== Running test suite: SyncTestSuite.ContinueOnFailure >+-- Running test case: 1 (Pass) >+-- Running test case: 2 (Fail) >+Throwing... >+!! EXCEPTION: {"x":"ContinueOnFailure throw"} >+Stack Trace: (suppressed) >+ >+-- Running test case: 3 (Pass) >+-- Running test case: 4 (Fail) >+Failing... >+PASS: Return value of SyncTestSuite.ContinueOnFailure.runTestCases() should be true even if a test case fails. >+PASS: SyncTestSuite.ContinueOnFailure should have executed 4 tests. >+PASS: SyncTestSuite.ContinueOnFailure should have passed 2 tests. >+PASS: SyncTestSuite.ContinueOnFailure should have failed 2 tests. >+PASS: SyncTestSuite.ContinueOnFailure should have skipped 0 tests. > > == Running test suite: SyncTestSuite.SetupAndTeardown > -- Running test setup. > -- Running test case: TestWithSetupAndTeardown >-PASS: Test should see side effects of running setup() action. > -- Running test teardown. >-PASS: Teardown should see side effects of running setup() action. > > -- Running test case: TestRunningAfterTeardown >-PASS: Test should see side effects of previous test's teardown() action. >+PASS: Return value of SyncTestSuite.SetupAndTeardown.runTestCases() should be true even if a test case fails. >+PASS: SyncTestSuite.SetupAndTeardown should have executed 2 tests. >+PASS: SyncTestSuite.SetupAndTeardown should have passed 2 tests. >+PASS: SyncTestSuite.SetupAndTeardown should have failed 0 tests. >+PASS: SyncTestSuite.SetupAndTeardown should have skipped 0 tests. > > == Running test suite: SyncTestSuite.SetupException > -- Running test setup. >-!! EXCEPTION: >+Throwing... >+!! EXCEPTION: SetupException throw > Stack Trace: (suppressed) > >+-- Running test setup. >+PASS: Setup action should still execute if previous test's setup action threw an exception. >+-- Running test case: TestAfterSetupException >+PASS: Test should still execute if previous test's setup action threw an exception. >+-- Running test teardown. >+PASS: Teardown action should still execute if previous test's setup action threw an exception. >+PASS: Return value of SyncTestSuite.SetupException.runTestCases() should be true even if a test case fails. >+PASS: SyncTestSuite.SetupException should have executed 1 tests. >+PASS: SyncTestSuite.SetupException should have passed 1 tests. >+PASS: SyncTestSuite.SetupException should have failed 1 tests. >+PASS: SyncTestSuite.SetupException should have skipped 1 tests. >+ > == Running test suite: SyncTestSuite.SetupFailure > -- Running test setup. >+Failing... > !! SETUP FAILED > >+-- Running test setup. >+PASS: Setup action should still execute if previous test's setup action failed. >+-- Running test case: TestAfterSetupException >+PASS: Test should still execute if previous test's setup action failed. >+-- Running test teardown. >+PASS: Teardown action should still execute if previous test's setup action failed. >+PASS: Return value of SyncTestSuite.SetupFailure.runTestCases() should be true even if a test case fails. >+PASS: SyncTestSuite.SetupFailure should have executed 1 tests. >+PASS: SyncTestSuite.SetupFailure should have passed 1 tests. >+PASS: SyncTestSuite.SetupFailure should have failed 1 tests. >+PASS: SyncTestSuite.SetupFailure should have skipped 1 tests. >+ > == Running test suite: SyncTestSuite.TeardownException > -- Running test case: TestWithExceptionDuringTeardown > -- Running test teardown. >-!! EXCEPTION: >+Throwing... >+!! EXCEPTION: TeardownException throw > Stack Trace: (suppressed) > >+-- Running test setup. >+PASS: Setup action should still execute if previous test's teardown action threw an exception. >+-- Running test case: TestAfterTeardownException >+PASS: Test should still execute if previous test's teardown action threw an exception. >+-- Running test teardown. >+PASS: Teardown action should still execute if previous test's teardown action threw an exception. >+PASS: Return value of SyncTestSuite.TeardownException.runTestCases() should be true even if a test case fails. >+PASS: SyncTestSuite.TeardownException should have executed 2 tests. >+PASS: SyncTestSuite.TeardownException should have passed 1 tests. >+PASS: SyncTestSuite.TeardownException should have failed 1 tests. >+PASS: SyncTestSuite.TeardownException should have skipped 0 tests. >+ > == Running test suite: SyncTestSuite.TeardownFailure > -- Running test case: TestWithExceptionDuringTeardown > -- Running test teardown. >+Failing... > !! TEARDOWN FAILED > >+-- Running test setup. >+PASS: Setup action should still execute if previous test's teardown action failed. >+-- Running test case: TestAfterTeardownException >+PASS: Test should still execute if previous test's teardown action failed. >+-- Running test teardown. >+PASS: Teardown action should still execute if previous test's teardown action failed. >+PASS: Return value of SyncTestSuite.TeardownFailure.runTestCases() should be true even if a test case fails. >+PASS: SyncTestSuite.TeardownFailure should have executed 2 tests. >+PASS: SyncTestSuite.TeardownFailure should have passed 1 tests. >+PASS: SyncTestSuite.TeardownFailure should have failed 1 tests. >+PASS: SyncTestSuite.TeardownFailure should have skipped 0 tests. >+ >diff --git a/LayoutTests/inspector/unit-tests/sync-test-suite.html b/LayoutTests/inspector/unit-tests/sync-test-suite.html >index 9390e8df7bc1a676021f95a679fbd5f03b4b029d..d65c095cea63a201b3eeb11b30835356bdf0f7c2 100644 >--- a/LayoutTests/inspector/unit-tests/sync-test-suite.html >+++ b/LayoutTests/inspector/unit-tests/sync-test-suite.html >@@ -1,4 +1,4 @@ >-<!doctype html> >+<!DOCTYPE html> > <html> > <head> > <script src="../../http/tests/inspector/resources/protocol-test.js"></script> >@@ -7,350 +7,404 @@ function test() > { > ProtocolTest.suppressStackTraces = true; > >- try { >- let result = new SyncTestSuite(this); >- ProtocolTest.log("FAIL: instantiating SyncTestSuite requires name argument."); >- } catch (e) { >- ProtocolTest.log("PASS: instantiating SyncTestSuite requires name argument."); >- } >+ ProtocolTest.expectException(() => { >+ new SyncTestSuite(this); >+ }); > >- try { >- let result = new SyncTestSuite(this, {}); >- ProtocolTest.log("FAIL: instantiating SyncTestSuite requires string name argument."); >- } catch (e) { >- ProtocolTest.log("PASS: instantiating SyncTestSuite requires string name argument."); >- } >+ ProtocolTest.expectException(() => { >+ new SyncTestSuite(this, {}); >+ }); > >- try { >- let result = new SyncTestSuite(this, " "); >- ProtocolTest.log("FAIL: instantiating SyncTestSuite requires non-whitespace name argument."); >- } catch (e) { >- ProtocolTest.log("PASS: instantiating SyncTestSuite requires non-whitespace name argument."); >- } >+ ProtocolTest.expectException(() => { >+ new SyncTestSuite(this, " "); >+ }); > >- try { >- let result = new SyncTestSuite("something", {}); >- ProtocolTest.log("FAIL: instantiating SyncTestSuite requires test harness argument."); >- } catch (e) { >- ProtocolTest.log("PASS: instantiating SyncTestSuite requires test harness argument."); >- } >+ ProtocolTest.expectException(() => { >+ new SyncTestSuite("something", {}); >+ }); > > let badArgsSuite = ProtocolTest.createSyncSuite("dummy"); >- try { >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase(); >- ProtocolTest.log("FAIL: should not be able to add empty test case."); >- } catch (e) { >- ProtocolTest.log("PASS: should not be able to add empty test case."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase("string"); >- ProtocolTest.log("FAIL: should not be able to add non-object test case."); >- } catch (e) { >- ProtocolTest.log("PASS: should not be able to add non-object test case."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: {}, >- test: () => true, >+ test() { >+ return true; >+ }, > }); >- ProtocolTest.log("FAIL: test case should require string name."); >- } catch (e) { >- ProtocolTest.log("PASS: test case should require string name."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: " ", >- test() {}, >+ test() { >+ }, > }); >- ProtocolTest.log("FAIL: test case should require non-whitespace name."); >- } catch (e) { >- ProtocolTest.log("PASS: test case should require non-whitespace name."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: "foo", > test: null, > }); >- ProtocolTest.log("FAIL: test case should require test function."); >- } catch (e) { >- ProtocolTest.log("PASS: test case should require test function."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: "foo", >- test() {}, >+ test() { >+ }, > setup: "astd" > }); >- ProtocolTest.log("FAIL: should not be able to specify non-Function `setup` parameter."); >- } catch (e) { >- ProtocolTest.log("PASS: should not be able to specify non-Function `setup` parameter."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: "foo", >- test() {}, >+ test() { >+ }, > setup: 123 > }); >- ProtocolTest.log("FAIL: should not be able to specify non-Function `setup` parameter."); >- } catch (e) { >- ProtocolTest.log("PASS: should not be able to specify non-Function `setup` parameter."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: "foo", >- test() {}, >+ test() { >+ }, > setup: Promise.resolve() > }); >- ProtocolTest.log("FAIL: should not be able to specify non-Function `setup` parameter."); >- } catch (e) { >- ProtocolTest.log("PASS: should not be able to specify non-Function `setup` parameter."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: "foo", >- test() {}, >+ test() { >+ }, > teardown: "astd" > }); >- ProtocolTest.log("FAIL: should not be able to specify non-Function `teardown` parameter."); >- } catch (e) { >- ProtocolTest.log("PASS: should not be able to specify non-Function `teardown` parameter."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: "foo", >- test() {}, >+ test() { >+ }, > teardown: 123 > }); >- ProtocolTest.log("FAIL: should not be able to specify non-Function `teardown` parameter."); >- } catch (e) { >- ProtocolTest.log("PASS: should not be able to specify non-Function `teardown` parameter."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: "foo", >- test() {}, >+ test() { >+ }, > teardown: Promise.resolve() > }); >- ProtocolTest.log("FAIL: should not be able to specify non-Function `teardown` parameter."); >- } catch (e) { >- ProtocolTest.log("PASS: should not be able to specify non-Function `teardown` parameter."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: "foo", >- async test() {}, >+ async test() { >+ }, > }); >- ProtocolTest.log("FAIL: should not be able to specify async `test` parameter."); >- } catch (e) { >- ProtocolTest.log("PASS: should not be able to specify async `test` parameter."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: "foo", >- async setup() {}, >- test() {}, >+ async setup() { >+ }, >+ test() { >+ }, > }); >- ProtocolTest.log("FAIL: should not be able to specify async `setup` parameter."); >- } catch (e) { >- ProtocolTest.log("PASS: should not be able to specify async `setup` parameter."); >- } >- try { >+ }); >+ ProtocolTest.expectException(() => { > badArgsSuite.addTestCase({ > name: "foo", >- test() {}, >- async teardown() {}, >+ test() { >+ }, >+ async teardown() { >+ }, > }); >- ProtocolTest.log("FAIL: should not be able to specify async `teardown` parameter."); >- } catch (e) { >- ProtocolTest.log("PASS: should not be able to specify async `teardown` parameter."); >- } >+ }); > >- let runEmptySuite = ProtocolTest.createSyncSuite("SyncTestSuite.RunEmptySuite"); >- try { >+ ProtocolTest.expectException(() => { >+ let runEmptySuite = ProtocolTest.createSyncSuite("SyncTestSuite.RunEmptySuite"); > runEmptySuite.runTestCases(); >- ProtocolTest.log("FAIL: should not be able to run empty test suite."); >- } catch (e) { >- ProtocolTest.log("PASS: should not be able to run empty test suite."); >- } >+ }); > > let runTwiceSuite = ProtocolTest.createSyncSuite("SyncTestSuite.RunTwiceSuite"); > runTwiceSuite.addTestCase({ > name: "DummyTest0", >- description: "Check that a suite can't run more than once.", >- test: () => true, >+ test() { >+ return true; >+ }, > }); > >- try { >- let result = runTwiceSuite.runTestCases(); >- ProtocolTest.expectThat(result === true, "Return value of runTwiceSuite.runTestCases() should be true when all tests pass."); >+ ProtocolTest.expectThat(runTwiceSuite.runTestCases() === true, "Return value of runTwiceSuite.runTestCases() should be true when all tests pass."); > >- runTwiceSuite.runTestCases(); // Try to trigger an error. >- ProtocolTest.log("FAIL: should not be able to run a test suite twice."); >- } catch (e) { >- ProtocolTest.log("PASS: should not be able to run a test suite twice."); >+ ProtocolTest.expectException(() => { >+ runTwiceSuite.runTestCases(); >+ }); >+ >+ function checkResult(suite, expectedCounts) { >+ ProtocolTest.expectThat(suite.runTestCases(), `Return value of ${suite.name}.runTestCases() should be true even if a test case fails.`); >+ ProtocolTest.expectEqual(expectedCounts.runCount, suite.runCount, `${suite.name} should have executed ${expectedCounts.runCount} tests.`); >+ ProtocolTest.expectEqual(expectedCounts.passCount, suite.passCount, `${suite.name} should have passed ${expectedCounts.passCount} tests.`); >+ ProtocolTest.expectEqual(expectedCounts.failCount, suite.failCount, `${suite.name} should have failed ${expectedCounts.failCount} tests.`); >+ ProtocolTest.expectEqual(expectedCounts.skipCount, suite.skipCount, `${suite.name} should have skipped ${expectedCounts.skipCount} tests.`); > } > > let thrownError = new Error({"token": 666}); > >+ let sequentialExecutionPhase = 0; > let sequentialExecutionSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SequentialExecution"); > sequentialExecutionSuite.addTestCase({ >- name: "DummyTest1", >- description: "Check test case execution order.", >- test: () => true >+ name: "1 (Pass)", >+ test() { >+ ProtocolTest.assert(sequentialExecutionPhase === 0); >+ sequentialExecutionPhase = 1; >+ return true; >+ }, > }); > sequentialExecutionSuite.addTestCase({ >- name: "DummyTest2", >- description: "Check test case execution order.", >- test: () => true >+ name: "2 (Pass)", >+ test() { >+ ProtocolTest.assert(sequentialExecutionPhase === 1); >+ sequentialExecutionPhase = 2; >+ return true; >+ }, > }); > sequentialExecutionSuite.addTestCase({ >- name: "DummyTest3", >- description: "Check test case execution order.", >- test: () => true >+ name: "3 (Pass)", >+ test() { >+ ProtocolTest.assert(sequentialExecutionPhase === 2); >+ sequentialExecutionPhase = 3; >+ return true; >+ }, > }); > sequentialExecutionSuite.addTestCase({ >- name: "FailingTest4", >- description: "Check that test fails by throwing an Error instance.", >- test() { throw thrownError; } >+ name: "4 (Pass)", >+ test() { >+ ProtocolTest.assert(sequentialExecutionPhase === 3); >+ sequentialExecutionPhase = 4; >+ return true; >+ }, >+ }); >+ checkResult(sequentialExecutionSuite, { >+ runCount: 4, >+ passCount: 4, >+ failCount: 0, >+ skipCount: 0, > }); > >- let result = sequentialExecutionSuite.runTestCases(); >- ProtocolTest.expectThat(result === false, "Return value of sequentialExecutionSuite.runTestCases() should be false when a test case fails."); >- ProtocolTest.expectThat(sequentialExecutionSuite.runCount === 4, "sequentialExecutionSuite should have executed four tests."); >- ProtocolTest.expectThat(sequentialExecutionSuite.passCount === 3, "sequentialExecutionSuite should have passed three tests."); >- ProtocolTest.expectThat(sequentialExecutionSuite.failCount === 1, "sequentialExecutionSuite should have failed 1 test."); >- ProtocolTest.expectThat(sequentialExecutionSuite.skipCount === 0, "sequentialExecutionSuite should have skipped zero tests."); >- >- let abortOnFailureSuite = ProtocolTest.createSyncSuite("SyncTestSuite.AbortOnFailure"); >- abortOnFailureSuite.addTestCase({ >- name: "PassingTest5", >- description: "This test is a dummy.", >- test: () => true >+ let continueOnFailureSuite = ProtocolTest.createSyncSuite("SyncTestSuite.ContinueOnFailure"); >+ continueOnFailureSuite.addTestCase({ >+ name: "1 (Pass)", >+ test() { >+ return true; >+ }, > }); >- abortOnFailureSuite.addTestCase({ >- name: "FailingTest6", >- description: "This test should fail by explicitly returning `false`.", >- test: () => false >+ continueOnFailureSuite.addTestCase({ >+ name: "2 (Fail)", >+ test() { >+ ProtocolTest.log("Throwing..."); >+ throw {x: "ContinueOnFailure throw"}; >+ }, > }); >- abortOnFailureSuite.addTestCase({ >- name: "PassingTest7", >- description: "This test should not executed when the preceding test fails.", >- test: () => true >+ continueOnFailureSuite.addTestCase({ >+ name: "3 (Pass)", >+ test() { >+ return true; >+ }, >+ }); >+ continueOnFailureSuite.addTestCase({ >+ name: "4 (Fail)", >+ test() { >+ ProtocolTest.log("Failing..."); >+ return false; >+ }, >+ }); >+ checkResult(continueOnFailureSuite, { >+ runCount: 4, >+ passCount: 2, >+ failCount: 2, >+ skipCount: 0, > }); > >- abortOnFailureSuite.runTestCases(); >- ProtocolTest.expectThat(result === false, "Return value of abortOnFailureSuite.runTestCases() should be false when a test case fails."); >- ProtocolTest.expectThat(abortOnFailureSuite.runCount === 2, "abortOnFailureSuite should have executed two tests."); >- ProtocolTest.expectThat(abortOnFailureSuite.passCount === 1, "abortOnFailureSuite should have passed one test."); >- ProtocolTest.expectThat(abortOnFailureSuite.failCount === 1, "abortOnFailureSuite should have failed one test."); >- ProtocolTest.expectThat(abortOnFailureSuite.skipCount === 1, "abortOnFailureSuite should have skipped one test."); >- >- let setupAndTeardownSymbol = Symbol("sync-suite-setup-and-teardown-token"); >- window[setupAndTeardownSymbol] = 0; >- >- let setupAndTeardownTestSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupAndTeardown"); >- setupAndTeardownTestSuite.addTestCase({ >+ let setupAndTeardownPhase = 0; >+ let setupAndTeardownSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupAndTeardown"); >+ setupAndTeardownSuite.addTestCase({ > name: "TestWithSetupAndTeardown", >- description: "Check execution order for setup and teardown actions.", >- setup: () => { >- window[setupAndTeardownSymbol] = 1; >+ setup() { >+ ProtocolTest.assert(setupAndTeardownPhase === 0); >+ setupAndTeardownPhase = 1; > return true; > }, > test() { >- ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 1, "Test should see side effects of running setup() action."); >- window[setupAndTeardownSymbol] = 2; >+ ProtocolTest.assert(setupAndTeardownPhase === 1); >+ setupAndTeardownPhase = 2; > return true; > }, >- teardown: () => { >- ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 2, "Teardown should see side effects of running setup() action."); >- window[setupAndTeardownSymbol] = 3; >+ teardown() { >+ ProtocolTest.assert(setupAndTeardownPhase === 2); >+ setupAndTeardownPhase = 3; > return true; >- } >+ }, > }); >- setupAndTeardownTestSuite.addTestCase({ >+ setupAndTeardownSuite.addTestCase({ > name: "TestRunningAfterTeardown", >- description: "Check execution order for test after a teardown action.", > test() { >- ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 3, "Test should see side effects of previous test's teardown() action."); >+ ProtocolTest.assert(setupAndTeardownPhase === 3); > return true; > }, > }); >+ checkResult(setupAndTeardownSuite, { >+ runCount: 2, >+ passCount: 2, >+ failCount: 0, >+ skipCount: 0, >+ }); > >- setupAndTeardownTestSuite.runTestCases(); >- >- let setupExceptionTestSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupException"); >- setupExceptionTestSuite.addTestCase({ >+ >+ let setupExceptionSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupException"); >+ setupExceptionSuite.addTestCase({ > name: "TestWithExceptionDuringSetup", >- description: "Check execution order for setup action that throws an exception.", >- setup: () => { throw new Error() }, >+ setup() { >+ ProtocolTest.log("Throwing..."); >+ throw "SetupException throw"; >+ }, > test() { >- ProtocolTest.assert(false, "Test should not execute if its setup action threw an exception."); >+ ProtocolTest.fail("Test should not execute if its setup action threw an exception."); >+ return false; >+ }, >+ teardown() { >+ ProtocolTest.fail("Teardown action should not execute if its setup action threw an exception."); > return false; > }, >- teardown: () => { >- ProtocolTest.assert(false, "Teardown action should not execute if its setup action threw an exception."); >- return false; >- } > }); >- setupExceptionTestSuite.runTestCases(); >+ setupExceptionSuite.addTestCase({ >+ name: "TestAfterSetupException", >+ setup() { >+ ProtocolTest.pass("Setup action should still execute if previous test's setup action threw an exception."); >+ return true; >+ }, >+ test() { >+ ProtocolTest.pass("Test should still execute if previous test's setup action threw an exception."); >+ return true; >+ }, >+ teardown() { >+ ProtocolTest.pass("Teardown action should still execute if previous test's setup action threw an exception."); >+ return true; >+ }, >+ }); >+ checkResult(setupExceptionSuite, { >+ runCount: 1, >+ passCount: 1, >+ failCount: 1, >+ skipCount: 1, >+ }); > >- let setupFailureTestSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupFailure"); >- setupFailureTestSuite.addTestCase({ >+ let setupFailureSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupFailure"); >+ setupFailureSuite.addTestCase({ > name: "TestWithFailureDuringSetup", >- description: "Check execution order for setup action that has failed.", >- setup: () => false, >- test() { >- ProtocolTest.assert(false, "Test should not execute if its setup action returned false.") >+ setup() { >+ ProtocolTest.log("Failing..."); >+ return false; >+ }, >+ test() { >+ ProtocolTest.fail("Test should not execute if its setup action failed."); >+ return false; >+ }, >+ teardown() { >+ ProtocolTest.fail("Teardown action should not execute if its setup action failed."); > return false; > }, >- teardown: () => { >- ProtocolTest.assert(false, "Teardown action should not execute if its setup action returned false.") >- return false; >- } > }); >- setupFailureTestSuite.runTestCases(); >+ setupFailureSuite.addTestCase({ >+ name: "TestAfterSetupException", >+ setup() { >+ ProtocolTest.pass("Setup action should still execute if previous test's setup action failed."); >+ return true; >+ }, >+ test() { >+ ProtocolTest.pass("Test should still execute if previous test's setup action failed."); >+ return true; >+ }, >+ teardown() { >+ ProtocolTest.pass("Teardown action should still execute if previous test's setup action failed."); >+ return true; >+ }, >+ }); >+ checkResult(setupFailureSuite, { >+ runCount: 1, >+ passCount: 1, >+ failCount: 1, >+ skipCount: 1, >+ }); > >- let teardownExceptionTestSuite = ProtocolTest.createSyncSuite("SyncTestSuite.TeardownException"); >- teardownExceptionTestSuite.addTestCase({ >+ let teardownExceptionSuite = ProtocolTest.createSyncSuite("SyncTestSuite.TeardownException"); >+ teardownExceptionSuite.addTestCase({ > name: "TestWithExceptionDuringTeardown", >- description: "Check execution order for teardown action that throws an exception.", >- test: () => true, >- teardown: () => { throw new Error() } >+ test() { >+ return true; >+ }, >+ teardown() { >+ ProtocolTest.log("Throwing..."); >+ throw "TeardownException throw"; >+ }, > }); >- teardownExceptionTestSuite.addTestCase({ >+ teardownExceptionSuite.addTestCase({ > name: "TestAfterTeardownException", >- descrption: "Check execution order for test after previous test's teardown throws an exception", >- setup: () => { >- ProtocolTest.assert(false, "Setup action should not execute if previous test's teardown action threw an exception."); >- return false; >+ setup() { >+ ProtocolTest.pass("Setup action should still execute if previous test's teardown action threw an exception."); >+ return true; > }, > test() { >- ProtocolTest.assert(false, "Test should not execute if previous test's teardown action threw an exception."); >- return false; >- } >+ ProtocolTest.pass("Test should still execute if previous test's teardown action threw an exception."); >+ return true; >+ }, >+ teardown() { >+ ProtocolTest.pass("Teardown action should still execute if previous test's teardown action threw an exception."); >+ return true; >+ }, >+ }); >+ checkResult(teardownExceptionSuite, { >+ runCount: 2, >+ passCount: 1, >+ failCount: 1, >+ skipCount: 0, > }); >- teardownExceptionTestSuite.runTestCases(); > >- let teardownFailureTestSuite = ProtocolTest.createSyncSuite("SyncTestSuite.TeardownFailure"); >- teardownFailureTestSuite.addTestCase({ >+ let teardownFailureSuite = ProtocolTest.createSyncSuite("SyncTestSuite.TeardownFailure"); >+ teardownFailureSuite.addTestCase({ > name: "TestWithExceptionDuringTeardown", >- description: "Check execution order for teardown action that has failed.", >- test: () => true, >- teardown: () => false, >- }); >- teardownFailureTestSuite.addTestCase({ >- name: "TestAfterTeardownException", >- descrption: "Check execution order for test after previous test's teardown throws an exception", >- setup: () => { >- ProtocolTest.assert(false, "Setup action should not execute if previous test's teardown action failed."); >+ test() { >+ return true; >+ }, >+ teardown() { >+ ProtocolTest.log("Failing..."); > return false; > }, >- test() { >- ProtocolTest.assert(false, "Test should not execute if previous test's teardown action failed."); >- return false; >- } > }); >- teardownFailureTestSuite.runTestCases(); >+ teardownFailureSuite.addTestCase({ >+ name: "TestAfterTeardownException", >+ setup() { >+ ProtocolTest.pass("Setup action should still execute if previous test's teardown action failed."); >+ return true; >+ }, >+ test() { >+ ProtocolTest.pass("Test should still execute if previous test's teardown action failed."); >+ return true; >+ }, >+ teardown() { >+ ProtocolTest.pass("Teardown action should still execute if previous test's teardown action failed."); >+ return true; >+ }, >+ }); >+ checkResult(teardownFailureSuite, { >+ runCount: 2, >+ passCount: 1, >+ failCount: 1, >+ skipCount: 0, >+ }); > > ProtocolTest.completeTest(); > }
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Formatted Diff
|
Diff
Attachments on
bug 162814
:
366781
|
366785
|
366787
|
366791
|
366793
|
366794
|
366799
|
366802
|
366804
|
366824