Attachment 345876 Details for Bug 188079 – Proposed patch

[patch] Proposed patch

ews_APITests_v2.patch (text/plain), 12.10 KB, created by Aakash Jain on 2018-07-26 15:07:14 PDT

(hide)

Description:

Filename:

MIME Type:

Creator: Aakash Jain

Created: 2018-07-26 15:07:14 PDT

Size: 12.10 KB

patch

obsolete

>Index: Tools/ChangeLog
>===================================================================
>--- Tools/ChangeLog	(revision 234277)
>+++ Tools/ChangeLog	(working copy)
>@@ -1,3 +1,20 @@
>+2018-07-26  Aakash Jain  <aakash_jain@apple.com>
>+
>+        [ews-build] Add build step to run API Tests
>+        https://bugs.webkit.org/show_bug.cgi?id=188079
>+
>+        Reviewed by NOBODY (OOPS!).
>+
>+        * BuildSlaveSupport/ews-build/steps.py:
>+        (TestWithFailureCount): Common class for various tests with failure count.
>+        (TestWithFailureCount.start): Configure Buildbot's logobserver.
>+        (TestWithFailureCount.commandComplete): Updates the failure count.
>+        (TestWithFailureCount.evaluateCommand): Set the step status appropriately.
>+        (TestWithFailureCount.getResultSummary): Set the step summary appropriately.
>+        (RunAPITests): Build step to run API tests.
>+        (RunAPITests.countFailures): Count the failures by parsing the output.
>+        * BuildSlaveSupport/ews-build/steps_unittest.py: Added unit-tests.
>+
> 2018-07-26  Commit Queue  <commit-queue@webkit.org>
> 
>         Unreviewed, rolling out r234181 and r234189.
>Index: Tools/BuildSlaveSupport/ews-build/steps.py
>===================================================================
>--- Tools/BuildSlaveSupport/ews-build/steps.py	(revision 234277)
>+++ Tools/BuildSlaveSupport/ews-build/steps.py	(working copy)
>@@ -20,12 +20,14 @@
> # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> 
>-from buildbot.process import buildstep, properties
>+from buildbot.process import buildstep, logobserver, properties
> from buildbot.process.results import Results, SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION, RETRY
> from buildbot.steps import shell, transfer
> from buildbot.steps.source import svn
> from twisted.internet import defer
> 
>+import re
>+
> EWS_URL = 'http://ews-build.webkit-uat.org/'
> WithProperties = properties.WithProperties
> 
>@@ -89,6 +91,44 @@ class CheckStyle(shell.ShellCommand):
>     command = ['Tools/Scripts/check-webkit-style']
> 
> 
>+class TestWithFailureCount(shell.Test):
>+    failedTestsFormatString = "%d test%s failed"
>+    failedTestCount = 0
>+
>+    def start(self):
>+        self.log_observer = logobserver.BufferLogObserver(wantStderr=True)
>+        self.addLogObserver('stdio', self.log_observer)
>+        return shell.Test.start(self)
>+
>+    def countFailures(self, cmd):
>+        raise NotImplementedError
>+
>+    def commandComplete(self, cmd):
>+        shell.Test.commandComplete(self, cmd)
>+        self.failedTestCount = self.countFailures(cmd)
>+        self.failedTestPluralSuffix = "" if self.failedTestCount == 1 else "s"
>+
>+    def evaluateCommand(self, cmd):
>+        if self.failedTestCount:
>+            return FAILURE
>+
>+        if cmd.rc != 0:
>+            return FAILURE
>+
>+        return SUCCESS
>+
>+    def getResultSummary(self):
>+        status = self.name
>+
>+        if self.results != SUCCESS and self.failedTestCount:
>+            status = self.failedTestsFormatString % (self.failedTestCount, self.failedTestPluralSuffix)
>+
>+        if self.results != SUCCESS:
>+            status += u' ({})'.format(Results[self.results])
>+
>+        return {u'step': status}
>+
>+
> class RunBindingsTests(shell.ShellCommand):
>     name = 'bindings-tests'
>     description = ['bindings-tests running']
>@@ -330,3 +370,23 @@ class ExtractBuiltProduct(shell.ShellCom
>     descriptionDone = ['extracted built product']
>     haltOnFailure = True
>     flunkOnFailure = True
>+
>+
>+class RunAPITests(TestWithFailureCount):
>+    name = 'run-api-tests'
>+    description = ['api tests running']
>+    descriptionDone = ['api-tests']
>+    command = ['python', 'Tools/Scripts/run-api-tests', '--no-build', WithProperties('--%(configuration)s'), '--verbose']
>+    failedTestsFormatString = '%d api test%s failed or timed out'
>+
>+    def start(self):
>+        appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
>+        return TestWithFailureCount.start(self)
>+
>+    def countFailures(self, cmd):
>+        log_text = self.log_observer.getStdout() + self.log_observer.getStderr()
>+
>+        match = re.search(r'Ran (?P<ran>\d+) tests of (?P<total>\d+) with (?P<passed>\d+) successful', log_text)
>+        if not match:
>+            return 0
>+        return int(match.group('ran')) - int(match.group('passed'))
>Index: Tools/BuildSlaveSupport/ews-build/steps_unittest.py
>===================================================================
>--- Tools/BuildSlaveSupport/ews-build/steps_unittest.py	(revision 234277)
>+++ Tools/BuildSlaveSupport/ews-build/steps_unittest.py	(working copy)
>@@ -858,5 +858,203 @@ class TestExtractBuiltProduct(BuildStepM
>         return self.runStep()
> 
> 
>+class TestRunAPITests(BuildStepMixinAdditions, unittest.TestCase):
>+    def setUp(self):
>+        self.longMessage = True
>+        return self.setUpBuildStep()
>+
>+    def tearDown(self):
>+        return self.tearDownBuildStep()
>+
>+    def test_success_mac(self):
>+        self.setupStep(RunAPITests())
>+        self.setProperty('fullPlatform', 'mac-sierra')
>+        self.setProperty('platform', 'mac')
>+        self.setProperty('configuration', 'release')
>+
>+        self.expectRemoteCommands(
>+            ExpectShell(workdir='wkdir',
>+                        command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--release', '--verbose'],
>+                        )
>+            + ExpectShell.log('stdio', stdout='''...
>+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
>+worker/0 TestWTF.WTF_Variant.Ref Passed
>+worker/0 TestWTF.WTF_Variant.RefPtr Passed
>+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
>+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
>+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
>+Ran 1888 tests of 1888 with 1888 successful
>+------------------------------
>+All tests successfully passed!
>+''')
>+            + 0,
>+        )
>+        self.expectOutcome(result=SUCCESS, state_string='run-api-tests')
>+        return self.runStep()
>+
>+    def test_success_ios_simulator(self):
>+        self.setupStep(RunAPITests())
>+        self.setProperty('fullPlatform', 'ios-simulator-11')
>+        self.setProperty('platform', 'ios')
>+        self.setProperty('configuration', 'debug')
>+
>+        self.expectRemoteCommands(
>+            ExpectShell(workdir='wkdir',
>+                        command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose', '--ios-simulator'],
>+                        )
>+            + ExpectShell.log('stdio', stdout='''...
>+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
>+worker/0 TestWTF.WTF_Variant.Ref Passed
>+worker/0 TestWTF.WTF_Variant.RefPtr Passed
>+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
>+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
>+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
>+Ran 1888 tests of 1888 with 1888 successful
>+------------------------------
>+All tests successfully passed!
>+''')
>+            + 0,
>+        )
>+        self.expectOutcome(result=SUCCESS, state_string='run-api-tests')
>+        return self.runStep()
>+
>+    def test_one_failure(self):
>+        self.setupStep(RunAPITests())
>+        self.setProperty('fullPlatform', 'mac-sierra')
>+        self.setProperty('platform', 'mac')
>+        self.setProperty('configuration', 'debug')
>+
>+        self.expectRemoteCommands(
>+            ExpectShell(workdir='wkdir',
>+                        command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose'],
>+                        )
>+            + ExpectShell.log('stdio', stdout='''
>+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
>+worker/0 TestWTF.WTF_Variant.Ref Passed
>+worker/0 TestWTF.WTF_Variant.RefPtr Passed
>+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
>+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
>+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
>+worker/0 exiting
>+Ran 1888 tests of 1888 with 1887 successful
>+------------------------------
>+Test suite failed
>+
>+Crashed
>+
>+    TestWTF.WTF.StringConcatenate_Unsigned
>+        **FAIL** WTF.StringConcatenate_Unsigned
>+
>+        Tools\\TestWebKitAPI\\Tests\\WTF\\StringConcatenate.cpp:84
>+        Value of: makeString('hello ', static_cast<unsigned short>(42) , ' world')
>+          Actual: hello 42 world
>+        Expected: 'hello * world'
>+        Which is: 74B00C9C
>+
>+Testing completed, Exit status: 3
>+''')
>+            + 1,
>+        )
>+        self.expectOutcome(result=FAILURE, state_string='1 api test failed or timed out (failure)')
>+        return self.runStep()
>+
>+    def test_multiple_failures_and_timeouts(self):
>+        self.setupStep(RunAPITests())
>+        self.setProperty('fullPlatform', 'mac-sierra')
>+        self.setProperty('platform', 'mac')
>+        self.setProperty('configuration', 'debug')
>+
>+        self.expectRemoteCommands(
>+            ExpectShell(workdir='wkdir',
>+                        command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose'],
>+                        )
>+            + ExpectShell.log('stdio', stdout='''...
>+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
>+worker/0 TestWTF.WTF_Variant.Ref Passed
>+worker/0 TestWTF.WTF_Variant.RefPtr Passed
>+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
>+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
>+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
>+worker/0 exiting
>+Ran 1888 tests of 1888 with 1884 successful
>+------------------------------
>+Test suite failed
>+
>+Failed
>+
>+    TestWTF.WTF.StringConcatenate_Unsigned
>+        **FAIL** WTF.StringConcatenate_Unsigned
>+
>+        Tools\\TestWebKitAPI\\Tests\\WTF\\StringConcatenate.cpp:84
>+        Value of: makeString('hello ', static_cast<unsigned short>(42) , ' world')
>+          Actual: hello 42 world
>+        Expected: 'hello * world'
>+        Which is: 74B00C9C
>+
>+    TestWTF.WTF_Expected.Unexpected
>+        **FAIL** WTF_Expected.Unexpected
>+
>+        Tools\TestWebKitAPI\Tests\WTF\Expected.cpp:96
>+        Value of: s1
>+          Actual: oops
>+        Expected: s0
>+        Which is: oops
>+
>+Timeout
>+
>+    TestWTF.WTF_PoisonedUniquePtrForTriviallyDestructibleArrays.Assignment
>+    TestWTF.WTF_Lock.ContendedShortSection
>+
>+Testing completed, Exit status: 3
>+''')
>+            + 4,
>+        )
>+        self.expectOutcome(result=FAILURE, state_string='4 api tests failed or timed out (failure)')
>+        return self.runStep()
>+
>+    def test_unexpecte_failure(self):
>+        self.setupStep(RunAPITests())
>+        self.setProperty('fullPlatform', 'mac-sierra')
>+        self.setProperty('platform', 'mac')
>+        self.setProperty('configuration', 'debug')
>+
>+        self.expectRemoteCommands(
>+            ExpectShell(workdir='wkdir',
>+                        command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose'],
>+                        )
>+            + ExpectShell.log('stdio', stdout='Unexpected failure. Failed to run api tests.')
>+            + 2,
>+        )
>+        self.expectOutcome(result=FAILURE, state_string='run-api-tests (failure)')
>+        return self.runStep()
>+
>+    def test_no_failures_or_timeouts_with_disabled(self):
>+        self.setupStep(RunAPITests())
>+        self.setProperty('fullPlatform', 'mac-sierra')
>+        self.setProperty('platform', 'mac')
>+        self.setProperty('configuration', 'debug')
>+
>+        self.expectRemoteCommands(
>+            ExpectShell(workdir='wkdir',
>+                        command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose'],
>+                        )
>+            + ExpectShell.log('stdio', stdout='''...
>+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
>+worker/0 TestWTF.WTF_Variant.Ref Passed
>+worker/0 TestWTF.WTF_Variant.RefPtr Passed
>+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
>+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
>+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
>+worker/0 exiting
>+Ran 1881 tests of 1888 with 1881 successful
>+------------------------------
>+All tests successfully passed!
>+''')
>+            + 0,
>+        )
>+        self.expectOutcome(result=SUCCESS, state_string='run-api-tests')
>+        return self.runStep()
>+
>+
> if __name__ == '__main__':
>     unittest.main()

Flags:

lforschler: review+

Actions: View | Formatted Diff | Diff

Attachments on bug 188079: 345875 | 345876