| Differences between
and this patch
- a/Tools/CISupport/ews-build/config.json -1 / +1 lines
Lines 335-341 a/Tools/CISupport/ews-build/config.json_sec1
335
      "name": "Commit-Queue", "shortname": "commit", "icon": "buildAndTest",
335
      "name": "Commit-Queue", "shortname": "commit", "icon": "buildAndTest",
336
      "factory": "CommitQueueFactory", "platform": "mac-bigsur",
336
      "factory": "CommitQueueFactory", "platform": "mac-bigsur",
337
      "configuration": "release", "architectures": ["x86_64"],
337
      "configuration": "release", "architectures": ["x86_64"],
338
      "workernames": ["webkit-cq-01", "webkit-cq-02", "webkit-cq-03"]
338
      "workernames": ["webkit-cq-01", "webkit-cq-02", "webkit-cq-03", "ews151"]
339
    },
339
    },
340
    {
340
    {
341
      "name": "Merge-Queue", "shortname": "merge", "icon": "buildAndTest",
341
      "name": "Merge-Queue", "shortname": "merge", "icon": "buildAndTest",
- a/Tools/CISupport/ews-build/factories.py -12 / +16 lines
Lines 24-30 a/Tools/CISupport/ews-build/factories.py_sec1
24
from buildbot.process import factory
24
from buildbot.process import factory
25
from buildbot.steps import trigger
25
from buildbot.steps import trigger
26
26
27
from steps import (AddAuthorToCommitMessage, AddReviewerToCommitMessage, AddReviewerToChangeLog, ApplyPatch, ApplyWatchList, Canonicalize,
27
from steps import (AddAuthorToCommitMessage, AddReviewerToCommitMessage, AddReviewerToChangeLog, ApplyPatch, ApplyWatchList, Canonicalize, CommitPatch,
28
                   CheckOutPullRequest, CheckOutSource, CheckOutSpecificRevision, CheckChangeRelevance,
28
                   CheckOutPullRequest, CheckOutSource, CheckOutSpecificRevision, CheckChangeRelevance,
29
                   CheckPatchStatusOnEWSQueues, CheckStyle, CleanGitRepo, CompileJSC, CompileWebKit, ConfigureBuild, CreateLocalGITCommit,
29
                   CheckPatchStatusOnEWSQueues, CheckStyle, CleanGitRepo, CompileJSC, CompileWebKit, ConfigureBuild, CreateLocalGITCommit,
30
                   DownloadBuiltProduct, ExtractBuiltProduct, FetchBranches, FindModifiedChangeLogs, FindModifiedLayoutTests, GitSvnFetch,
30
                   DownloadBuiltProduct, ExtractBuiltProduct, FetchBranches, FindModifiedChangeLogs, FindModifiedLayoutTests, GitSvnFetch,
Lines 34-40 from steps import (AddAuthorToCommitMessage, AddReviewerToCommitMessage, AddRevi a/Tools/CISupport/ews-build/factories.py_sec2
34
                   RunWebKitPyPython3Tests, RunWebKitTests, RunWebKitTestsRedTree, RunWebKitTestsInStressMode, RunWebKitTestsInStressGuardmallocMode,
34
                   RunWebKitPyPython3Tests, RunWebKitTests, RunWebKitTestsRedTree, RunWebKitTestsInStressMode, RunWebKitTestsInStressGuardmallocMode,
35
                   SetBuildSummary, ShowIdentifier, TriggerCrashLogSubmission, UpdateWorkingDirectory,
35
                   SetBuildSummary, ShowIdentifier, TriggerCrashLogSubmission, UpdateWorkingDirectory,
36
                   ValidateCommitMessage, ValidateChange, ValidateChangeLogAndReviewer, ValidateCommitterAndReviewer, WaitForCrashCollection,
36
                   ValidateCommitMessage, ValidateChange, ValidateChangeLogAndReviewer, ValidateCommitterAndReviewer, WaitForCrashCollection,
37
                   InstallBuiltProduct, VerifyGitHubIntegrity, ValidateSquashed)
37
                   InstallBuiltProduct, VerifyGitHubIntegrity, ValidateSquashed, ShowCommitMessage)
38
38
39
39
40
class Factory(factory.BuildFactory):
40
class Factory(factory.BuildFactory):
Lines 292-306 class CommitQueueFactory(factory.BuildFactory): a/Tools/CISupport/ews-build/factories.py_sec3
292
        self.addStep(ValidateChange(verifycqplus=True))
292
        self.addStep(ValidateChange(verifycqplus=True))
293
        self.addStep(ValidateCommitterAndReviewer())
293
        self.addStep(ValidateCommitterAndReviewer())
294
        self.addStep(PrintConfiguration())
294
        self.addStep(PrintConfiguration())
295
        self.addStep(CleanGitRepo(default_branch='master'))
295
        self.addStep(CleanGitRepo())
296
        self.addStep(CheckOutSource(repourl='https://git.webkit.org/git/WebKit-https'))
296
        self.addStep(CheckOutSource())
297
        self.addStep(GitSvnFetch())  # FIXME: Remove when migrating to pure git
297
        self.addStep(FetchBranches())
298
        self.addStep(FetchBranches())
298
        self.addStep(ShowIdentifier())
299
        self.addStep(ShowIdentifier())
299
        self.addStep(VerifyGitHubIntegrity())
300
        self.addStep(VerifyGitHubIntegrity())
300
        self.addStep(UpdateWorkingDirectory())
301
        self.addStep(UpdateWorkingDirectory())
301
        self.addStep(ApplyPatch())
302
        self.addStep(CommitPatch())
302
        self.addStep(ValidateChangeLogAndReviewer())
303
303
        self.addStep(FindModifiedChangeLogs())
304
        self.addStep(ValidateSquashed())
305
        self.addStep(AddReviewerToCommitMessage())
306
        self.addStep(AddAuthorToCommitMessage())
307
        self.addStep(ShowCommitMessage())
308
        self.addStep(ValidateCommitMessage())
309
        self.addStep(ShowCommitMessage())
310
304
        self.addStep(KillOldProcesses())
311
        self.addStep(KillOldProcesses())
305
        self.addStep(CompileWebKit(skipUpload=True))
312
        self.addStep(CompileWebKit(skipUpload=True))
306
        self.addStep(KillOldProcesses())
313
        self.addStep(KillOldProcesses())
Lines 308-318 class CommitQueueFactory(factory.BuildFactory): a/Tools/CISupport/ews-build/factories.py_sec4
308
        self.addStep(CheckPatchStatusOnEWSQueues())
315
        self.addStep(CheckPatchStatusOnEWSQueues())
309
        self.addStep(RunWebKitTests())
316
        self.addStep(RunWebKitTests())
310
        self.addStep(ValidateChange(addURLs=False, verifycqplus=True))
317
        self.addStep(ValidateChange(addURLs=False, verifycqplus=True))
311
        self.addStep(CheckOutSource(repourl='https://git.webkit.org/git/WebKit-https'))
318
312
        self.addStep(ShowIdentifier())
319
        self.addStep(Canonicalize())
313
        self.addStep(UpdateWorkingDirectory())
314
        self.addStep(ApplyPatch())
315
        self.addStep(CreateLocalGITCommit())
316
        self.addStep(PushCommitToWebKitRepo())
320
        self.addStep(PushCommitToWebKitRepo())
317
        self.addStep(SetBuildSummary())
321
        self.addStep(SetBuildSummary())
318
322
- a/Tools/CISupport/ews-build/steps.py -36 / +90 lines
Lines 845-850 class ApplyPatch(shell.ShellCommand, CompositeStepMixin, ShellMixin): a/Tools/CISupport/ews-build/steps.py_sec1
845
        return rc
845
        return rc
846
846
847
847
848
class CommitPatch(steps.ShellSequence, CompositeStepMixin, ShellMixin):
849
    name = 'commit-patch'
850
    description = ['commit-patch']
851
    descriptionDone = ['Created commit from patch']
852
    haltOnFailure = True
853
    env = dict(FILTER_BRANCH_SQUELCH_WARNING='1')
854
    FILTER_BRANCH_PROGRAM = '''import re
855
import sys
856
857
lines = [l for l in sys.stdin]
858
for s in re.split(r' (Need the bug URL \(OOPS!\).)|(\S+:\/\/\S+)', lines[0].rstrip()):
859
    if s and s != ' ':
860
        print(s)
861
for l in lines[1:]:
862
    sys.stdout.write(l)
863
'''
864
865
    def __init__(self, **kwargs):
866
        super(CommitPatch, self).__init__(timeout=10 * 60, logEnviron=False, **kwargs)
867
868
    def doStepIf(self, step):
869
        return self.getProperty('patch_id', False)
870
871
    def _get_patch(self):
872
        sourcestamp = self.build.getSourceStamp(self.getProperty('codebase', ''))
873
        if not sourcestamp or not sourcestamp.patch:
874
            return None
875
        return sourcestamp.patch[1]
876
877
    def run(self):
878
        self.commands = []
879
        patch = self._get_patch()
880
881
        commands = []
882
        if not patch:
883
            commands += [['curl', '-L', 'https://bugs.webkit.org/attachment.cgi?id={}'.format(self.getProperty('patch_id', '')), '-o', '.buildbot-diff']]
884
        commands += [
885
            ['git', 'am', '.buildbot-diff'],
886
            ['git', 'filter-branch', '-f', '--msg-filter', 'python3 -c "{}"'.format(self.FILTER_BRANCH_PROGRAM), 'HEAD...HEAD~1'],
887
        ]
888
        for command in commands:
889
            self.commands.append(util.ShellArg(command=command, logname='stdio', haltOnFailure=True))
890
891
        return super(CommitPatch, self).run()
892
893
    def hideStepIf(self, results, step):
894
        return not self.doStepIf(step) or (results == SUCCESS and self.getProperty('sensitive', False))
895
896
    def getResultSummary(self):
897
        if self.results == SKIPPED:
898
            return {'step': "Skipping creating commitsince patch_id isn't provided"}
899
        if self.results != SUCCESS:
900
            return {'step': 'git am failed to apply patch to trunk'}
901
        return super(CommitPatch, self).getResultSummary()
902
903
848
class CheckOutPullRequest(steps.ShellSequence, ShellMixin):
904
class CheckOutPullRequest(steps.ShellSequence, ShellMixin):
849
    name = 'checkout-pull-request'
905
    name = 'checkout-pull-request'
850
    description = ['checking-out-pull-request']
906
    description = ['checking-out-pull-request']
Lines 4576-4587 class PushCommitToWebKitRepo(shell.ShellCommand): a/Tools/CISupport/ews-build/steps.py_sec2
4576
                    ])
4632
                    ])
4577
                else:
4633
                else:
4578
                    self.build.addStepsAfterCurrentStep([
4634
                    self.build.addStepsAfterCurrentStep([
4579
                        GitResetHard(),
4635
                        ResetGitSvn(),
4580
                        CheckOutSource(repourl='https://git.webkit.org/git/WebKit-https'),
4636
                        CleanGitRepo(),
4637
                        CheckOutSource(),
4638
                        GitSvnFetch(),
4581
                        ShowIdentifier(),
4639
                        ShowIdentifier(),
4582
                        UpdateWorkingDirectory(),
4640
                        UpdateWorkingDirectory(),
4583
                        ApplyPatch(),
4641
                        CommitPatch(),
4584
                        CreateLocalGITCommit(),
4642
                        AddReviewerToCommitMessage(),
4643
                        AddAuthorToCommitMessage(),
4644
                        ValidateChange(addURLs=False, verifycqplus=True),
4645
                        Canonicalize(),
4585
                        PushCommitToWebKitRepo(),
4646
                        PushCommitToWebKitRepo(),
4586
                    ])
4647
                    ])
4587
                return rc
4648
                return rc
Lines 4762-4769 class ValidateSquashed(shell.ShellCommand): a/Tools/CISupport/ews-build/steps.py_sec3
4762
        super(ValidateSquashed, self).__init__(logEnviron=False, **kwargs)
4823
        super(ValidateSquashed, self).__init__(logEnviron=False, **kwargs)
4763
4824
4764
    def start(self, BufferLogObserverClass=logobserver.BufferLogObserver):
4825
    def start(self, BufferLogObserverClass=logobserver.BufferLogObserver):
4765
        base_ref = self.getProperty('github.base.ref', DEFAULT_BRANCH)
4826
        base_ref = self.getProperty('github.base.ref', f'origin/{DEFAULT_BRANCH}')
4766
        head_ref = self.getProperty('github.head.ref', DEFAULT_BRANCH)
4827
        head_ref = self.getProperty('github.head.ref', 'HEAD')
4767
        self.command = ['git', 'log', '--oneline', head_ref, f'^{base_ref}', '--max-count=2']
4828
        self.command = ['git', 'log', '--oneline', head_ref, f'^{base_ref}', '--max-count=2']
4768
4829
4769
        self.log_observer = BufferLogObserverClass(wantStderr=True)
4830
        self.log_observer = BufferLogObserverClass(wantStderr=True)
Lines 4788-4799 class ValidateSquashed(shell.ShellCommand): a/Tools/CISupport/ews-build/steps.py_sec4
4788
            return SUCCESS
4849
            return SUCCESS
4789
        return FAILURE
4850
        return FAILURE
4790
4851
4791
    def doStepIf(self, step):
4792
        return self.getProperty('github.number')
4793
4794
    def hideStepIf(self, results, step):
4795
        return not self.doStepIf(step)
4796
4797
4852
4798
class AddReviewerMixin(object):
4853
class AddReviewerMixin(object):
4799
    NOBODY_SED = 's/NOBODY (OO*PP*S!*)/{}/g'
4854
    NOBODY_SED = 's/NOBODY (OO*PP*S!*)/{}/g'
Lines 4831-4838 class AddReviewerToCommitMessage(shell.ShellCommand, AddReviewerMixin): a/Tools/CISupport/ews-build/steps.py_sec5
4831
        super(AddReviewerToCommitMessage, self).__init__(logEnviron=False, timeout=60, **kwargs)
4886
        super(AddReviewerToCommitMessage, self).__init__(logEnviron=False, timeout=60, **kwargs)
4832
4887
4833
    def start(self, BufferLogObserverClass=logobserver.BufferLogObserver):
4888
    def start(self, BufferLogObserverClass=logobserver.BufferLogObserver):
4834
        base_ref = self.getProperty('github.base.ref', DEFAULT_BRANCH)
4889
        base_ref = self.getProperty('github.base.ref', f'origin/{DEFAULT_BRANCH}')
4835
        head_ref = self.getProperty('github.head.ref', DEFAULT_BRANCH)
4890
        head_ref = self.getProperty('github.head.ref', 'HEAD')
4836
4891
4837
        gmtoffset = int(time.localtime().tm_gmtoff * 100 / (60 * 60))
4892
        gmtoffset = int(time.localtime().tm_gmtoff * 100 / (60 * 60))
4838
        date = f'{int(time.time())} {gmtoffset}'
4893
        date = f'{int(time.time())} {gmtoffset}'
Lines 4856-4862 class AddReviewerToCommitMessage(shell.ShellCommand, AddReviewerMixin): a/Tools/CISupport/ews-build/steps.py_sec6
4856
        return super(AddReviewerToCommitMessage, self).getResultSummary()
4911
        return super(AddReviewerToCommitMessage, self).getResultSummary()
4857
4912
4858
    def doStepIf(self, step):
4913
    def doStepIf(self, step):
4859
        return self.getProperty('github.number') and self.getProperty('reviewers_full_names')
4914
        return self.getProperty('reviewers_full_names')
4860
4915
4861
    def hideStepIf(self, results, step):
4916
    def hideStepIf(self, results, step):
4862
        return not self.doStepIf(step)
4917
        return not self.doStepIf(step)
Lines 4886-4893 class AddAuthorToCommitMessage(shell.ShellCommand, AddReviewerMixin): a/Tools/CISupport/ews-build/steps.py_sec7
4886
        return None, None
4941
        return None, None
4887
4942
4888
    def start(self):
4943
    def start(self):
4889
        base_ref = self.getProperty('github.base.ref', DEFAULT_BRANCH)
4944
        base_ref = self.getProperty('github.base.ref', f'origin/{DEFAULT_BRANCH}')
4890
        head_ref = self.getProperty('github.head.ref', DEFAULT_BRANCH)
4945
        head_ref = self.getProperty('github.head.ref', 'HEAD')
4891
4946
4892
        gmtoffset = int(time.localtime().tm_gmtoff * 100 / (60 * 60))
4947
        gmtoffset = int(time.localtime().tm_gmtoff * 100 / (60 * 60))
4893
        timestamp = f'{int(time.time())} {gmtoffset}'
4948
        timestamp = f'{int(time.time())} {gmtoffset}'
Lines 4916-4924 class AddAuthorToCommitMessage(shell.ShellCommand, AddReviewerMixin): a/Tools/CISupport/ews-build/steps.py_sec8
4916
        return super(AddAuthorToCommitMessage, self).getResultSummary()
4971
        return super(AddAuthorToCommitMessage, self).getResultSummary()
4917
4972
4918
    def doStepIf(self, step):
4973
    def doStepIf(self, step):
4919
        if not self.getProperty('github.number'):
4920
            return False
4921
4922
        name, email = self.author()
4974
        name, email = self.author()
4923
        return name and email
4975
        return name and email
4924
4976
Lines 4991-4998 class ValidateCommitMessage(steps.ShellSequence, ShellMixin): a/Tools/CISupport/ews-build/steps.py_sec9
4991
5043
4992
    @defer.inlineCallbacks
5044
    @defer.inlineCallbacks
4993
    def run(self, BufferLogObserverClass=logobserver.BufferLogObserver):
5045
    def run(self, BufferLogObserverClass=logobserver.BufferLogObserver):
4994
        base_ref = self.getProperty('github.base.ref', DEFAULT_BRANCH)
5046
        base_ref = self.getProperty('github.base.ref', f'origin/{DEFAULT_BRANCH}')
4995
        head_ref = self.getProperty('github.head.ref', DEFAULT_BRANCH)
5047
        head_ref = self.getProperty('github.head.ref', 'HEAD')
4996
5048
4997
        self.commands = []
5049
        self.commands = []
4998
        commands = [
5050
        commands = [
Lines 5027-5033 class ValidateCommitMessage(steps.ShellSequence, ShellMixin): a/Tools/CISupport/ews-build/steps.py_sec10
5027
        if rc == FAILURE:
5079
        if rc == FAILURE:
5028
            self.setProperty('comment_text', f"{self.summary}, blocking PR #{self.getProperty('github.number')}")
5080
            self.setProperty('comment_text', f"{self.summary}, blocking PR #{self.getProperty('github.number')}")
5029
            self.setProperty('build_finish_summary', 'Commit message validation failed')
5081
            self.setProperty('build_finish_summary', 'Commit message validation failed')
5030
            self.build.addStepsAfterCurrentStep([LeaveComment(),  BlockPullRequest()])
5082
            self.build.addStepsAfterCurrentStep([LeaveComment(), SetCommitQueueMinusFlagOnPatch(), BlockPullRequest()])
5031
        return rc
5083
        return rc
5032
5084
5033
    def getResultSummary(self):
5085
    def getResultSummary(self):
Lines 5035-5046 class ValidateCommitMessage(steps.ShellSequence, ShellMixin): a/Tools/CISupport/ews-build/steps.py_sec11
5035
            return {'step': self.summary}
5087
            return {'step': self.summary}
5036
        return super(ValidateCommitMessage, self).getResultSummary()
5088
        return super(ValidateCommitMessage, self).getResultSummary()
5037
5089
5038
    def doStepIf(self, step):
5039
        return self.getProperty('github.number')
5040
5041
    def hideStepIf(self, results, step):
5042
        return not self.doStepIf(step)
5043
5044
5090
5045
class Canonicalize(steps.ShellSequence, ShellMixin):
5091
class Canonicalize(steps.ShellSequence, ShellMixin):
5046
    name = 'canonicalize-commit'
5092
    name = 'canonicalize-commit'
Lines 5056-5070 class Canonicalize(steps.ShellSequence, ShellMixin): a/Tools/CISupport/ews-build/steps.py_sec12
5056
        self.commands = []
5102
        self.commands = []
5057
5103
5058
        base_ref = self.getProperty('github.base.ref', DEFAULT_BRANCH)
5104
        base_ref = self.getProperty('github.base.ref', DEFAULT_BRANCH)
5059
        head_ref = self.getProperty('github.head.ref', DEFAULT_BRANCH)
5105
        head_ref = self.getProperty('github.head.ref', None)
5060
5106
5061
        commands = []
5107
        commands = []
5062
        if self.rebase_enabled:
5108
        if self.rebase_enabled:
5063
            commands = [
5109
            commands = [['git', 'pull', 'origin', base_ref, '--rebase']]
5064
                ['git', 'pull', 'origin', base_ref, '--rebase'],
5110
            if head_ref:
5065
                ['git', 'branch', '-f', base_ref, head_ref],
5111
                commands += [['git', 'branch', '-f', base_ref, head_ref]]
5066
                ['git', 'checkout', base_ref],
5112
            commands += [['git', 'checkout', base_ref]]
5067
            ]
5068
        commands.append(['python3', 'Tools/Scripts/git-webkit', 'canonicalize', '-n', '1' if self.rebase_enabled else '3'])
5113
        commands.append(['python3', 'Tools/Scripts/git-webkit', 'canonicalize', '-n', '1' if self.rebase_enabled else '3'])
5069
5114
5070
        for command in commands:
5115
        for command in commands:
Lines 5246-5248 class ResetGitSvn(shell.ShellCommand): a/Tools/CISupport/ews-build/steps.py_sec13
5246
        if self.results == FAILURE:
5291
        if self.results == FAILURE:
5247
            return {'step': 'Failed to remove git-svn references'}
5292
            return {'step': 'Failed to remove git-svn references'}
5248
        return super(GitSvnFetch, self).getResultSummary()
5293
        return super(GitSvnFetch, self).getResultSummary()
5249
- 
5294
5295
5296
class ShowCommitMessage(shell.ShellCommand):
5297
    name = 'show-commit-message'
5298
    haltOnFailure = False
5299
    flunkOnFailure = False
5300
    command = ['git', 'log', '-1']
5301
5302
    def __init__(self, **kwargs):
5303
        super(ShowCommitMessage, self).__init__(logEnviron=False, timeout=300, **kwargs)

Return to Bug 240461