WebKit Bugzilla
Attachment 369970 Details for
Bug 197914
: [ews-build] Enabling uploading EWS archives to S3
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Requests
|
Help
|
New Account
|
Log In
Remember
[x]
|
Forgot Password
Login:
[x]
[patch]
Patch
bug-197914-20190515134011.patch (text/plain), 11.72 KB, created by
Aakash Jain
on 2019-05-15 10:40:12 PDT
(
hide
)
Description:
Patch
Filename:
MIME Type:
Creator:
Aakash Jain
Created:
2019-05-15 10:40:12 PDT
Size:
11.72 KB
patch
obsolete
>Index: Tools/ChangeLog >=================================================================== >--- Tools/ChangeLog (revision 245327) >+++ Tools/ChangeLog (working copy) >@@ -1,3 +1,18 @@ >+2019-05-15 Aakash Jain <aakash_jain@apple.com> >+ >+ [ews-build] Enabling uploading EWS archives to S3 >+ https://bugs.webkit.org/show_bug.cgi?id=197914 >+ >+ Reviewed by NOBODY (OOPS!). >+ >+ * BuildSlaveSupport/Shared: Added. >+ * BuildSlaveSupport/Shared/transfer-archive-to-s3: Moved from Tools/BuildSlaveSupport/build.webkit.org-config/transfer-archive-to-s3. >+ (archiveExists): Replace tab with space. >+ (main): Added main method. >+ * BuildSlaveSupport/build.webkit.org-config/steps.py: >+ (TransferToS3): Updated patch to the script. >+ * BuildSlaveSupport/build.webkit.org-config/transfer-archive-to-s3: Moved to Shared folder. >+ > 2019-05-15 Youenn Fablet <youenn@apple.com> > > Constant crashes under WebPage::isThrottleable() after r245299 >Index: Tools/BuildSlaveSupport/Shared/transfer-archive-to-s3 >=================================================================== >--- Tools/BuildSlaveSupport/Shared/transfer-archive-to-s3 (revision 245321) >+++ Tools/BuildSlaveSupport/Shared/transfer-archive-to-s3 (working copy) >@@ -2,10 +2,10 @@ > import argparse > import boto3 > import os >-import os.path > import sys > >-S3_BUCKET = 'archives.webkit.org' >+S3_DEFAULT_BUCKET = 'archives.webkit.org' >+S3_EWS_BUCKET = 'ews-archives.webkit.org' > S3_MINIFIED_BUCKET = 'minified-archives.webkit.org' > S3_REGION_PREFIX = 'https://s3-us-west-2.amazonaws.com' > >@@ -19,23 +19,32 @@ def uploadToS3(archive_path, bucket, ide > def archiveExists(archive): > if archive: > if os.path.exists(archive): >- return True >+ return True > else: > print 'WARNING: Archive does not exist: {}'.format(archive) > return False >- >-parser = argparse.ArgumentParser(add_help=True) >-parser.add_argument('--revision', action="store", required=True, help='Revision number for the built archive') >-parser.add_argument('--identifier', action="store", required=True, help='S3 destination identifier, in the form of fullPlatform-architecture-configuration. [mac-sierra-x86_64-release]') >-parser.add_argument('--archive', action="store", required=True, help='Path to the full size archive. [path/to/123456.zip]') >-args = parser.parse_args() >- >- >-head, tail = os.path.split(str(args.archive)) >-minifiedArchive = head + '/minified-' + tail >-s3 = boto3.client('s3') >- >-if archiveExists(args.archive): >- uploadToS3(args.archive, S3_BUCKET, args.identifier, args.revision) >-if archiveExists(minifiedArchive): >- uploadToS3(minifiedArchive, S3_MINIFIED_BUCKET, args.identifier, args.revision) >\ No newline at end of file >+ >+def main(): >+ parser = argparse.ArgumentParser(add_help=True) >+ parser.add_argument('--revision', '--patch_id', action="store", required=True, help='Revision number or patch_id for the built archive') >+ parser.add_argument('--identifier', action="store", required=True, help='S3 destination identifier, in the form of fullPlatform-architecture-configuration. [mac-sierra-x86_64-release]') >+ parser.add_argument('--archive', action="store", required=True, help='Path to the full size archive. [path/to/123456.zip]') >+ parser.add_argument('--ews', action='store_true') >+ args = parser.parse_args() >+ >+ >+ parentdir, filename = os.path.split(str(args.archive)) >+ minifiedArchive = os.path.join(parentdir, 'minified-' + filename) >+ s3 = boto3.client('s3') >+ >+ S3_BUCKET = S3_DEFAULT_BUCKET >+ if args.ews: >+ S3_BUCKET = S3_EWS_BUCKET >+ >+ if archiveExists(args.archive): >+ uploadToS3(args.archive, S3_BUCKET, args.identifier, args.revision) >+ if not args.ews and archiveExists(minifiedArchive): >+ uploadToS3(minifiedArchive, S3_MINIFIED_BUCKET, args.identifier, args.revision) >+ >+if __name__ == "__main__": >+ main() >Index: Tools/BuildSlaveSupport/Shared/transfer-archive-to-s3 >=================================================================== >--- Tools/BuildSlaveSupport/Shared/transfer-archive-to-s3 (revision 245321) (from Tools/BuildSlaveSupport/build.webkit.org-config/transfer-archive-to-s3:245321) >+++ Tools/BuildSlaveSupport/Shared/transfer-archive-to-s3 (working copy) >@@ -0,0 +1,41 @@ >+#!/usr/bin/env python >+import argparse >+import boto3 >+import os >+import os.path >+import sys >+ >+S3_BUCKET = 'archives.webkit.org' >+S3_MINIFIED_BUCKET = 'minified-archives.webkit.org' >+S3_REGION_PREFIX = 'https://s3-us-west-2.amazonaws.com' >+ >+def uploadToS3(archive_path, bucket, identifier, revision): >+ print 'Transferring {} to S3...'.format(archive_path) >+ key = '/'.join([identifier, revision + '.zip']) >+ print '\tS3 Bucket: {}\n\tS3 Key: {}'.format(bucket, key) >+ s3.upload_file(archive_path, bucket, key) >+ print('\tS3 URL: {}/{}/{}'.format(S3_REGION_PREFIX, bucket, key)) >+ >+def archiveExists(archive): >+ if archive: >+ if os.path.exists(archive): >+ return True >+ else: >+ print 'WARNING: Archive does not exist: {}'.format(archive) >+ return False >+ >+parser = argparse.ArgumentParser(add_help=True) >+parser.add_argument('--revision', action="store", required=True, help='Revision number for the built archive') >+parser.add_argument('--identifier', action="store", required=True, help='S3 destination identifier, in the form of fullPlatform-architecture-configuration. [mac-sierra-x86_64-release]') >+parser.add_argument('--archive', action="store", required=True, help='Path to the full size archive. [path/to/123456.zip]') >+args = parser.parse_args() >+ >+ >+head, tail = os.path.split(str(args.archive)) >+minifiedArchive = head + '/minified-' + tail >+s3 = boto3.client('s3') >+ >+if archiveExists(args.archive): >+ uploadToS3(args.archive, S3_BUCKET, args.identifier, args.revision) >+if archiveExists(minifiedArchive): >+ uploadToS3(minifiedArchive, S3_MINIFIED_BUCKET, args.identifier, args.revision) >\ No newline at end of file >Index: Tools/BuildSlaveSupport/Shared/transfer-archive-to-s3 >=================================================================== >--- Tools/BuildSlaveSupport/Shared/transfer-archive-to-s3 (revision 245321) >+++ Tools/BuildSlaveSupport/Shared/transfer-archive-to-s3 (working copy) >@@ -2,10 +2,10 @@ > import argparse > import boto3 > import os >-import os.path > import sys > >-S3_BUCKET = 'archives.webkit.org' >+S3_DEFAULT_BUCKET = 'archives.webkit.org' >+S3_EWS_BUCKET = 'ews-archives.webkit.org' > S3_MINIFIED_BUCKET = 'minified-archives.webkit.org' > S3_REGION_PREFIX = 'https://s3-us-west-2.amazonaws.com' > >@@ -19,23 +19,32 @@ def uploadToS3(archive_path, bucket, ide > def archiveExists(archive): > if archive: > if os.path.exists(archive): >- return True >+ return True > else: > print 'WARNING: Archive does not exist: {}'.format(archive) > return False >- >-parser = argparse.ArgumentParser(add_help=True) >-parser.add_argument('--revision', action="store", required=True, help='Revision number for the built archive') >-parser.add_argument('--identifier', action="store", required=True, help='S3 destination identifier, in the form of fullPlatform-architecture-configuration. [mac-sierra-x86_64-release]') >-parser.add_argument('--archive', action="store", required=True, help='Path to the full size archive. [path/to/123456.zip]') >-args = parser.parse_args() >- >- >-head, tail = os.path.split(str(args.archive)) >-minifiedArchive = head + '/minified-' + tail >-s3 = boto3.client('s3') >- >-if archiveExists(args.archive): >- uploadToS3(args.archive, S3_BUCKET, args.identifier, args.revision) >-if archiveExists(minifiedArchive): >- uploadToS3(minifiedArchive, S3_MINIFIED_BUCKET, args.identifier, args.revision) >\ No newline at end of file >+ >+def main(): >+ parser = argparse.ArgumentParser(add_help=True) >+ parser.add_argument('--revision', '--patch_id', action="store", required=True, help='Revision number or patch_id for the built archive') >+ parser.add_argument('--identifier', action="store", required=True, help='S3 destination identifier, in the form of fullPlatform-architecture-configuration. [mac-sierra-x86_64-release]') >+ parser.add_argument('--archive', action="store", required=True, help='Path to the full size archive. [path/to/123456.zip]') >+ parser.add_argument('--ews', action='store_true') >+ args = parser.parse_args() >+ >+ >+ parentdir, filename = os.path.split(str(args.archive)) >+ minifiedArchive = os.path.join(parentdir, 'minified-' + filename) >+ s3 = boto3.client('s3') >+ >+ S3_BUCKET = S3_DEFAULT_BUCKET >+ if args.ews: >+ S3_BUCKET = S3_EWS_BUCKET >+ >+ if archiveExists(args.archive): >+ uploadToS3(args.archive, S3_BUCKET, args.identifier, args.revision) >+ if not args.ews and archiveExists(minifiedArchive): >+ uploadToS3(minifiedArchive, S3_MINIFIED_BUCKET, args.identifier, args.revision) >+ >+if __name__ == "__main__": >+ main() >Index: Tools/BuildSlaveSupport/build.webkit.org-config/steps.py >=================================================================== >--- Tools/BuildSlaveSupport/build.webkit.org-config/steps.py (revision 245321) >+++ Tools/BuildSlaveSupport/build.webkit.org-config/steps.py (working copy) >@@ -874,7 +874,7 @@ class TransferToS3(master.MasterShellCom > minifiedArchive = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/minified-%(got_revision)s.zip") > identifier = WithProperties("%(fullPlatform)s-%(architecture)s-%(configuration)s") > revision = WithProperties("%(got_revision)s") >- command = ["python", "./transfer-archive-to-s3", "--revision", revision, "--identifier", identifier, "--archive", archive] >+ command = ["python", "../Shared/transfer-archive-to-s3", "--revision", revision, "--identifier", identifier, "--archive", archive] > haltOnFailure = True > > def __init__(self, **kwargs): >Index: Tools/BuildSlaveSupport/build.webkit.org-config/transfer-archive-to-s3 >=================================================================== >--- Tools/BuildSlaveSupport/build.webkit.org-config/transfer-archive-to-s3 (revision 245321) >+++ Tools/BuildSlaveSupport/build.webkit.org-config/transfer-archive-to-s3 (nonexistent) >@@ -1,41 +0,0 @@ >-#!/usr/bin/env python >-import argparse >-import boto3 >-import os >-import os.path >-import sys >- >-S3_BUCKET = 'archives.webkit.org' >-S3_MINIFIED_BUCKET = 'minified-archives.webkit.org' >-S3_REGION_PREFIX = 'https://s3-us-west-2.amazonaws.com' >- >-def uploadToS3(archive_path, bucket, identifier, revision): >- print 'Transferring {} to S3...'.format(archive_path) >- key = '/'.join([identifier, revision + '.zip']) >- print '\tS3 Bucket: {}\n\tS3 Key: {}'.format(bucket, key) >- s3.upload_file(archive_path, bucket, key) >- print('\tS3 URL: {}/{}/{}'.format(S3_REGION_PREFIX, bucket, key)) >- >-def archiveExists(archive): >- if archive: >- if os.path.exists(archive): >- return True >- else: >- print 'WARNING: Archive does not exist: {}'.format(archive) >- return False >- >-parser = argparse.ArgumentParser(add_help=True) >-parser.add_argument('--revision', action="store", required=True, help='Revision number for the built archive') >-parser.add_argument('--identifier', action="store", required=True, help='S3 destination identifier, in the form of fullPlatform-architecture-configuration. [mac-sierra-x86_64-release]') >-parser.add_argument('--archive', action="store", required=True, help='Path to the full size archive. [path/to/123456.zip]') >-args = parser.parse_args() >- >- >-head, tail = os.path.split(str(args.archive)) >-minifiedArchive = head + '/minified-' + tail >-s3 = boto3.client('s3') >- >-if archiveExists(args.archive): >- uploadToS3(args.archive, S3_BUCKET, args.identifier, args.revision) >-if archiveExists(minifiedArchive): >- uploadToS3(minifiedArchive, S3_MINIFIED_BUCKET, args.identifier, args.revision) >\ No newline at end of file
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Formatted Diff
|
Diff
Attachments on
bug 197914
:
369949
|
369970
|
369977
|
369991