WebKit Bugzilla
Attachment 357273 Details for
Bug 191557
: Extend commits table to contain testability warning information.
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Requests
|
Help
|
New Account
|
Log In
Remember
[x]
|
Forgot Password
Login:
[x]
[patch]
Patch
bug-191557-20181213170551.patch (text/plain), 60.42 KB, created by
dewei_zhu
on 2018-12-13 17:05:52 PST
(
hide
)
Description:
Patch
Filename:
MIME Type:
Creator:
dewei_zhu
Created:
2018-12-13 17:05:52 PST
Size:
60.42 KB
patch
obsolete
>Subversion Revision: 239135 >diff --git a/Websites/perf.webkit.org/ChangeLog b/Websites/perf.webkit.org/ChangeLog >index d06dc0cadaa935ed13879d5d3ac841a8f7c2a029..6fc73320f1916708eb14700791613a1a85ed79a9 100644 >--- a/Websites/perf.webkit.org/ChangeLog >+++ b/Websites/perf.webkit.org/ChangeLog >@@ -1,3 +1,48 @@ >+2018-12-13 Dewei Zhu <dewei_zhu@apple.com> >+ >+ Extend commits table to contain testability warning information. >+ https://bugs.webkit.org/show_bug.cgi?id=191557 >+ >+ Reviewed by NOBODY (OOPS!). >+ >+ Added the ability to store testability warning message in commits table. >+ Added '/api/update-commits' to support updating an existing commit. >+ Update os version syncing script to be able to update testability warning information. >+ >+ * init-database.sql: Added 'commit_testability_warning' field to 'commits' table. >+ * public/api/report-commits.php: Refactor to share code with '/api/update-commits'. >+ * public/api/update-commits.php: Added an API to update commit info. >+ * public/include/commit-log-fetcher.php: Expose testability warning information in this API. >+ * public/include/json-header.php: Move code from '/api/report-commits.php' to this file so that >+ it can be shared with '/api/update-commits.php'.git >+ * public/v3/models/commit-log.js: CommitLog object should expose testability warning information. >+ (CommitLog.prototype.updateSingleton): >+ (CommitLog.prototype.testabilityWarning): >+ * public/v3/models/commit-set.js: Added 'commitsWithTestabilityWarnings' and 'commits' to CommitSet and >+ IntermediateCommitSet instances. >+ (CommitSet.prototype.commitsWithTestabilityWarnings): >+ (CommitSet.prototype.commits): >+ (IntermediateCommitSet.prototype.commitsWithTestabilityWarnings): >+ (IntermediateCommitSet.prototype.commits): >+ * server-tests/api-update-commits.js: Added unit tests for '/api/update-commit-testability'. >+ * server-tests/tools-os-build-fetcher-tests.js: Added and updated unit tests. >+ * tools/js/os-build-fetcher.js: Added the ability to update commit testability warnings. >+ (async.fetchReportAndUpdateCommits): >+ (prototype._splitArrayByLimit): >+ (prototype.async.fetchReportAndUpdateBuilds): >+ (prototype.async._fetchAvailableBuilds): >+ (prototype.async._commitsForAvailableBuilds): >+ (prototype._commitsWithinRange): >+ (prototype.async._reportToEndPoint): >+ (fetchAndReportAllInOrder): Deleted. >+ (prototype.fetchAndReportNewBuilds): Deleted. >+ (prototype._fetchAvailableBuilds): Deleted. >+ (prototype._commitsForAvailableBuilds): Deleted. >+ (prototype._submitCommits): Deleted. >+ * tools/sync-os-versions.js: >+ * unit-tests/commit-log-tests.js: Added a unit test for 'testabilityWarning'. >+ * unit-tests/commit-set-tests.js: Added unit tests for 'commitsWithTestabilityWarnings'. >+ > 2018-11-16 Ryosuke Niwa <rniwa@webkit.org> > > Manifest file can contain a test metric which references a non-existent test >diff --git a/Websites/perf.webkit.org/init-database.sql b/Websites/perf.webkit.org/init-database.sql >index 8161efe3e281e34a0dc44edb80cab6c3bbc37ba6..2ee17ad4c92bf52567332c601ff2e2b14dd4f377 100644 >--- a/Websites/perf.webkit.org/init-database.sql >+++ b/Websites/perf.webkit.org/init-database.sql >@@ -101,6 +101,7 @@ CREATE TABLE commits ( > commit_committer integer REFERENCES committers ON DELETE CASCADE, > commit_message text, > commit_reported boolean NOT NULL DEFAULT FALSE, >+ commit_testability_warning varchar(64) DEFAULT NULL, > CONSTRAINT commit_in_repository_must_be_unique UNIQUE(commit_repository, commit_revision)); > CREATE INDEX commit_time_index ON commits(commit_time); > CREATE INDEX commit_order_index ON commits(commit_order); >diff --git a/Websites/perf.webkit.org/public/api/report-commits.php b/Websites/perf.webkit.org/public/api/report-commits.php >index 6c651a98aa4ff77ff290a2823a3afb78bc92145b..c77d752e1b2a9e9e6da6cf9ee30f75b394a5f065 100644 >--- a/Websites/perf.webkit.org/public/api/report-commits.php >+++ b/Websites/perf.webkit.org/public/api/report-commits.php >@@ -4,25 +4,7 @@ require('../include/json-header.php'); > > function main($post_data) > { >- $db = new Database; >- if (!$db->connect()) >- exit_with_error('DatabaseConnectionFailure'); >- >- $report = json_decode($post_data, true); >- >- verify_slave($db, $report); >- >- $commits = array_get($report, 'commits', array()); >- >- foreach ($commits as $commit_info) { >- if (!array_key_exists('repository', $commit_info)) >- exit_with_error('MissingRepositoryName', array('commit' => $commit_info)); >- if (!array_key_exists('revision', $commit_info)) >- exit_with_error('MissingRevision', array('commit' => $commit_info)); >- require_format('Revision', $commit_info['revision'], '/^[A-Za-z0-9 \.]+$/'); >- if (array_key_exists('author', $commit_info) && !is_array($commit_info['author'])) >- exit_with_error('InvalidAuthorFormat', array('commit' => $commit_info)); >- } >+ list($db, $commits) = connect_database_and_sanitize_commits($post_data); > > $db->begin_transaction(); > foreach ($commits as $commit_info) { >@@ -56,31 +38,10 @@ function main($post_data) > function insert_commit($db, $commit_info, $repository_id, $owner_commit_id) > { > $author = array_get($commit_info, 'author'); >- $committer_id = NULL; >- if ($author) { >- $account = array_get($author, 'account'); >- $committer_query = array('repository' => $repository_id, 'account' => $account); >- $committer_data = $committer_query; >- $name = array_get($author, 'name'); >- if ($name) >- $committer_data['name'] = $name; >- $committer_id = $db->update_or_insert_row('committers', 'committer', $committer_query, $committer_data); >- if (!$committer_id) { >- $db->rollback_transaction(); >- exit_with_error('FailedToInsertCommitter', array('committer' => $committer_data)); >- } >- } >+ $committer_id = $author ? find_or_update_author_for_repository($db, $author, $repository_id) : NULL; > > $previous_commit_revision = array_get($commit_info, 'previousCommit'); >- $previous_commit_id = NULL; >- if ($previous_commit_revision) { >- $previous_commit = $db->select_first_row('commits', 'commit', array('repository' => $repository_id, 'revision' => $previous_commit_revision)); >- if (!$previous_commit) { >- $db->rollback_transaction(); >- exit_with_error('FailedToFindPreviousCommit', array('commit' => $commit_info)); >- } >- $previous_commit_id = $previous_commit['commit_id']; >- } >+ $previous_commit_id = $previous_commit_revision ? find_previous_commit($db, $repository_id, $previous_commit_revision, $commit_info) : NULL; > > $data = array( > 'repository' => $repository_id, >diff --git a/Websites/perf.webkit.org/public/api/update-commits.php b/Websites/perf.webkit.org/public/api/update-commits.php >new file mode 100644 >index 0000000000000000000000000000000000000000..2f48d37c293401bea2d585828192f18cedda3ce5 >--- /dev/null >+++ b/Websites/perf.webkit.org/public/api/update-commits.php >@@ -0,0 +1,65 @@ >+<?php >+ >+require('../include/json-header.php'); >+ >+function main($post_data) >+{ >+ list($db, $commits) = connect_database_and_sanitize_commits($post_data); >+ >+ $db->begin_transaction(); >+ foreach ($commits as $commit_info) { >+ $repository = $db->select_first_row('repositories', 'repository', array('name' => $commit_info['repository'], 'owner' => NULL)); >+ if (!$repository) { >+ $db->rollback_transaction(); >+ exit_with_error('InvalidRepository', array('commit' => $commit_info)); >+ } >+ >+ $repository_id = $repository['repository_id']; >+ $commit_update = array(); >+ foreach($commit_info as $key => $value) { >+ switch ($key) { >+ case "repository": >+ case "revision": >+ break; >+ case "time": >+ case "message": >+ case "order": >+ $commit_update[$key] = $value; >+ break; >+ case "testabilityWarning": >+ $commit_update['testability_warning'] = $value; >+ break; >+ case "author": >+ $commit_update['committer'] = find_or_update_author_for_repository($db, $value, $repository_id); >+ break; >+ case "previousCommit": >+ $commit_update['previous_commit'] = find_previous_commit($db, $repository_id, $value, $commit_info); >+ break; >+ case "ownedCommits": >+ $db->rollback_transaction(); >+ exit_with_error('OwnedCommitsUpdateNotSupportedYet', array('commit' => $commit_info)); >+ break; >+ default: >+ $db->rollback_transaction(); >+ exit_with_error('UnrecognizedKey', array('key' => $key)); >+ } >+ } >+ >+ if (!count($commit_update)) { >+ $db->rollback_transaction(); >+ exit_with_error('NothingToUpdate', array('commit' => $commit_info)); >+ } >+ >+ $updated_commit = $db->update_row('commits', 'commit', array('revision' => $commit_info['revision'], 'repository' => $repository_id), >+ $commit_update); >+ if (!$updated_commit) { >+ $db->rollback_transaction(); >+ exit_with_error('FailedToUpdateCommit', array('commit' => $commit_info)); >+ } >+ } >+ $db->commit_transaction(); >+ exit_with_success(); >+} >+ >+main(file_get_contents('php://input')); >+?> >\ No newline at end of file >diff --git a/Websites/perf.webkit.org/public/include/commit-log-fetcher.php b/Websites/perf.webkit.org/public/include/commit-log-fetcher.php >index 9c76263cb6616b595b1dddfb9909472ea1589d08..26476d0d90d1a1840cb310877b7e793e4a004100 100644 >--- a/Websites/perf.webkit.org/public/include/commit-log-fetcher.php >+++ b/Websites/perf.webkit.org/public/include/commit-log-fetcher.php >@@ -62,6 +62,7 @@ class CommitLogFetcher { > commit_repository as "repository", > commit_message as "message", > commit_order as "order", >+ commit_testability_warning as "testabilityWarning", > EXISTS(SELECT * FROM commit_ownerships WHERE commit_owner = commit_id) as "ownsCommits" > FROM commits LEFT OUTER JOIN committers ON commit_committer = committer_id > WHERE commit_repository = $1 AND commit_reported = true'; >@@ -211,6 +212,7 @@ class CommitLogFetcher { > 'authorName' => $committer_row ? $committer_row['committer_name'] : null, > 'authorEmail' => $committer_row ? $committer_row['committer_account'] : null, > 'message' => $commit_row['commit_message'], >+ 'testabilityWarning' => $commit_row['commit_testability_warning'], > 'ownsCommits' => $owns_commits > ); > } >diff --git a/Websites/perf.webkit.org/public/include/json-header.php b/Websites/perf.webkit.org/public/include/json-header.php >index a27641c919d428dadb0388d6bac4275e24db8c77..c22994eaa914b6d1bb0201940c56d705512142ab 100644 >--- a/Websites/perf.webkit.org/public/include/json-header.php >+++ b/Websites/perf.webkit.org/public/include/json-header.php >@@ -200,4 +200,54 @@ function find_triggerable_for_task($db, $task_id) { > return NULL; > } > >+function connect_database_and_sanitize_commits($post_data) >+{ >+ $db = new Database; >+ if (!$db->connect()) >+ exit_with_error('DatabaseConnectionFailure'); >+ >+ $report = json_decode($post_data, true); >+ >+ verify_slave($db, $report); >+ >+ $commits = array_get($report, 'commits', array()); >+ >+ foreach ($commits as $commit_info) { >+ if (!array_key_exists('repository', $commit_info)) >+ exit_with_error('MissingRepositoryName', array('commit' => $commit_info)); >+ if (!array_key_exists('revision', $commit_info)) >+ exit_with_error('MissingRevision', array('commit' => $commit_info)); >+ require_format('Revision', $commit_info['revision'], '/^[A-Za-z0-9 \.]+$/'); >+ if (array_key_exists('author', $commit_info) && !is_array($commit_info['author'])) >+ exit_with_error('InvalidAuthorFormat', array('commit' => $commit_info)); >+ } >+ >+ return array($db, $commits); >+} >+ >+function find_or_update_author_for_repository($db, $author, $repository_id) >+{ >+ $account = array_get($author, 'account'); >+ $committer_query = array('repository' => $repository_id, 'account' => $account); >+ $committer_data = $committer_query; >+ $name = array_get($author, 'name'); >+ if ($name) >+ $committer_data['name'] = $name; >+ $committer_id = $db->update_or_insert_row('committers', 'committer', $committer_query, $committer_data); >+ if (!$committer_id) { >+ $db->rollback_transaction(); >+ exit_with_error('FailedToInsertCommitter', array('committer' => $committer_data)); >+ } >+ return $committer_id; >+} >+ >+function find_previous_commit($db, $repository_id, $previous_commit_revision, $commit_info) >+{ >+ $previous_commit = $db->select_first_row('commits', 'commit', array('repository' => $repository_id, 'revision' => $previous_commit_revision)); >+ if (!$previous_commit) { >+ $db->rollback_transaction(); >+ exit_with_error('FailedToFindPreviousCommit', array('commit' => $commit_info)); >+ } >+ return $previous_commit['commit_id']; >+} > ?> >diff --git a/Websites/perf.webkit.org/public/v3/models/commit-log.js b/Websites/perf.webkit.org/public/v3/models/commit-log.js >index 130cb90f1498aebe309907f43c5535e4703bec38..2fa2ea0694688b89649fabcdaa20f2ac8d6fc0ca 100644 >--- a/Websites/perf.webkit.org/public/v3/models/commit-log.js >+++ b/Websites/perf.webkit.org/public/v3/models/commit-log.js >@@ -29,11 +29,14 @@ class CommitLog extends DataModelObject { > this._rawData.ownsCommits = rawData.ownsCommits; > if (rawData.order) > this._rawData.order = rawData.order; >+ if (rawData.testabilityWarning) >+ this._rawData.testabilityWarning = rawData.testabilityWarning; > } > > repository() { return this._repository; } > time() { return new Date(this._rawData['time']); } > hasCommitTime() { return this._rawData['time'] > 0 && this._rawData['time'] != null; } >+ testabilityWarning() { return this._rawData['testabilityWarning']; } > author() { return this._rawData['authorName']; } > revision() { return this._rawData['revision']; } > message() { return this._rawData['message']; } >diff --git a/Websites/perf.webkit.org/public/v3/models/commit-set.js b/Websites/perf.webkit.org/public/v3/models/commit-set.js >index 94acc5332898cb4900afe78b7f6cb6979fc33516..45946c887e24f739955a8b52bbe5db7df17a4588 100644 >--- a/Websites/perf.webkit.org/public/v3/models/commit-set.js >+++ b/Websites/perf.webkit.org/public/v3/models/commit-set.js >@@ -71,6 +71,8 @@ class CommitSet extends DataModelObject { > ownerCommitForRepository(repository) { return this._repositoryToCommitOwnerMap.get(repository); } > topLevelRepositories() { return Repository.sortByNamePreferringOnesWithURL(this._repositories.filter((repository) => !this.ownerRevisionForRepository(repository))); } > ownedRepositoriesForOwnerRepository(repository) { return this._ownerRepositoryToOwnedRepositoriesMap.get(repository); } >+ commitsWithTestabilityWarnings() { return this.commits().filter((commit) => !!commit.testabilityWarning()); } >+ commits() { return Array.from(this._repositoryToCommitMap.values()); } > > revisionForRepository(repository) > { >@@ -387,6 +389,9 @@ class IntermediateCommitSet { > return Promise.all(fetchingPromises); > } > >+ commitsWithTestabilityWarnings() { return this.commits().filter((commit) => !!commit.testabilityWarning()); } >+ commits() { return Array.from(this._commitByRepository.values()); } >+ > _fetchCommitLogAndOwnedCommits(repository, revision) > { > return CommitLog.fetchForSingleRevision(repository, revision).then((commits) => { >diff --git a/Websites/perf.webkit.org/server-tests/api-update-commits.js b/Websites/perf.webkit.org/server-tests/api-update-commits.js >new file mode 100644 >index 0000000000000000000000000000000000000000..32846b0925f723d02d6733c7c7914ffe33b92102 >--- /dev/null >+++ b/Websites/perf.webkit.org/server-tests/api-update-commits.js >@@ -0,0 +1,179 @@ >+'use strict'; >+ >+const assert = require('assert'); >+ >+const TestServer = require('./resources/test-server.js'); >+const addSlaveForReport = require('./resources/common-operations.js').addSlaveForReport; >+const prepareServerTest = require('./resources/common-operations.js').prepareServerTest; >+const assertThrows = require('./resources/common-operations.js').assertThrows; >+ >+describe("/api/update-commits", function () { >+ prepareServerTest(this); >+ >+ const subversionCommits = { >+ "slaveName": "someSlave", >+ "slavePassword": "somePassword", >+ "commits": [ >+ { >+ "repository": "WebKit", >+ "revision": "210948", >+ "time": "2017-01-20T02:52:34.577Z", >+ "author": {"name": "Zalan Bujtas", "account": "zalan@apple.com"}, >+ "message": "a message", >+ }, >+ { >+ "repository": "WebKit", >+ "revision": "210949", >+ "time": "2017-01-20T03:23:50.645Z", >+ "author": {"name": "Chris Dumez", "account": "cdumez@apple.com"}, >+ "message": "some message", >+ } >+ ] >+ }; >+ >+ const commitsUpdate = { >+ "slaveName": "someSlave", >+ "slavePassword": "somePassword", >+ "commits": [ >+ { >+ "repository": "WebKit", >+ "revision": "210948", >+ "testabilityWarning": "Breaks builds", >+ "message": "another message", >+ "order": 210948, >+ "time": "2017-01-20T03:52:34.577Z", >+ "author": {"name": "Chris Dumez", "account": "cdumez@apple.com"}, >+ }, >+ { >+ "repository": "WebKit", >+ "revision": "210949", >+ "previousCommit": "210948", >+ "testabilityWarning": "Crashes WebKit", >+ "message": "another message", >+ "order": 210949, >+ "time": "2017-01-20T04:23:50.645Z", >+ "author": {"name": "Zalan Bujtas", "account": "zalan@apple.com"}, >+ } >+ ] >+ }; >+ >+ >+ const commitsUpdateWithMissingRevision = { >+ "slaveName": "someSlave", >+ "slavePassword": "somePassword", >+ "commits": [ >+ { >+ "repository": "WebKit", >+ "testabilityWarning": "Breaks builds" >+ } >+ ] >+ }; >+ >+ const commitsUpdateWithMissingRepository = { >+ "slaveName": "someSlave", >+ "slavePassword": "somePassword", >+ "commits": [ >+ { >+ "revision": "210948", >+ "testabilityWarning": "Breaks builds" >+ } >+ ] >+ }; >+ >+ const commitsUpdateWithoutAnyUpdate = { >+ "slaveName": "someSlave", >+ "slavePassword": "somePassword", >+ "commits": [ >+ { >+ "repository": "WebKit", >+ "revision": "210948" >+ } >+ ] >+ }; >+ >+ const commitsUpdateWithOwnedCommitsUpdate = { >+ "slaveName": "someSlave", >+ "slavePassword": "somePassword", >+ "commits": [ >+ { >+ "repository": "WebKit", >+ "revision": "210948", >+ "ownedCommits": [], >+ } >+ ] >+ }; >+ >+ const commitsUpdateWithUnrecognizedField = { >+ "slaveName": "someSlave", >+ "slavePassword": "somePassword", >+ "commits": [ >+ { >+ "repository": "WebKit", >+ "revision": "210948", >+ "unrecognized": "foo", >+ } >+ ] >+ }; >+ >+ async function setUpTestsWithExpectedStatus(commitsUpdate, expectedStatus) >+ { >+ await addSlaveForReport(subversionCommits); >+ await TestServer.remoteAPI().postJSON('/api/report-commits/', subversionCommits); >+ let result = await TestServer.remoteAPI().getJSON('/api/commits/WebKit/'); >+ assert.equal(result['status'], 'OK'); >+ let commits = result['commits']; >+ assert.equal(commits.length, 2); >+ assert.equal(commits[0].testabilityWarning, null); >+ assert.equal(commits[1].testabilityWarning, null); >+ >+ result = await TestServer.remoteAPI().postJSON('/api/update-commits', commitsUpdate); >+ assert.equal(result['status'], expectedStatus); >+ result = await TestServer.remoteAPI().getJSON('/api/commits/WebKit/'); >+ return result['commits']; >+ } >+ >+ async function testWithExpectedFailure(commitsUpdate, expectedFailureName) >+ { >+ const commits = await setUpTestsWithExpectedStatus(commitsUpdate, expectedFailureName); >+ assert.equal(commits.length, 2); >+ assert.equal(commits[0].testabilityWarning, null); >+ assert.equal(commits[1].testabilityWarning, null); >+ } >+ >+ it('should fail with "MissingRevision" if commit update does not have commit revision specified', async () => { >+ await testWithExpectedFailure(commitsUpdateWithMissingRevision, 'MissingRevision'); >+ }); >+ >+ it('should fail with "MissingRepositoryName" if commit update does not have commit revision specified', async () => { >+ await testWithExpectedFailure(commitsUpdateWithMissingRepository, 'MissingRepositoryName'); >+ }); >+ >+ it('should fail with "NothingToUpdate" if commit update does not have commit revision specified', async () => { >+ await testWithExpectedFailure(commitsUpdateWithoutAnyUpdate, 'NothingToUpdate'); >+ }); >+ >+ it('should fail with "OwnedCommitsUpdateNotSupportedYet" when trying to update owned commits info', async () => { >+ await testWithExpectedFailure(commitsUpdateWithOwnedCommitsUpdate, 'OwnedCommitsUpdateNotSupportedYet'); >+ }); >+ >+ it('should fail with "UnrecognizedKey" when trying to update an unrecognized commit field', async () => { >+ await testWithExpectedFailure(commitsUpdateWithUnrecognizedField, 'UnrecognizedKey'); >+ }); >+ >+ it("should be able to update commits message, time, order, author and previous commit", async () => { >+ const commits = await setUpTestsWithExpectedStatus(commitsUpdate, 'OK'); >+ >+ assert.equal(commits.length, 2); >+ assert.equal(commits[0].testabilityWarning, 'Breaks builds'); >+ assert.equal(commits[1].testabilityWarning, 'Crashes WebKit'); >+ assert.equal(commits[0].message, 'another message'); >+ assert.equal(commits[1].message, 'another message'); >+ assert.equal(commits[0].authorName, 'Chris Dumez'); >+ assert.equal(commits[1].authorName, 'Zalan Bujtas'); >+ assert.equal(commits[0].order, 210948); >+ assert.equal(commits[1].order, 210949); >+ assert.equal(commits[0].time, 1484884354577); >+ assert.equal(commits[1].time, 1484886230645); >+ assert.equal(commits[1].previousCommit, commits[0].id); >+ }); >+}); >\ No newline at end of file >diff --git a/Websites/perf.webkit.org/server-tests/tools-os-build-fetcher-tests.js b/Websites/perf.webkit.org/server-tests/tools-os-build-fetcher-tests.js >index c6f0dc93bbe3286878ca7f90c9c41ceff54ea779..ce9eafde3e37314b3600e41b73608f08aa7bad1c 100644 >--- a/Websites/perf.webkit.org/server-tests/tools-os-build-fetcher-tests.js >+++ b/Websites/perf.webkit.org/server-tests/tools-os-build-fetcher-tests.js >@@ -134,39 +134,41 @@ describe('OSBuildFetcher', function() { > }); > > describe('OSBuilderFetcher._commitsForAvailableBuilds', () => { >- it('should only return commits whose orders are higher than specified order', () => { >+ it('should parse the command output as JSON format', async () => { > const logger = new MockLogger; > const fetcher = new OSBuildFetcher({}, null, null, MockSubprocess, logger); > const waitForInvocationPromise = MockSubprocess.waitForInvocation(); >- const fetchCommitsPromise = fetcher._commitsForAvailableBuilds('OSX', ['list', 'build1'], '^\\.*$', 1604000000, 1606000000); >+ const fetchCommitsPromise = fetcher._commitsForAvailableBuilds(['list', 'build1']); > >- return waitForInvocationPromise.then(() => { >- assert.equal(MockSubprocess.invocations.length, 1); >- assert.deepEqual(MockSubprocess.invocations[0].command, ['list', 'build1']); >- MockSubprocess.invocations[0].resolve('16D321\n16E321z\n\n16F321'); >- return fetchCommitsPromise; >- }).then((results) => { >- assert.equal(results.length, 2); >- assert.deepEqual(results[0], {repository: 'OSX', order: 1604032126, revision: '16E321z'}); >- assert.deepEqual(results[1], {repository: 'OSX', order: 1605032100, revision: '16F321'}); >- }); >+ await waitForInvocationPromise; >+ assert.equal(MockSubprocess.invocations.length, 1); >+ assert.deepEqual(MockSubprocess.invocations[0].command, ['list', 'build1']); >+ >+ const outputObject = {allRevisions: ["16D321", "16E321z", "16F321"], commitsWithTestabilityWarnings: {"16D321": "Panic"}}; >+ await MockSubprocess.invocations[0].resolve(JSON.stringify(outputObject)); >+ const buildInfo = await fetchCommitsPromise; >+ assert.deepEqual(outputObject, buildInfo); > }); >+ }); >+ > >- it('should only return commits whose orders are higher than minOrder and lower than the maxOrder', () => { >+ describe('OSBuilderFetcher._commitsWithinRange', () => { >+ it('should only return commits whose orders are higher than specified order', async () => { > const logger = new MockLogger; > const fetcher = new OSBuildFetcher({}, null, null, MockSubprocess, logger); >- const waitForInvocationPromise = MockSubprocess.waitForInvocation(); >- const fetchCommitsPromise = fetcher._commitsForAvailableBuilds('OSX', ['list', 'build1'], '^\\.*$', 1604000000, 1605000000); >+ const results = fetcher._commitsWithinRange(["16D321", "16E321z", "16F321"], "OSX", 1604000000, 1606000000); >+ assert.equal(results.length, 2); >+ assert.deepEqual(results[0], {repository: 'OSX', order: 1604032126, revision: '16E321z'}); >+ assert.deepEqual(results[1], {repository: 'OSX', order: 1605032100, revision: '16F321'}); > >- return waitForInvocationPromise.then(() => { >- assert.equal(MockSubprocess.invocations.length, 1); >- assert.deepEqual(MockSubprocess.invocations[0].command, ['list', 'build1']); >- MockSubprocess.invocations[0].resolve('16D321\n16E321z\n\n16F321'); >- return fetchCommitsPromise; >- }).then((results) => { >- assert.equal(results.length, 1); >- assert.deepEqual(results[0], {repository: 'OSX', order: 1604032126, revision: '16E321z'}); >- }); >+ }); >+ >+ it('should only return commits whose orders are higher than minOrder and lower than the maxOrder', async () => { >+ const logger = new MockLogger; >+ const fetcher = new OSBuildFetcher({}, null, null, MockSubprocess, logger); >+ const results = fetcher._commitsWithinRange(["16D321", "16E321z", "16F321"], "OSX", 1604000000, 1605000000); >+ assert.equal(results.length, 1); >+ assert.deepEqual(results[0], {repository: 'OSX', order: 1604032126, revision: '16E321z'}); > }); > }); > >@@ -239,9 +241,9 @@ describe('OSBuildFetcher', function() { > }); > }); > }) >- }) >+ }); > >- describe('OSBuildFetcher.fetchAndReportNewBuilds', () => { >+ describe('OSBuildFetcher.fetchReportAndUpdateBuilds', () => { > const invocations = MockSubprocess.invocations; > > beforeEach(function () { >@@ -256,7 +258,10 @@ describe('OSBuildFetcher', function() { > const logger = new MockLogger; > const fetcher = new OSBuildFetcher(config, TestServer.remoteAPI(), slaveAuth, MockSubprocess, logger); > const db = TestServer.database(); >- let fetchAndReportPromise = null; >+ const resultsForSierraD = {allRevisions: ["Sierra16D68", "Sierra16D69"], commitsWithTestabilityWarnings: {}}; >+ const resultsForSierraE = {allRevisions: ["Sierra16E32", "Sierra16E33", "Sierra16E33h", "Sierra16E34"], commitsWithTestabilityWarnings: {}}; >+ >+ let fetchReportAndUpdateBuildsPromise = null; > let fetchAvailableBuildsPromise = null; > > return addSlaveForReport(emptyReport).then(() => { >@@ -285,7 +290,7 @@ describe('OSBuildFetcher', function() { > }).then(() => { > assert.equal(invocations.length, 1); > assert.deepEqual(invocations[0].command, ['list', 'all osx 16Dxx builds']); >- invocations[0].resolve('\n\nSierra16D68\nSierra16D69\n'); >+ invocations[0].resolve(JSON.stringify(resultsForSierraD)); > return MockSubprocess.resetAndWaitForInvocation(); > }).then(() => { > assert.equal(invocations.length, 1); >@@ -295,7 +300,7 @@ describe('OSBuildFetcher', function() { > }).then(() => { > assert.equal(invocations.length, 1); > assert.deepEqual(invocations[0].command, ['list', 'all osx 16Exx builds']); >- invocations[0].resolve('\n\nSierra16E32\nSierra16E33\nSierra16E33h\nSierra16E34'); >+ invocations[0].resolve(JSON.stringify(resultsForSierraE)); > return MockSubprocess.resetAndWaitForInvocation(); > }).then(() => { > assert.equal(invocations.length, 1); >@@ -307,15 +312,14 @@ describe('OSBuildFetcher', function() { > assert.deepEqual(invocations[0].command, ['list', 'ownedCommit', 'for', 'revision', 'Sierra16E34']); > invocations[0].resolve(JSON.stringify(anotherownedCommitWithWebKitAndJavaScriptCore)); > return fetchAvailableBuildsPromise; >- }).then((results) => { >- assert.equal(results.length, 3); >+ }).then(() => { > MockSubprocess.reset(); >- fetchAndReportPromise = fetcher.fetchAndReportNewBuilds(); >+ fetchReportAndUpdateBuildsPromise = fetcher.fetchReportAndUpdateBuilds(); > return MockSubprocess.waitForInvocation(); > }).then(() => { > assert.equal(invocations.length, 1); > assert.deepEqual(invocations[0].command, ['list', 'all osx 16Dxx builds']); >- invocations[0].resolve('\n\nSierra16D68\nSierra16D69\n'); >+ invocations[0].resolve(JSON.stringify(resultsForSierraD)); > return MockSubprocess.resetAndWaitForInvocation(); > }).then(() => { > assert.equal(invocations.length, 1); >@@ -325,7 +329,7 @@ describe('OSBuildFetcher', function() { > }).then(() => { > assert.equal(invocations.length, 1); > assert.deepEqual(invocations[0].command, ['list', 'all osx 16Exx builds']); >- invocations[0].resolve('\n\nSierra16E32\nSierra16E33\nSierra16E33h\nSierra16E34'); >+ invocations[0].resolve(JSON.stringify(resultsForSierraE)); > return MockSubprocess.resetAndWaitForInvocation(); > }).then(() => { > assert.equal(invocations.length, 1); >@@ -337,9 +341,8 @@ describe('OSBuildFetcher', function() { > invocations[0].resolve(JSON.stringify(anotherownedCommitWithWebKitAndJavaScriptCore)); > assert.deepEqual(invocations[0].command, ['list', 'ownedCommit', 'for', 'revision', 'Sierra16E34']); > >- return fetchAndReportPromise; >- }).then((result) => { >- assert.equal(result['status'], 'OK'); >+ return fetchReportAndUpdateBuildsPromise; >+ }).then(() => { > return Promise.all([ > db.selectRows('repositories', {'name': 'WebKit'}), > db.selectRows('repositories', {'name': 'JavaScriptCore'}), >@@ -384,11 +387,119 @@ describe('OSBuildFetcher', function() { > }); > }); > >+ it('should update testability warning for commits', async () => { >+ const logger = new MockLogger; >+ const fetcher = new OSBuildFetcher(config, TestServer.remoteAPI(), slaveAuth, MockSubprocess, logger); >+ const db = TestServer.database(); >+ const resultsForSierraD = {allRevisions: ["Sierra16D68", "Sierra16D69"], commitsWithTestabilityWarnings: {"Sierra16D68": "Panic", "Sierra16D69": "Spin CPU"}}; >+ const resultsForSierraE = {allRevisions: ["Sierra16E32", "Sierra16E33", "Sierra16E33h", "Sierra16E34"], commitsWithTestabilityWarnings: {"Sierra16E31": "WebKit crashes"}}; >+ >+ await addSlaveForReport(emptyReport); >+ >+ await Promise.all([ >+ db.insert('repositories', {'id': 10, 'name': 'OSX'}), >+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16D67', 'order': 1603006700, 'reported': true}), >+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16D68', 'order': 1603006800, 'reported': true}), >+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16D69', 'order': 1603006900, 'reported': false}), >+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16E31', 'order': 1604003100, 'reported': true}), >+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16E32', 'order': 1604003200, 'reported': true}), >+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16E33', 'order': 1604003300, 'reported': true}), >+ db.insert('commits', {'repository': 10, 'revision': 'Sierra16E33g', 'order': 1604003307, 'reported': true})]); >+ >+ let result = await TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=1603000000&to=1603099900'); >+ >+ assert.equal(result['commits'].length, 1); >+ assert.equal(result['commits'][0]['revision'], 'Sierra16D68'); >+ assert.equal(result['commits'][0]['order'], 1603006800); >+ result = await TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=1604000000&to=1604099900'); >+ >+ assert.equal(result['commits'].length, 1); >+ assert.equal(result['commits'][0]['revision'], 'Sierra16E33g'); >+ assert.equal(result['commits'][0]['order'], 1604003307); >+ >+ const fetchReportAndUpdatePromise = fetcher.fetchReportAndUpdateBuilds(); >+ await MockSubprocess.waitForInvocation(); >+ >+ assert.equal(invocations.length, 1); >+ assert.deepEqual(invocations[0].command, ['list', 'all osx 16Dxx builds']); >+ invocations[0].resolve(JSON.stringify(resultsForSierraD)); >+ await MockSubprocess.resetAndWaitForInvocation(); >+ >+ assert.equal(invocations.length, 1); >+ assert.deepEqual(invocations[0].command, ['list', 'ownedCommit', 'for', 'revision', 'Sierra16D69']); >+ invocations[0].resolve(JSON.stringify(ownedCommitWithWebKit)); >+ await MockSubprocess.resetAndWaitForInvocation(); >+ >+ assert.equal(invocations.length, 1); >+ assert.deepEqual(invocations[0].command, ['list', 'all osx 16Exx builds']); >+ invocations[0].resolve(JSON.stringify(resultsForSierraE)); >+ >+ await MockSubprocess.resetAndWaitForInvocation(); >+ >+ assert.equal(invocations.length, 1); >+ assert.deepEqual(invocations[0].command, ['list', 'ownedCommit', 'for', 'revision', 'Sierra16E33h']); >+ invocations[0].resolve(JSON.stringify(anotherownedCommitWithWebKit)); >+ await MockSubprocess.resetAndWaitForInvocation(); >+ assert.equal(invocations.length, 1); >+ invocations[0].resolve(JSON.stringify(anotherownedCommitWithWebKitAndJavaScriptCore)); >+ assert.deepEqual(invocations[0].command, ['list', 'ownedCommit', 'for', 'revision', 'Sierra16E34']); >+ >+ await fetchReportAndUpdatePromise; >+ >+ const webkitRepository = await db.selectRows('repositories', {'name': 'WebKit'}); >+ const jscRepository = await db.selectRows('repositories', {'name': 'JavaScriptCore'}); >+ const osxCommit16D68 = await db.selectRows('commits', {'revision': 'Sierra16D68'}); >+ const osxCommit16D69 = await db.selectRows('commits', {'revision': 'Sierra16D69'}); >+ const osxCommit16E31 = await db.selectRows('commits', {'revision': 'Sierra16E31'}); >+ const osxCommit16E33h = await db.selectRows('commits', {'revision': 'Sierra16E33h'}); >+ const osxCommit16E34 = await db.selectRows('commits', {'revision': 'Sierra16E34'}); >+ >+ assert.equal(webkitRepository.length, 1); >+ assert.equal(webkitRepository[0]['owner'], 10); >+ assert.equal(jscRepository.length, 1); >+ assert.equal(jscRepository[0]['owner'], 10); >+ >+ assert.equal(osxCommit16D68.length, 1); >+ assert.equal(osxCommit16D68[0]['repository'], 10); >+ assert.equal(osxCommit16D68[0]['order'], 1603006800); >+ assert.equal(osxCommit16D68[0]['testability_warning'], "Panic"); >+ >+ assert.equal(osxCommit16D69.length, 1); >+ assert.equal(osxCommit16D69[0]['repository'], 10); >+ assert.equal(osxCommit16D69[0]['order'], 1603006900); >+ assert.equal(osxCommit16D69[0]['testability_warning'], "Spin CPU"); >+ >+ assert.equal(osxCommit16E31.length, 1); >+ assert.equal(osxCommit16E31[0]['repository'], 10); >+ assert.equal(osxCommit16E31[0]['order'], 1604003100); >+ assert.equal(osxCommit16E31[0]['testability_warning'], "WebKit crashes"); >+ >+ assert.equal(osxCommit16E33h.length, 1); >+ assert.equal(osxCommit16E33h[0]['repository'], 10); >+ assert.equal(osxCommit16E33h[0]['order'], 1604003308); >+ >+ assert.equal(osxCommit16E34.length, 1); >+ assert.equal(osxCommit16E34[0]['repository'], 10); >+ assert.equal(osxCommit16E34[0]['order'], 1604003400); >+ >+ result = await TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=1603000000&to=1603099900'); >+ assert.equal(result['commits'].length, 1); >+ assert.equal(result['commits'][0]['revision'], 'Sierra16D69'); >+ assert.equal(result['commits'][0]['order'], 1603006900); >+ >+ result = await TestServer.remoteAPI().getJSON('/api/commits/OSX/last-reported?from=1604000000&to=1604099900'); >+ assert.equal(result['commits'].length, 1); >+ assert.equal(result['commits'][0]['revision'], 'Sierra16E34'); >+ assert.equal(result['commits'][0]['order'], 1604003400); >+ }); >+ > it('should report commits without owned-commits if "ownedCommitCommand" is not specified in config', async () => { > > const logger = new MockLogger; > const fetcher = new OSBuildFetcher(configWithoutOwnedCommitCommand, TestServer.remoteAPI(), slaveAuth, MockSubprocess, logger); > const db = TestServer.database(); >+ const resultsForSierraD = {allRevisions: ["Sierra16D68", "Sierra16D69"], commitsWithTestabilityWarnings: {}}; >+ const resultsForSierraE = {allRevisions: ["Sierra16E32", "Sierra16E33", "Sierra16E33h", "Sierra16E34"], commitsWithTestabilityWarnings: {}}; > > await addSlaveForReport(emptyReport); > await Promise.all([ >@@ -411,19 +522,18 @@ describe('OSBuildFetcher', function() { > assert.equal(result['commits'][0]['order'], 1604003307); > > const waitForInvocationPromise = MockSubprocess.waitForInvocation(); >- const fetchAndReportPromise = fetcher.fetchAndReportNewBuilds(); >+ const fetchReportAndUpdatePromise = fetcher.fetchReportAndUpdateBuilds(); > await waitForInvocationPromise; > assert.equal(invocations.length, 1); > assert.deepEqual(invocations[0].command, ['list', 'all osx 16Dxx builds']); >- invocations[0].resolve('\n\nSierra16D68\nSierra16D69'); >+ invocations[0].resolve(JSON.stringify(resultsForSierraD)); > > await MockSubprocess.resetAndWaitForInvocation(); > assert.equal(invocations.length, 1); > assert.deepEqual(invocations[0].command, ['list', 'all osx 16Exx builds']); >- invocations[0].resolve('\n\nSierra16E32\nSierra16E33\nSierra16E33h\nSierra16E34'); >+ invocations[0].resolve(JSON.stringify(resultsForSierraE)); > >- result = await fetchAndReportPromise; >- assert.equal(result['status'], 'OK'); >+ result = await fetchReportAndUpdatePromise; > const results = await Promise.all([ > db.selectRows('repositories', {'name': 'WebKit'}), > db.selectRows('repositories', {'name': 'JavaScriptCore'}), >@@ -467,6 +577,8 @@ describe('OSBuildFetcher', function() { > const logger = new MockLogger; > const fetcher = new OSBuildFetcher(configWithoutOwnedCommitCommand, TestServer.remoteAPI(), slaveAuth, MockSubprocess, logger); > const db = TestServer.database(); >+ const resultsForSierraD = {allRevisions: ["Sierra16D68", "Sierra16D69", "Sierra16D10000"], commitsWithTestabilityWarnings: {}}; >+ const resultsForSierraE = {allRevisions: ["Sierra16E32", "Sierra16E33", "Sierra16E33h", "Sierra16E34", "Sierra16E10000"], commitsWithTestabilityWarnings: {}}; > > await addSlaveForReport(emptyReport); > await Promise.all([ >@@ -488,20 +600,19 @@ describe('OSBuildFetcher', function() { > assert.equal(result['commits'][0]['revision'], 'Sierra16E33g'); > assert.equal(result['commits'][0]['order'], 1604003307); > const waitForInvocationPromise = MockSubprocess.waitForInvocation(); >- const fetchAndReportPromise = fetcher.fetchAndReportNewBuilds(); >+ const fetchReportAndUpdatePromise = fetcher.fetchReportAndUpdateBuilds(); > > await waitForInvocationPromise; > assert.equal(invocations.length, 1); > assert.deepEqual(invocations[0].command, ['list', 'all osx 16Dxx builds']); >- invocations[0].resolve('\n\nSierra16D68\nSierra16D69\nSierra16D10000'); >+ invocations[0].resolve(JSON.stringify(resultsForSierraD)); > > await MockSubprocess.resetAndWaitForInvocation(); > assert.equal(invocations.length, 1); > assert.deepEqual(invocations[0].command, ['list', 'all osx 16Exx builds']); >- invocations[0].resolve('\n\nSierra16E32\nSierra16E33\nSierra16E33h\nSierra16E34\nSierra16E10000'); >+ invocations[0].resolve(JSON.stringify(resultsForSierraE)); > >- result = await fetchAndReportPromise; >- assert.equal(result['status'], 'OK'); >+ result = await fetchReportAndUpdatePromise; > const results = await Promise.all([ > db.selectRows('repositories', {'name': 'WebKit'}), > db.selectRows('repositories', {'name': 'JavaScriptCore'}), >@@ -545,6 +656,7 @@ describe('OSBuildFetcher', function() { > const logger = new MockLogger; > const fetcher = new OSBuildFetcher(configTrackingOneOS, TestServer.remoteAPI(), slaveAuth, MockSubprocess, logger); > const db = TestServer.database(); >+ const resultsForSierraD = {allRevisions: ["Sierra16D68", "Sierra16D69", "Sierra16D100", "Sierra16D100a"], commitsWithTestabilityWarnings: {}}; > > await addSlaveForReport(emptyReport); > await db.insert('repositories', {'id': 10, 'name': 'OSX'}); >@@ -556,14 +668,13 @@ describe('OSBuildFetcher', function() { > assert.equal(result['commits'][0]['order'], 1603010000); > > const waitForInvocationPromise = MockSubprocess.waitForInvocation(); >- const fetchAndReportPromise = fetcher.fetchAndReportNewBuilds(); >+ const fetchAndReportPromise = fetcher.fetchReportAndUpdateBuilds(); > await waitForInvocationPromise; > assert.equal(invocations.length, 1); > assert.deepEqual(invocations[0].command, ['list', 'all osx 16Dxx builds']); >- invocations[0].resolve('\n\nSierra16D68\nSierra16D69\nSierra16D100\nSierra16D100a\n'); >+ invocations[0].resolve(JSON.stringify(resultsForSierraD)); > > result = await fetchAndReportPromise; >- assert.equal(result['status'], 'OK'); > const results = await Promise.all([ > db.selectRows('repositories', {'name': 'WebKit'}), > db.selectRows('repositories', {'name': 'JavaScriptCore'}), >@@ -594,6 +705,7 @@ describe('OSBuildFetcher', function() { > const logger = new MockLogger; > const fetcher = new OSBuildFetcher(configTrackingOneOS, TestServer.remoteAPI(), slaveAuth, MockSubprocess, logger); > const db = TestServer.database(); >+ const resultsForSierraD = {allRevisions: ["Sierra16D68", "Sierra16D69", "Sierra16D100", "Sierra16D101"], commitsWithTestabilityWarnings: {}}; > > await addSlaveForReport(emptyReport); > await db.insert('repositories', {'id': 10, 'name': 'OSX'}); >@@ -602,14 +714,13 @@ describe('OSBuildFetcher', function() { > assert.equal(result['commits'].length, 0); > > const waitForInvocationPromise = MockSubprocess.waitForInvocation(); >- const fetchAndReportPromise = fetcher.fetchAndReportNewBuilds(); >+ const fetchReportAndUpdatePromise = fetcher.fetchReportAndUpdateBuilds(); > await waitForInvocationPromise; > assert.equal(invocations.length, 1); > assert.deepEqual(invocations[0].command, ['list', 'all osx 16Dxx builds']); >- invocations[0].resolve('\n\nSierra16D68\nSierra16D69\nSierra16D100\nSierra16D101\n'); >+ invocations[0].resolve(JSON.stringify(resultsForSierraD)); > >- result = await fetchAndReportPromise; >- assert.equal(result['status'], 'OK'); >+ result = await fetchReportAndUpdatePromise; > const results = await Promise.all([ > db.selectRows('repositories', {'name': 'WebKit'}), > db.selectRows('repositories', {'name': 'JavaScriptCore'}), >@@ -646,6 +757,8 @@ describe('OSBuildFetcher', function() { > const logger = new MockLogger; > const fetcher = new OSBuildFetcher(config, TestServer.remoteAPI(), slaveAuth, MockSubprocess, logger); > const db = TestServer.database(); >+ const resultsForSierraD = {allRevisions: ["Sierra16D68", "Sierra16D69"], commitsWithTestabilityWarnings: {}}; >+ const resultsForSierraE = {allRevisions: ["Sierra16E32", "Sierra16E33", "Sierra16E33h", "Sierra16E34"], commitsWithTestabilityWarnings: {}}; > let fetchAndReportPromise = null; > > return addSlaveForReport(emptyReport).then(() => { >@@ -671,12 +784,12 @@ describe('OSBuildFetcher', function() { > assert.equal(result['commits'][0]['order'], 1604003307); > > const waitForInvocationPromise = MockSubprocess.waitForInvocation(); >- fetchAndReportPromise = fetcher.fetchAndReportNewBuilds(); >+ fetchAndReportPromise = fetcher.fetchReportAndUpdateBuilds(); > return waitForInvocationPromise; > }).then(() => { > assert.equal(invocations.length, 1); > assert.deepEqual(invocations[0].command, ['list', 'all osx 16Dxx builds']); >- invocations[0].resolve('\n\nSierra16D68\nSierra16D69\n'); >+ invocations[0].resolve(JSON.stringify(resultsForSierraD)); > return MockSubprocess.resetAndWaitForInvocation(); > }).then(() => { > assert.equal(invocations.length, 1); >@@ -686,7 +799,7 @@ describe('OSBuildFetcher', function() { > }).then(() => { > assert.equal(invocations.length, 1); > assert.deepEqual(invocations[0].command, ['list', 'all osx 16Exx builds']); >- invocations[0].resolve('\n\nSierra16E32\nSierra16E33\nSierra16E33h\nSierra16E34'); >+ invocations[0].resolve(JSON.stringify(resultsForSierraE)); > return MockSubprocess.resetAndWaitForInvocation(); > }).then(() => { > assert.deepEqual(invocations[0].command, ['list', 'ownedCommit', 'for', 'revision', 'Sierra16E33h']); >diff --git a/Websites/perf.webkit.org/tools/js/os-build-fetcher.js b/Websites/perf.webkit.org/tools/js/os-build-fetcher.js >index 051c29fcb7b86336bf4172d55f7a4986d1714fe1..63bf0c95fb18fd1571f6ffb9aa0ca935882a3fa1 100644 >--- a/Websites/perf.webkit.org/tools/js/os-build-fetcher.js >+++ b/Websites/perf.webkit.org/tools/js/os-build-fetcher.js >@@ -22,59 +22,64 @@ class OSBuildFetcher { > this._maxSubmitCount = osConfig['maxSubmitCount'] || 20; > } > >- static fetchAndReportAllInOrder(fetcherList) >+ static async fetchReportAndUpdateCommits(fetcherList) > { >- return mapInSerialPromiseChain(fetcherList, (fetcher) => fetcher.fetchAndReportNewBuilds()); >+ for (const fetcher of fetcherList) >+ await fetcher.fetchReportAndUpdateBuilds(); > } > >- fetchAndReportNewBuilds() >+ _splitArrayByLimit(array, maxCount) > { >- return this._fetchAvailableBuilds().then((results) => { >- this._logger.log(`Submitting ${results.length} builds for ${this._osConfig['name']}`); >- if (results.length == 0) >- return this._submitCommits(results); >- >- const splittedResults = []; >- for (let startIndex = 0; startIndex < results.length; startIndex += this._maxSubmitCount) >- splittedResults.push(results.slice(startIndex, startIndex + this._maxSubmitCount)); >- >- return mapInSerialPromiseChain(splittedResults, this._submitCommits.bind(this)).then((responses) => { >- assert(responses.every((response) => response['status'] == 'OK')); >- assert(responses.length > 0); >- return responses[0]; >- }); >- }); >+ const splittedArrayList = []; >+ for (let startIndex = 0; startIndex < array.length; startIndex += maxCount) >+ splittedArrayList.push(array.slice(startIndex, startIndex + maxCount)); >+ return splittedArrayList; >+ } >+ >+ async fetchReportAndUpdateBuilds() >+ { >+ const {newCommitsToReport, commitsToUpdate} = await this._fetchAvailableBuilds(); >+ >+ this._logger.log(`Submitting ${newCommitsToReport.length} builds for ${this._osConfig['name']}`); >+ await this._reportToEndPoint('/api/report-commits/', newCommitsToReport); >+ >+ this._logger.log(`Updating ${commitsToUpdate.length} builds for ${this._osConfig['name']}`); >+ await this._reportToEndPoint('/api/update-commits/', commitsToUpdate); > } > >- _fetchAvailableBuilds() >+ async _fetchAvailableBuilds() > { > const config = this._osConfig; >- const repositoryName = config['name']; >- let customCommands = config['customCommands']; >+ const repository = config['name']; >+ const newCommitsToReport = []; >+ const commitsToUpdate = []; >+ const customCommands = config['customCommands']; > >- return mapInSerialPromiseChain(customCommands, (command) => { >+ for (const command of customCommands) { > assert(command['minRevision']); > assert(command['maxRevision']); > const minRevisionOrder = this._computeOrder(command['minRevision']); > const maxRevisionOrder = this._computeOrder(command['maxRevision']); > >- const url = `/api/commits/${escape(repositoryName)}/last-reported?from=${minRevisionOrder}&to=${maxRevisionOrder}`; >+ const url = `/api/commits/${escape(repository)}/last-reported?from=${minRevisionOrder}&to=${maxRevisionOrder}`; >+ const result = await this._remoteAPI.getJSONWithStatus(url); >+ const minOrder = result['commits'].length == 1 ? parseInt(result['commits'][0]['order']) + 1 : minRevisionOrder; > >- return this._remoteAPI.getJSONWithStatus(url).then((result) => { >- const minOrder = result['commits'].length == 1 ? parseInt(result['commits'][0]['order']) + 1 : minRevisionOrder; >- return this._commitsForAvailableBuilds(repositoryName, command['command'], command['linesToIgnore'], minOrder, maxRevisionOrder); >- }).then((commits) => { >- const label = 'name' in command ? `"${command['name']}"` : `"${command['minRevision']}" to "${command['maxRevision']}"`; >- this._logger.log(`Found ${commits.length} builds for ${label}`); >+ const commitInfo = await this._commitsForAvailableBuilds(command['command']); >+ const commits = this._commitsWithinRange(commitInfo.allRevisions, repository, minOrder, maxRevisionOrder); > >- if ('ownedCommitCommand' in command) { >- this._logger.log(`Resolving ownedCommits for ${label}`); >- return this._addOwnedCommitsForBuild(commits, command['ownedCommitCommand']); >- } >+ const label = 'name' in command ? `"${command['name']}"` : `"${command['minRevision']}" to "${command['maxRevision']}"`; >+ this._logger.log(`Found ${commits.length} builds for ${label}`); >+ if ('ownedCommitCommand' in command) { >+ this._logger.log(`Resolving ownedCommits for ${label}`); >+ await this._addOwnedCommitsForBuild(commits, command['ownedCommitCommand']); >+ } >+ newCommitsToReport.push(...commits); > >- return commits; >- }); >- }).then((results) => [].concat(...results)); >+ for (const [revision, testabilityWarning] of Object.entries(commitInfo.commitsWithTestabilityWarnings)) >+ commitsToUpdate.push({repository, revision, testabilityWarning}); >+ } >+ return {newCommitsToReport, commitsToUpdate}; > } > > _computeOrder(revision) >@@ -89,17 +94,16 @@ class OSBuildFetcher { > return ((major * 100 + kind) * 10000 + minor) * 100 + variant; > } > >- _commitsForAvailableBuilds(repository, command, linesToIgnore, minOrder, maxOrder) >+ async _commitsForAvailableBuilds(command) > { >- return this._subprocess.execute(command).then((output) => { >- let lines = output.split('\n'); >- if (linesToIgnore){ >- const regex = new RegExp(linesToIgnore); >- lines = lines.filter((line) => !regex.exec(line)); >- } >- return lines.map((revision) => ({repository, revision, 'order': this._computeOrder(revision)})) >- .filter((commit) => commit['order'] >= minOrder && commit['order'] <= maxOrder); >- }); >+ const output = await this._subprocess.execute(command); >+ return JSON.parse(output); >+ } >+ >+ _commitsWithinRange(revisions, repository, minOrder, maxOrder) >+ { >+ return revisions.map((revision) => ({repository, revision, 'order': this._computeOrder(revision)})) >+ .filter((commit) => commit['order'] >= minOrder && commit['order'] <= maxOrder); > } > > _addOwnedCommitsForBuild(commits, command) >@@ -119,10 +123,16 @@ class OSBuildFetcher { > }); > } > >- _submitCommits(commits) >+ async _reportToEndPoint(endPoint, commitsToPost) > { >- const commitsToReport = {"slaveName": this._slaveAuth['name'], "slavePassword": this._slaveAuth['password'], 'commits': commits}; >- return this._remoteAPI.postJSONWithStatus('/api/report-commits/', commitsToReport); >+ if (!commitsToPost.length) >+ return; >+ >+ for (const commitsToSubmit of this._splitArrayByLimit(commitsToPost, this._maxSubmitCount)) { >+ const report = {"slaveName": this._slaveAuth['name'], "slavePassword": this._slaveAuth['password'], 'commits': commitsToSubmit}; >+ const response = await this._remoteAPI.postJSONWithStatus(endPoint, report); >+ assert(response['status'] === 'OK'); >+ } > } > } > >diff --git a/Websites/perf.webkit.org/tools/sync-os-versions.js b/Websites/perf.webkit.org/tools/sync-os-versions.js >index 77f4a0a7d34381943076b83ec48d5bc90b483586..4fbe582bf0388da4a4b3a75307d2f16b62297c78 100644 >--- a/Websites/perf.webkit.org/tools/sync-os-versions.js >+++ b/Websites/perf.webkit.org/tools/sync-os-versions.js >@@ -29,7 +29,7 @@ function syncLoop(options) > const remoteAPI = new RemoteAPI(serverConfig.server); > > const fetchers = osConfigList.map((osConfig) => new OSBuildFetcher(osConfig, remoteAPI, serverConfig.slave, new Subprocess, console)); >- OSBuildFetcher.fetchAndReportAllInOrder(fetchers).catch((error) => { >+ OSBuildFetcher.fetchReportAndUpdateCommits(fetchers).catch((error) => { > console.error(error); > if (typeof(error.stack) == 'string') { > for (let line of error.stack.split('\n')) >diff --git a/Websites/perf.webkit.org/unit-tests/commit-log-tests.js b/Websites/perf.webkit.org/unit-tests/commit-log-tests.js >index 5f8852492060eea6f3305bbc0f54eb60c010b2a3..9c6edf917d6414def71c8e6bd8a14ca531fecc1b 100644 >--- a/Websites/perf.webkit.org/unit-tests/commit-log-tests.js >+++ b/Websites/perf.webkit.org/unit-tests/commit-log-tests.js >@@ -58,6 +58,18 @@ function osxCommit() > }); > } > >+function osxCommitWithTestabilityWarning() >+{ >+ return new CommitLog(5, { >+ id: 5, >+ repository: MockModels.osx, >+ revision: '10.11.4 15E65', >+ time: null, >+ order: 1504065, >+ testabilityWarning: "Panic on boot" >+ }); >+} >+ > function oldOSXCommit() > { > return new CommitLog(6, { >@@ -417,6 +429,13 @@ describe('CommitLog', function () { > }); > }); > >+ describe('testabilityWarning', () => { >+ it('should return "testabilityWarning" message', () => { >+ const commit = osxCommitWithTestabilityWarning(); >+ assert.equal(commit.testabilityWarning(), 'Panic on boot'); >+ }); >+ }); >+ > const commitsAPIResponse = { > "commits":[{ > "id": "831151", >diff --git a/Websites/perf.webkit.org/unit-tests/commit-set-tests.js b/Websites/perf.webkit.org/unit-tests/commit-set-tests.js >index 39fccc54759f879f1ce0383b0a9b2ce7fc5b9a49..ba6c107d089aec195cb2e9a1a80c9fc258905e32 100644 >--- a/Websites/perf.webkit.org/unit-tests/commit-set-tests.js >+++ b/Websites/perf.webkit.org/unit-tests/commit-set-tests.js >@@ -170,6 +170,18 @@ function anotherCommitWithGitRevision() > }); > } > >+function commitWithTestabilityWarning() >+{ >+ return CommitLog.ensureSingleton(2023, { >+ id: 2023, >+ repository: MockModels.webkitGit, >+ revision: 'f24fc67873a0068fd5a3a9adac595f0036c21315', >+ ownsCommits: false, >+ time: 1456932773000, >+ testabilityWarning: 'Failed compilation' >+ }); >+} >+ > describe('CommitSet', () => { > MockRemoteAPI.inject(null, BrowserPrivilegedAPI); > MockModels.inject(); >@@ -294,6 +306,15 @@ describe('CommitSet', () => { > }); > } > >+ function commitSetWithTestabilityWarning() >+ { >+ return CommitSet.ensureSingleton(16, { >+ revisionItems: [{ commit: commitWithTestabilityWarning(), requiresBuild: false }, >+ { commit: webkitCommit(), requiresBuild: false }], >+ customRoots: [] >+ }); >+ } >+ > function oneMeasurementCommitSet() > { > return MeasurementCommitSet.ensureSingleton(1, [ >@@ -406,6 +427,25 @@ describe('CommitSet', () => { > {'11': { revision: 'webkit-commit-0', ownerRevision: null, patch: null}, customRoots: [456, 458]}]); > }); > }); >+ >+ describe('commitsWithTestabilityWarnings', () => { >+ it('should return commits with testability warning', () => { >+ const commits = commitSetWithTestabilityWarning().commitsWithTestabilityWarnings(); >+ assert.equal(commits.length, 1); >+ assert.equal(commits[0], commitWithTestabilityWarning()); >+ }); >+ >+ it('should return empty list if no commit in commit set contains testablity warning', () => { >+ const commits = commitSetWithTwoCommits().commitsWithTestabilityWarnings(); >+ assert.equal(commits.length, 0); >+ }); >+ }); >+ >+ describe('commits', () => { >+ it('should return all commits in commit set', () => { >+ assert.equal(commitSetWithTwoCommits().commits().length, 2); >+ }); >+ }); > }); > > describe('IntermediateCommitSet', () => { >@@ -629,9 +669,7 @@ describe('IntermediateCommitSet', () => { > assert(!commitSet.commitForRepository(MockModels.webkit)); > }); > }); >- > }); >- > }); > > describe('CustomCommitSet', () => {
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Formatted Diff
|
Diff
Attachments on
bug 191557
:
354587
|
357273
|
357308
|
357486
|
357656
|
357803