diff options
author | Ben Noordhuis <info@bnoordhuis.nl> | 2015-03-27 12:04:12 +0100 |
---|---|---|
committer | Chris Dickinson <christopher.s.dickinson@gmail.com> | 2015-04-28 14:38:16 -0700 |
commit | 36cd5fb9d27b830320e57213f5b8829ffbb93324 (patch) | |
tree | bbab4215d26f8597019135206426fccf27a3089e /deps/v8/tools/release | |
parent | b57cc51d8d3f4ad279591ae8fa6584ee22773b97 (diff) | |
download | android-node-v8-36cd5fb9d27b830320e57213f5b8829ffbb93324.tar.gz android-node-v8-36cd5fb9d27b830320e57213f5b8829ffbb93324.tar.bz2 android-node-v8-36cd5fb9d27b830320e57213f5b8829ffbb93324.zip |
deps: upgrade v8 to 4.2.77.13
This commit applies some secondary changes in order to make `make test`
pass cleanly:
* disable broken postmortem debugging in common.gypi
* drop obsolete strict mode test in parallel/test-repl
* drop obsolete test parallel/test-v8-features
PR-URL: https://github.com/iojs/io.js/pull/1232
Reviewed-By: Fedor Indutny <fedor@indutny.com>
Diffstat (limited to 'deps/v8/tools/release')
-rwxr-xr-x | deps/v8/tools/release/auto_push.py | 120 | ||||
-rwxr-xr-x | deps/v8/tools/release/auto_roll.py | 144 | ||||
-rwxr-xr-x | deps/v8/tools/release/auto_tag.py | 201 | ||||
-rwxr-xr-x | deps/v8/tools/release/check_clusterfuzz.py | 174 | ||||
-rwxr-xr-x | deps/v8/tools/release/chromium_roll.py | 179 | ||||
-rw-r--r-- | deps/v8/tools/release/common_includes.py | 898 | ||||
-rwxr-xr-x | deps/v8/tools/release/create_release.py | 313 | ||||
-rw-r--r-- | deps/v8/tools/release/git_recipes.py | 275 | ||||
-rwxr-xr-x | deps/v8/tools/release/merge_to_branch.py | 316 | ||||
-rwxr-xr-x | deps/v8/tools/release/push_to_candidates.py | 415 | ||||
-rwxr-xr-x | deps/v8/tools/release/releases.py | 513 | ||||
-rwxr-xr-x | deps/v8/tools/release/script_test.py | 54 | ||||
-rw-r--r-- | deps/v8/tools/release/test_scripts.py | 1558 |
13 files changed, 5160 insertions, 0 deletions
diff --git a/deps/v8/tools/release/auto_push.py b/deps/v8/tools/release/auto_push.py new file mode 100755 index 0000000000..121288f5b5 --- /dev/null +++ b/deps/v8/tools/release/auto_push.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# Copyright 2013 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse +import json +import os +import re +import sys +import urllib + +from common_includes import * +import push_to_candidates + + +class Preparation(Step): + MESSAGE = "Preparation." + + def RunStep(self): + self.InitialEnvironmentChecks(self.default_cwd) + self.CommonPrepare() + + +class FetchCandidate(Step): + MESSAGE = "Fetching V8 roll ref." + + def RunStep(self): + # The roll ref points to the candidate to be rolled. + self.Git("fetch origin +refs/heads/roll:refs/heads/roll") + self["candidate"] = self.Git("show-ref -s refs/heads/roll").strip() + + +class LastReleaseBailout(Step): + MESSAGE = "Checking last V8 release base." + + def RunStep(self): + last_release = self.GetLatestReleaseBase() + commits = self.GitLog( + format="%H", git_hash="%s..%s" % (last_release, self["candidate"])) + + if not commits: + print "Already pushed current candidate %s" % self["candidate"] + return True + + +class PushToCandidates(Step): + MESSAGE = "Pushing to candidates if specified." + + def RunStep(self): + print "Pushing candidate %s to candidates." % self["candidate"] + + args = [ + "--author", self._options.author, + "--reviewer", self._options.reviewer, + "--revision", self["candidate"], + "--force", + ] + + if self._options.work_dir: + args.extend(["--work-dir", self._options.work_dir]) + + # TODO(machenbach): Update the script before calling it. + if self._options.push: + self._side_effect_handler.Call( + push_to_candidates.PushToCandidates().Run, args) + + +class AutoPush(ScriptsBase): + def _PrepareOptions(self, parser): + parser.add_argument("-p", "--push", + help="Push to candidates. Dry run if unspecified.", + default=False, action="store_true") + + def _ProcessOptions(self, options): + if not options.author or not options.reviewer: # pragma: no cover + print "You need to specify author and reviewer." + return False + options.requires_editor = False + return True + + def _Config(self): + return { + "PERSISTFILE_BASENAME": "/tmp/v8-auto-push-tempfile", + } + + def _Steps(self): + return [ + Preparation, + FetchCandidate, + LastReleaseBailout, + PushToCandidates, + ] + + +if __name__ == "__main__": # pragma: no cover + sys.exit(AutoPush().Run()) diff --git a/deps/v8/tools/release/auto_roll.py b/deps/v8/tools/release/auto_roll.py new file mode 100755 index 0000000000..315a4bc2a0 --- /dev/null +++ b/deps/v8/tools/release/auto_roll.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python +# Copyright 2014 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import json +import os +import sys +import urllib + +from common_includes import * +import chromium_roll + + +class CheckActiveRoll(Step): + MESSAGE = "Check active roll." + + @staticmethod + def ContainsChromiumRoll(changes): + for change in changes: + if change["subject"].startswith("Update V8 to"): + return True + return False + + def RunStep(self): + params = { + "closed": 3, + "owner": self._options.author, + "limit": 30, + "format": "json", + } + params = urllib.urlencode(params) + search_url = "https://codereview.chromium.org/search" + result = self.ReadURL(search_url, params, wait_plan=[5, 20]) + if self.ContainsChromiumRoll(json.loads(result)["results"]): + print "Stop due to existing Chromium roll." + return True + + +class DetectLastRoll(Step): + MESSAGE = "Detect commit ID of the last Chromium roll." + + def RunStep(self): + # The revision that should be rolled. + latest_release = self.GetLatestRelease() + + # Interpret the DEPS file to retrieve the v8 revision. + # TODO(machenbach): This should be part or the roll-deps api of + # depot_tools. + Var = lambda var: '%s' + exec(FileToText(os.path.join(self._options.chromium, "DEPS"))) + + # The revision rolled last. + self["last_roll"] = vars['v8_revision'] + + # TODO(machenbach): It is possible that the auto-push script made a new + # fast-forward release (e.g. 4.2.3) while somebody patches the last + # candidate (e.g. 4.2.2.1). In this case, the auto-roller would pick + # the fast-forward release. Should there be a way to prioritize the + # patched version? + + if latest_release == self["last_roll"]: + # We always try to roll if the latest revision is not the revision in + # chromium. + print("There is no newer v8 revision than the one in Chromium (%s)." + % self["last_roll"]) + return True + + +class CheckClusterFuzz(Step): + MESSAGE = "Check ClusterFuzz api for new problems." + + def RunStep(self): + if not os.path.exists(self.Config("CLUSTERFUZZ_API_KEY_FILE")): + print "Skipping ClusterFuzz check. No api key file found." + return False + api_key = FileToText(self.Config("CLUSTERFUZZ_API_KEY_FILE")) + # Check for open, reproducible issues that have no associated bug. + result = self._side_effect_handler.ReadClusterFuzzAPI( + api_key, job_type="linux_asan_d8_dbg", reproducible="True", + open="True", bug_information="", + revision_greater_or_equal=str(self["last_push"])) + if result: + print "Stop due to pending ClusterFuzz issues." + return True + + +class RollChromium(Step): + MESSAGE = "Roll V8 into Chromium." + + def RunStep(self): + if self._options.roll: + args = [ + "--author", self._options.author, + "--reviewer", self._options.reviewer, + "--chromium", self._options.chromium, + "--last-roll", self["last_roll"], + "--use-commit-queue", + ] + if self._options.sheriff: + args.extend([ + "--sheriff", "--googlers-mapping", self._options.googlers_mapping]) + if self._options.dry_run: + args.extend(["--dry-run"]) + if self._options.work_dir: + args.extend(["--work-dir", self._options.work_dir]) + self._side_effect_handler.Call(chromium_roll.ChromiumRoll().Run, args) + + +class AutoRoll(ScriptsBase): + def _PrepareOptions(self, parser): + parser.add_argument("-c", "--chromium", required=True, + help=("The path to your Chromium src/ " + "directory to automate the V8 roll.")) + parser.add_argument("--roll", help="Call Chromium roll script.", + default=False, action="store_true") + + def _ProcessOptions(self, options): # pragma: no cover + if not options.reviewer: + print "A reviewer (-r) is required." + return False + if not options.author: + print "An author (-a) is required." + return False + return True + + def _Config(self): + return { + "PERSISTFILE_BASENAME": "/tmp/v8-auto-roll-tempfile", + "CLUSTERFUZZ_API_KEY_FILE": ".cf_api_key", + } + + def _Steps(self): + return [ + CheckActiveRoll, + DetectLastRoll, + CheckClusterFuzz, + RollChromium, + ] + + +if __name__ == "__main__": # pragma: no cover + sys.exit(AutoRoll().Run()) diff --git a/deps/v8/tools/release/auto_tag.py b/deps/v8/tools/release/auto_tag.py new file mode 100755 index 0000000000..a52a028697 --- /dev/null +++ b/deps/v8/tools/release/auto_tag.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python +# Copyright 2014 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import sys + +from common_includes import * + + +class Preparation(Step): + MESSAGE = "Preparation." + + def RunStep(self): + # TODO(machenbach): Remove after the git switch. + if self.Config("PERSISTFILE_BASENAME") == "/tmp/v8-auto-tag-tempfile": + print "This script is disabled until after the v8 git migration." + return True + + self.CommonPrepare() + self.PrepareBranch() + self.GitCheckout("master") + self.vc.Pull() + + +class GetTags(Step): + MESSAGE = "Get all V8 tags." + + def RunStep(self): + self.GitCreateBranch(self._config["BRANCHNAME"]) + self["tags"] = self.vc.GetTags() + + +class GetOldestUntaggedVersion(Step): + MESSAGE = "Check if there's a version on bleeding edge without a tag." + + def RunStep(self): + tags = set(self["tags"]) + self["candidate"] = None + self["candidate_version"] = None + self["next"] = None + self["next_version"] = None + + # Iterate backwards through all automatic version updates. + for git_hash in self.GitLog( + format="%H", grep="\\[Auto\\-roll\\] Bump up version to").splitlines(): + + # Get the version. + if not self.GitCheckoutFileSafe(VERSION_FILE, git_hash): + continue + + self.ReadAndPersistVersion() + version = self.ArrayToVersion("") + + # Strip off trailing patch level (tags don't include tag level 0). + if version.endswith(".0"): + version = version[:-2] + + # Clean up checked-out version file. + self.GitCheckoutFileSafe(VERSION_FILE, "HEAD") + + if version in tags: + if self["candidate"]: + # Revision "git_hash" is tagged already and "candidate" was the next + # newer revision without a tag. + break + else: + print("Stop as %s is the latest version and it has been tagged." % + version) + self.CommonCleanup() + return True + else: + # This is the second oldest version without a tag. + self["next"] = self["candidate"] + self["next_version"] = self["candidate_version"] + + # This is the oldest version without a tag. + self["candidate"] = git_hash + self["candidate_version"] = version + + if not self["candidate"] or not self["candidate_version"]: + print "Nothing found to tag." + self.CommonCleanup() + return True + + print("Candidate for tagging is %s with version %s" % + (self["candidate"], self["candidate_version"])) + + +class GetLKGRs(Step): + MESSAGE = "Get the last lkgrs." + + def RunStep(self): + revision_url = "https://v8-status.appspot.com/revisions?format=json" + status_json = self.ReadURL(revision_url, wait_plan=[5, 20]) + self["lkgrs"] = [entry["revision"] + for entry in json.loads(status_json) if entry["status"]] + + +class CalculateTagRevision(Step): + MESSAGE = "Calculate the revision to tag." + + def LastLKGR(self, min_rev, max_rev): + """Finds the newest lkgr between min_rev (inclusive) and max_rev + (exclusive). + """ + for lkgr in self["lkgrs"]: + # LKGRs are reverse sorted. + if int(min_rev) <= int(lkgr) and int(lkgr) < int(max_rev): + return lkgr + return None + + def RunStep(self): + # Get the lkgr after the tag candidate and before the next tag candidate. + candidate_svn = self.vc.GitSvn(self["candidate"]) + if self["next"]: + next_svn = self.vc.GitSvn(self["next"]) + else: + # Don't include the version change commit itself if there is no upper + # limit yet. + candidate_svn = str(int(candidate_svn) + 1) + next_svn = sys.maxint + lkgr_svn = self.LastLKGR(candidate_svn, next_svn) + + if not lkgr_svn: + print "There is no lkgr since the candidate version yet." + self.CommonCleanup() + return True + + # Let's check if the lkgr is at least three hours old. + self["lkgr"] = self.vc.SvnGit(lkgr_svn) + if not self["lkgr"]: + print "Couldn't find git hash for lkgr %s" % lkgr_svn + self.CommonCleanup() + return True + + lkgr_utc_time = int(self.GitLog(n=1, format="%at", git_hash=self["lkgr"])) + current_utc_time = self._side_effect_handler.GetUTCStamp() + + if current_utc_time < lkgr_utc_time + 10800: + print "Candidate lkgr %s is too recent for tagging." % lkgr_svn + self.CommonCleanup() + return True + + print "Tagging revision %s with %s" % (lkgr_svn, self["candidate_version"]) + + +class MakeTag(Step): + MESSAGE = "Tag the version." + + def RunStep(self): + if not self._options.dry_run: + self.GitReset(self["lkgr"]) + # FIXME(machenbach): Make this work with the git repo. + self.vc.Tag(self["candidate_version"], + "svn/bleeding_edge", + "This won't work!") + + +class CleanUp(Step): + MESSAGE = "Clean up." + + def RunStep(self): + self.CommonCleanup() + + +class AutoTag(ScriptsBase): + def _PrepareOptions(self, parser): + parser.add_argument("--dry_run", help="Don't tag the new version.", + default=False, action="store_true") + + def _ProcessOptions(self, options): # pragma: no cover + if not options.dry_run and not options.author: + print "Specify your chromium.org email with -a" + return False + options.wait_for_lgtm = False + options.force_readline_defaults = True + options.force_upload = True + return True + + def _Config(self): + return { + "BRANCHNAME": "auto-tag-v8", + "PERSISTFILE_BASENAME": "/tmp/v8-auto-tag-tempfile", + } + + def _Steps(self): + return [ + Preparation, + GetTags, + GetOldestUntaggedVersion, + GetLKGRs, + CalculateTagRevision, + MakeTag, + CleanUp, + ] + + +if __name__ == "__main__": # pragma: no cover + sys.exit(AutoTag().Run()) diff --git a/deps/v8/tools/release/check_clusterfuzz.py b/deps/v8/tools/release/check_clusterfuzz.py new file mode 100755 index 0000000000..d4ba90ba48 --- /dev/null +++ b/deps/v8/tools/release/check_clusterfuzz.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python +# Copyright 2014 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Script to check for new clusterfuzz issues since the last rolled v8 revision. + +Returns a json list with test case IDs if any. + +Security considerations: The security key and request data must never be +written to public logs. Public automated callers of this script should +suppress stdout and stderr and only process contents of the results_file. +""" + + +import argparse +import httplib +import json +import os +import re +import sys +import urllib +import urllib2 + + +# Constants to git repos. +BASE_URL = "https://chromium.googlesource.com" +DEPS_LOG = BASE_URL + "/chromium/src/+log/master/DEPS?format=JSON" + +# Constants for retrieving v8 rolls. +CRREV = "https://cr-rev.appspot.com/_ah/api/crrev/v1/commit/%s" +V8_COMMIT_RE = re.compile( + r"^Update V8 to version \d+\.\d+\.\d+ \(based on ([a-fA-F0-9]+)\)\..*") + +# Constants for the clusterfuzz backend. +HOSTNAME = "backend-dot-cluster-fuzz.appspot.com" + +# Crash patterns. +V8_INTERNAL_RE = re.compile(r"^v8::internal.*") +ANY_RE = re.compile(r".*") + +# List of all api requests. +BUG_SPECS = [ + { + "args": { + "job_type": "linux_asan_chrome_v8", + "reproducible": "True", + "open": "True", + "bug_information": "", + }, + "crash_state": V8_INTERNAL_RE, + }, + { + "args": { + "job_type": "linux_asan_d8_dbg", + "reproducible": "True", + "open": "True", + "bug_information": "", + }, + "crash_state": ANY_RE, + }, +] + + +def GetRequest(url): + url_fh = urllib2.urlopen(url, None, 60) + try: + return url_fh.read() + finally: + url_fh.close() + + +def GetLatestV8InChromium(): + """Returns the commit position number of the latest v8 roll in chromium.""" + + # Check currently rolled v8 revision. + result = GetRequest(DEPS_LOG) + if not result: + return None + + # Strip security header and load json. + commits = json.loads(result[5:]) + + git_revision = None + for commit in commits["log"]: + # Get latest commit that matches the v8 roll pattern. Ignore cherry-picks. + match = re.match(V8_COMMIT_RE, commit["message"]) + if match: + git_revision = match.group(1) + break + else: + return None + + # Get commit position number for v8 revision. + result = GetRequest(CRREV % git_revision) + if not result: + return None + + commit = json.loads(result) + assert commit["repo"] == "v8/v8" + return commit["number"] + + +def APIRequest(key, **params): + """Send a request to the clusterfuzz api. + + Returns a json dict of the response. + """ + + params["api_key"] = key + params = urllib.urlencode(params) + + headers = {"Content-type": "application/x-www-form-urlencoded"} + + try: + conn = httplib.HTTPSConnection(HOSTNAME) + conn.request("POST", "/_api/", params, headers) + + response = conn.getresponse() + + # Never leak "data" into public logs. + data = response.read() + except: + raise Exception("ERROR: Connection problem.") + + try: + return json.loads(data) + except: + raise Exception("ERROR: Could not read response. Is your key valid?") + + return None + + +def Main(): + parser = argparse.ArgumentParser() + parser.add_argument("-k", "--key-file", required=True, + help="A file with the clusterfuzz api key.") + parser.add_argument("-r", "--results-file", + help="A file to write the results to.") + options = parser.parse_args() + + # Get api key. The key's content must never be logged. + assert options.key_file + with open(options.key_file) as f: + key = f.read().strip() + assert key + + revision_number = GetLatestV8InChromium() + + results = [] + for spec in BUG_SPECS: + args = dict(spec["args"]) + # Use incremented revision as we're interested in all revision greater than + # what's currently rolled into chromium. + if revision_number: + args["revision_greater_or_equal"] = str(int(revision_number) + 1) + + # Never print issue details in public logs. + issues = APIRequest(key, **args) + assert issues is not None + for issue in issues: + if re.match(spec["crash_state"], issue["crash_state"]): + results.append(issue["id"]) + + if options.results_file: + with open(options.results_file, "w") as f: + f.write(json.dumps(results)) + else: + print results + + +if __name__ == "__main__": + sys.exit(Main()) diff --git a/deps/v8/tools/release/chromium_roll.py b/deps/v8/tools/release/chromium_roll.py new file mode 100755 index 0000000000..8a3ff4a0a7 --- /dev/null +++ b/deps/v8/tools/release/chromium_roll.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +# Copyright 2014 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import os +import sys + +from common_includes import * + + +ROLL_SUMMARY = ("Summary of changes available at:\n" + "https://chromium.googlesource.com/v8/v8/+log/%s..%s") + + +class Preparation(Step): + MESSAGE = "Preparation." + + def RunStep(self): + # Update v8 remote tracking branches. + self.GitFetchOrigin() + + +class DetectLastPush(Step): + MESSAGE = "Detect commit ID of last release." + + def RunStep(self): + # The revision that should be rolled. + self["last_push"] = self._options.last_push or self.GetLatestRelease() + self["push_title"] = self.GitLog(n=1, format="%s", + git_hash=self["last_push"]) + + # The master revision this release is based on. + self["push_base"] = self.GetLatestReleaseBase() + + # FIXME(machenbach): Manually specifying a revision doesn't work at the + # moment. Needs more complicated logic to find the correct push_base above. + # Maybe delete that parameter entirely? + assert not self._options.last_push + + # Determine the master revision of the last roll. + version = self.GetVersionTag(self._options.last_roll) + assert version + self["last_rolled_base"] = self.GetLatestReleaseBase(version=version) + assert self["last_rolled_base"] + + +class SwitchChromium(Step): + MESSAGE = "Switch to Chromium checkout." + + def RunStep(self): + self["v8_path"] = os.getcwd() + cwd = self._options.chromium + os.chdir(cwd) + self.InitialEnvironmentChecks(cwd) + # Check for a clean workdir. + if not self.GitIsWorkdirClean(cwd=cwd): # pragma: no cover + self.Die("Workspace is not clean. Please commit or undo your changes.") + # Assert that the DEPS file is there. + if not os.path.exists(os.path.join(cwd, "DEPS")): # pragma: no cover + self.Die("DEPS file not present.") + + +class UpdateChromiumCheckout(Step): + MESSAGE = "Update the checkout and create a new branch." + + def RunStep(self): + self.GitCheckout("master", cwd=self._options.chromium) + self.Command("gclient", "sync --nohooks", cwd=self._options.chromium) + self.GitPull(cwd=self._options.chromium) + + # Update v8 remotes. + self.GitFetchOrigin() + + self.GitCreateBranch("v8-roll-%s" % self["last_push"], + cwd=self._options.chromium) + + +class UploadCL(Step): + MESSAGE = "Create and upload CL." + + def RunStep(self): + # Patch DEPS file. + if self.Command( + "roll-dep", "v8 %s" % self["last_push"], + cwd=self._options.chromium) is None: + self.Die("Failed to create deps for %s" % self["last_push"]) + + message = [] + message.append("Update V8 to %s." % self["push_title"].lower()) + + message.append( + ROLL_SUMMARY % (self["last_rolled_base"][:8], self["push_base"][:8])) + + if self["sheriff"]: + message.append("Please reply to the V8 sheriff %s in case of problems." + % self["sheriff"]) + message.append("TBR=%s" % self._options.reviewer) + self.GitCommit("\n\n".join(message), + author=self._options.author, + cwd=self._options.chromium) + if not self._options.dry_run: + self.GitUpload(author=self._options.author, + force=True, + cq=self._options.use_commit_queue, + cwd=self._options.chromium) + print "CL uploaded." + else: + self.GitCheckout("master", cwd=self._options.chromium) + self.GitDeleteBranch("v8-roll-%s" % self["last_push"], + cwd=self._options.chromium) + print "Dry run - don't upload." + + +# TODO(machenbach): Make this obsolete. We are only in the chromium chechout +# for the initial .git check. +class SwitchV8(Step): + MESSAGE = "Returning to V8 checkout." + + def RunStep(self): + os.chdir(self["v8_path"]) + + +class CleanUp(Step): + MESSAGE = "Done!" + + def RunStep(self): + print("Congratulations, you have successfully rolled %s into " + "Chromium. Please don't forget to update the v8rel spreadsheet." + % self["last_push"]) + + # Clean up all temporary files. + Command("rm", "-f %s*" % self._config["PERSISTFILE_BASENAME"]) + + +class ChromiumRoll(ScriptsBase): + def _PrepareOptions(self, parser): + parser.add_argument("-c", "--chromium", required=True, + help=("The path to your Chromium src/ " + "directory to automate the V8 roll.")) + parser.add_argument("-l", "--last-push", + help="The git commit ID of the last candidates push.") + parser.add_argument("--last-roll", required=True, + help="The git commit ID of the last rolled version.") + parser.add_argument("--use-commit-queue", + help="Check the CQ bit on upload.", + default=False, action="store_true") + + def _ProcessOptions(self, options): # pragma: no cover + if not options.author or not options.reviewer: + print "A reviewer (-r) and an author (-a) are required." + return False + + options.requires_editor = False + options.force = True + options.manual = False + return True + + def _Config(self): + return { + "PERSISTFILE_BASENAME": "/tmp/v8-chromium-roll-tempfile", + } + + def _Steps(self): + return [ + Preparation, + DetectLastPush, + DetermineV8Sheriff, + SwitchChromium, + UpdateChromiumCheckout, + UploadCL, + SwitchV8, + CleanUp, + ] + + +if __name__ == "__main__": # pragma: no cover + sys.exit(ChromiumRoll().Run()) diff --git a/deps/v8/tools/release/common_includes.py b/deps/v8/tools/release/common_includes.py new file mode 100644 index 0000000000..bae05bc6b5 --- /dev/null +++ b/deps/v8/tools/release/common_includes.py @@ -0,0 +1,898 @@ +#!/usr/bin/env python +# Copyright 2013 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse +import datetime +import httplib +import glob +import imp +import json +import os +import re +import shutil +import subprocess +import sys +import textwrap +import time +import urllib +import urllib2 + +from git_recipes import GitRecipesMixin +from git_recipes import GitFailedException + +CHANGELOG_FILE = "ChangeLog" +PUSH_MSG_GIT_RE = re.compile(r".* \(based on (?P<git_rev>[a-fA-F0-9]+)\)$") +PUSH_MSG_NEW_RE = re.compile(r"^Version \d+\.\d+\.\d+$") +VERSION_FILE = os.path.join("src", "version.cc") +VERSION_RE = re.compile(r"^\d+\.\d+\.\d+(?:\.\d+)?$") + +# V8 base directory. +V8_BASE = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +def TextToFile(text, file_name): + with open(file_name, "w") as f: + f.write(text) + + +def AppendToFile(text, file_name): + with open(file_name, "a") as f: + f.write(text) + + +def LinesInFile(file_name): + with open(file_name) as f: + for line in f: + yield line + + +def FileToText(file_name): + with open(file_name) as f: + return f.read() + + +def MSub(rexp, replacement, text): + return re.sub(rexp, replacement, text, flags=re.MULTILINE) + + +def Fill80(line): + # Replace tabs and remove surrounding space. + line = re.sub(r"\t", r" ", line.strip()) + + # Format with 8 characters indentation and line width 80. + return textwrap.fill(line, width=80, initial_indent=" ", + subsequent_indent=" ") + + +def MakeComment(text): + return MSub(r"^( ?)", "#", text) + + +def StripComments(text): + # Use split not splitlines to keep terminal newlines. + return "\n".join(filter(lambda x: not x.startswith("#"), text.split("\n"))) + + +def MakeChangeLogBody(commit_messages, auto_format=False): + result = "" + added_titles = set() + for (title, body, author) in commit_messages: + # TODO(machenbach): Better check for reverts. A revert should remove the + # original CL from the actual log entry. + title = title.strip() + if auto_format: + # Only add commits that set the LOG flag correctly. + log_exp = r"^[ \t]*LOG[ \t]*=[ \t]*(?:(?:Y(?:ES)?)|TRUE)" + if not re.search(log_exp, body, flags=re.I | re.M): + continue + # Never include reverts. + if title.startswith("Revert "): + continue + # Don't include duplicates. + if title in added_titles: + continue + + # Add and format the commit's title and bug reference. Move dot to the end. + added_titles.add(title) + raw_title = re.sub(r"(\.|\?|!)$", "", title) + bug_reference = MakeChangeLogBugReference(body) + space = " " if bug_reference else "" + result += "%s\n" % Fill80("%s%s%s." % (raw_title, space, bug_reference)) + + # Append the commit's author for reference if not in auto-format mode. + if not auto_format: + result += "%s\n" % Fill80("(%s)" % author.strip()) + + result += "\n" + return result + + +def MakeChangeLogBugReference(body): + """Grep for "BUG=xxxx" lines in the commit message and convert them to + "(issue xxxx)". + """ + crbugs = [] + v8bugs = [] + + def AddIssues(text): + ref = re.match(r"^BUG[ \t]*=[ \t]*(.+)$", text.strip()) + if not ref: + return + for bug in ref.group(1).split(","): + bug = bug.strip() + match = re.match(r"^v8:(\d+)$", bug) + if match: v8bugs.append(int(match.group(1))) + else: + match = re.match(r"^(?:chromium:)?(\d+)$", bug) + if match: crbugs.append(int(match.group(1))) + + # Add issues to crbugs and v8bugs. + map(AddIssues, body.splitlines()) + + # Filter duplicates, sort, stringify. + crbugs = map(str, sorted(set(crbugs))) + v8bugs = map(str, sorted(set(v8bugs))) + + bug_groups = [] + def FormatIssues(prefix, bugs): + if len(bugs) > 0: + plural = "s" if len(bugs) > 1 else "" + bug_groups.append("%sissue%s %s" % (prefix, plural, ", ".join(bugs))) + + FormatIssues("", v8bugs) + FormatIssues("Chromium ", crbugs) + + if len(bug_groups) > 0: + return "(%s)" % ", ".join(bug_groups) + else: + return "" + + +def SortingKey(version): + """Key for sorting version number strings: '3.11' > '3.2.1.1'""" + version_keys = map(int, version.split(".")) + # Fill up to full version numbers to normalize comparison. + while len(version_keys) < 4: # pragma: no cover + version_keys.append(0) + # Fill digits. + return ".".join(map("{0:04d}".format, version_keys)) + + +# Some commands don't like the pipe, e.g. calling vi from within the script or +# from subscripts like git cl upload. +def Command(cmd, args="", prefix="", pipe=True, cwd=None): + cwd = cwd or os.getcwd() + # TODO(machenbach): Use timeout. + cmd_line = "%s %s %s" % (prefix, cmd, args) + print "Command: %s" % cmd_line + print "in %s" % cwd + sys.stdout.flush() + try: + if pipe: + return subprocess.check_output(cmd_line, shell=True, cwd=cwd) + else: + return subprocess.check_call(cmd_line, shell=True, cwd=cwd) + except subprocess.CalledProcessError: + return None + finally: + sys.stdout.flush() + sys.stderr.flush() + + +# Wrapper for side effects. +class SideEffectHandler(object): # pragma: no cover + def Call(self, fun, *args, **kwargs): + return fun(*args, **kwargs) + + def Command(self, cmd, args="", prefix="", pipe=True, cwd=None): + return Command(cmd, args, prefix, pipe, cwd=cwd) + + def ReadLine(self): + return sys.stdin.readline().strip() + + def ReadURL(self, url, params=None): + # pylint: disable=E1121 + url_fh = urllib2.urlopen(url, params, 60) + try: + return url_fh.read() + finally: + url_fh.close() + + def ReadClusterFuzzAPI(self, api_key, **params): + params["api_key"] = api_key.strip() + params = urllib.urlencode(params) + + headers = {"Content-type": "application/x-www-form-urlencoded"} + + conn = httplib.HTTPSConnection("backend-dot-cluster-fuzz.appspot.com") + conn.request("POST", "/_api/", params, headers) + + response = conn.getresponse() + data = response.read() + + try: + return json.loads(data) + except: + print data + print "ERROR: Could not read response. Is your key valid?" + raise + + def Sleep(self, seconds): + time.sleep(seconds) + + def GetDate(self): + return datetime.date.today().strftime("%Y-%m-%d") + + def GetUTCStamp(self): + return time.mktime(datetime.datetime.utcnow().timetuple()) + +DEFAULT_SIDE_EFFECT_HANDLER = SideEffectHandler() + + +class NoRetryException(Exception): + pass + + +class VCInterface(object): + def InjectStep(self, step): + self.step=step + + def Pull(self): + raise NotImplementedError() + + def Fetch(self): + raise NotImplementedError() + + def GetTags(self): + raise NotImplementedError() + + def GetBranches(self): + raise NotImplementedError() + + def MasterBranch(self): + raise NotImplementedError() + + def CandidateBranch(self): + raise NotImplementedError() + + def RemoteMasterBranch(self): + raise NotImplementedError() + + def RemoteCandidateBranch(self): + raise NotImplementedError() + + def RemoteBranch(self, name): + raise NotImplementedError() + + def CLLand(self): + raise NotImplementedError() + + def Tag(self, tag, remote, message): + """Sets a tag for the current commit. + + Assumptions: The commit already landed and the commit message is unique. + """ + raise NotImplementedError() + + +class GitInterface(VCInterface): + def Pull(self): + self.step.GitPull() + + def Fetch(self): + self.step.Git("fetch") + + def GetTags(self): + return self.step.Git("tag").strip().splitlines() + + def GetBranches(self): + # Get relevant remote branches, e.g. "branch-heads/3.25". + branches = filter( + lambda s: re.match(r"^branch\-heads/\d+\.\d+$", s), + self.step.GitRemotes()) + # Remove 'branch-heads/' prefix. + return map(lambda s: s[13:], branches) + + def MasterBranch(self): + return "master" + + def CandidateBranch(self): + return "candidates" + + def RemoteMasterBranch(self): + return "origin/master" + + def RemoteCandidateBranch(self): + return "origin/candidates" + + def RemoteBranch(self, name): + # Assume that if someone "fully qualified" the ref, they know what they + # want. + if name.startswith('refs/'): + return name + if name in ["candidates", "master"]: + return "refs/remotes/origin/%s" % name + try: + # Check if branch is in heads. + if self.step.Git("show-ref refs/remotes/origin/%s" % name).strip(): + return "refs/remotes/origin/%s" % name + except GitFailedException: + pass + try: + # Check if branch is in branch-heads. + if self.step.Git("show-ref refs/remotes/branch-heads/%s" % name).strip(): + return "refs/remotes/branch-heads/%s" % name + except GitFailedException: + pass + self.Die("Can't find remote of %s" % name) + + def Tag(self, tag, remote, message): + # Wait for the commit to appear. Assumes unique commit message titles (this + # is the case for all automated merge and push commits - also no title is + # the prefix of another title). + commit = None + for wait_interval in [3, 7, 15, 35, 45, 60]: + self.step.Git("fetch") + commit = self.step.GitLog(n=1, format="%H", grep=message, branch=remote) + if commit: + break + print("The commit has not replicated to git. Waiting for %s seconds." % + wait_interval) + self.step._side_effect_handler.Sleep(wait_interval) + else: + self.step.Die("Couldn't determine commit for setting the tag. Maybe the " + "git updater is lagging behind?") + + self.step.Git("tag %s %s" % (tag, commit)) + self.step.Git("push origin %s" % tag) + + def CLLand(self): + self.step.GitCLLand() + + +class Step(GitRecipesMixin): + def __init__(self, text, number, config, state, options, handler): + self._text = text + self._number = number + self._config = config + self._state = state + self._options = options + self._side_effect_handler = handler + self.vc = GitInterface() + self.vc.InjectStep(self) + + # The testing configuration might set a different default cwd. + self.default_cwd = (self._config.get("DEFAULT_CWD") or + os.path.join(self._options.work_dir, "v8")) + + assert self._number >= 0 + assert self._config is not None + assert self._state is not None + assert self._side_effect_handler is not None + + def __getitem__(self, key): + # Convenience method to allow direct [] access on step classes for + # manipulating the backed state dict. + return self._state.get(key) + + def __setitem__(self, key, value): + # Convenience method to allow direct [] access on step classes for + # manipulating the backed state dict. + self._state[key] = value + + def Config(self, key): + return self._config[key] + + def Run(self): + # Restore state. + state_file = "%s-state.json" % self._config["PERSISTFILE_BASENAME"] + if not self._state and os.path.exists(state_file): + self._state.update(json.loads(FileToText(state_file))) + + print ">>> Step %d: %s" % (self._number, self._text) + try: + return self.RunStep() + finally: + # Persist state. + TextToFile(json.dumps(self._state), state_file) + + def RunStep(self): # pragma: no cover + raise NotImplementedError + + def Retry(self, cb, retry_on=None, wait_plan=None): + """ Retry a function. + Params: + cb: The function to retry. + retry_on: A callback that takes the result of the function and returns + True if the function should be retried. A function throwing an + exception is always retried. + wait_plan: A list of waiting delays between retries in seconds. The + maximum number of retries is len(wait_plan). + """ + retry_on = retry_on or (lambda x: False) + wait_plan = list(wait_plan or []) + wait_plan.reverse() + while True: + got_exception = False + try: + result = cb() + except NoRetryException as e: + raise e + except Exception as e: + got_exception = e + if got_exception or retry_on(result): + if not wait_plan: # pragma: no cover + raise Exception("Retried too often. Giving up. Reason: %s" % + str(got_exception)) + wait_time = wait_plan.pop() + print "Waiting for %f seconds." % wait_time + self._side_effect_handler.Sleep(wait_time) + print "Retrying..." + else: + return result + + def ReadLine(self, default=None): + # Don't prompt in forced mode. + if self._options.force_readline_defaults and default is not None: + print "%s (forced)" % default + return default + else: + return self._side_effect_handler.ReadLine() + + def Command(self, name, args, cwd=None): + cmd = lambda: self._side_effect_handler.Command( + name, args, "", True, cwd=cwd or self.default_cwd) + return self.Retry(cmd, None, [5]) + + def Git(self, args="", prefix="", pipe=True, retry_on=None, cwd=None): + cmd = lambda: self._side_effect_handler.Command( + "git", args, prefix, pipe, cwd=cwd or self.default_cwd) + result = self.Retry(cmd, retry_on, [5, 30]) + if result is None: + raise GitFailedException("'git %s' failed." % args) + return result + + def Editor(self, args): + if self._options.requires_editor: + return self._side_effect_handler.Command( + os.environ["EDITOR"], + args, + pipe=False, + cwd=self.default_cwd) + + def ReadURL(self, url, params=None, retry_on=None, wait_plan=None): + wait_plan = wait_plan or [3, 60, 600] + cmd = lambda: self._side_effect_handler.ReadURL(url, params) + return self.Retry(cmd, retry_on, wait_plan) + + def GetDate(self): + return self._side_effect_handler.GetDate() + + def Die(self, msg=""): + if msg != "": + print "Error: %s" % msg + print "Exiting" + raise Exception(msg) + + def DieNoManualMode(self, msg=""): + if not self._options.manual: # pragma: no cover + msg = msg or "Only available in manual mode." + self.Die(msg) + + def Confirm(self, msg): + print "%s [Y/n] " % msg, + answer = self.ReadLine(default="Y") + return answer == "" or answer == "Y" or answer == "y" + + def DeleteBranch(self, name): + for line in self.GitBranch().splitlines(): + if re.match(r"\*?\s*%s$" % re.escape(name), line): + msg = "Branch %s exists, do you want to delete it?" % name + if self.Confirm(msg): + self.GitDeleteBranch(name) + print "Branch %s deleted." % name + else: + msg = "Can't continue. Please delete branch %s and try again." % name + self.Die(msg) + + def InitialEnvironmentChecks(self, cwd): + # Cancel if this is not a git checkout. + if not os.path.exists(os.path.join(cwd, ".git")): # pragma: no cover + self.Die("This is not a git checkout, this script won't work for you.") + + # Cancel if EDITOR is unset or not executable. + if (self._options.requires_editor and (not os.environ.get("EDITOR") or + self.Command( + "which", os.environ["EDITOR"]) is None)): # pragma: no cover + self.Die("Please set your EDITOR environment variable, you'll need it.") + + def CommonPrepare(self): + # Check for a clean workdir. + if not self.GitIsWorkdirClean(): # pragma: no cover + self.Die("Workspace is not clean. Please commit or undo your changes.") + + # Persist current branch. + self["current_branch"] = self.GitCurrentBranch() + + # Fetch unfetched revisions. + self.vc.Fetch() + + def PrepareBranch(self): + # Delete the branch that will be created later if it exists already. + self.DeleteBranch(self._config["BRANCHNAME"]) + + def CommonCleanup(self): + if ' ' in self["current_branch"]: + self.GitCheckout('master') + else: + self.GitCheckout(self["current_branch"]) + if self._config["BRANCHNAME"] != self["current_branch"]: + self.GitDeleteBranch(self._config["BRANCHNAME"]) + + # Clean up all temporary files. + for f in glob.iglob("%s*" % self._config["PERSISTFILE_BASENAME"]): + if os.path.isfile(f): + os.remove(f) + if os.path.isdir(f): + shutil.rmtree(f) + + def ReadAndPersistVersion(self, prefix=""): + def ReadAndPersist(var_name, def_name): + match = re.match(r"^#define %s\s+(\d*)" % def_name, line) + if match: + value = match.group(1) + self["%s%s" % (prefix, var_name)] = value + for line in LinesInFile(os.path.join(self.default_cwd, VERSION_FILE)): + for (var_name, def_name) in [("major", "MAJOR_VERSION"), + ("minor", "MINOR_VERSION"), + ("build", "BUILD_NUMBER"), + ("patch", "PATCH_LEVEL")]: + ReadAndPersist(var_name, def_name) + + def WaitForLGTM(self): + print ("Please wait for an LGTM, then type \"LGTM<Return>\" to commit " + "your change. (If you need to iterate on the patch or double check " + "that it's sane, do so in another shell, but remember to not " + "change the headline of the uploaded CL.") + answer = "" + while answer != "LGTM": + print "> ", + answer = self.ReadLine(None if self._options.wait_for_lgtm else "LGTM") + if answer != "LGTM": + print "That was not 'LGTM'." + + def WaitForResolvingConflicts(self, patch_file): + print("Applying the patch \"%s\" failed. Either type \"ABORT<Return>\", " + "or resolve the conflicts, stage *all* touched files with " + "'git add', and type \"RESOLVED<Return>\"") + self.DieNoManualMode() + answer = "" + while answer != "RESOLVED": + if answer == "ABORT": + self.Die("Applying the patch failed.") + if answer != "": + print "That was not 'RESOLVED' or 'ABORT'." + print "> ", + answer = self.ReadLine() + + # Takes a file containing the patch to apply as first argument. + def ApplyPatch(self, patch_file, revert=False): + try: + self.GitApplyPatch(patch_file, revert) + except GitFailedException: + self.WaitForResolvingConflicts(patch_file) + + def GetVersionTag(self, revision): + tag = self.Git("describe --tags %s" % revision).strip() + if VERSION_RE.match(tag): + return tag + else: + return None + + def GetRecentReleases(self, max_age): + # Make sure tags are fetched. + self.Git("fetch origin +refs/tags/*:refs/tags/*") + + # Current timestamp. + time_now = int(self._side_effect_handler.GetUTCStamp()) + + # List every tag from a given period. + revisions = self.Git("rev-list --max-age=%d --tags" % + int(time_now - max_age)).strip() + + # Filter out revisions who's tag is off by one or more commits. + return filter(lambda r: self.GetVersionTag(r), revisions.splitlines()) + + def GetLatestVersion(self): + # Use cached version if available. + if self["latest_version"]: + return self["latest_version"] + + # Make sure tags are fetched. + self.Git("fetch origin +refs/tags/*:refs/tags/*") + version = sorted(filter(VERSION_RE.match, self.vc.GetTags()), + key=SortingKey, reverse=True)[0] + self["latest_version"] = version + return version + + def GetLatestRelease(self): + """The latest release is the git hash of the latest tagged version. + + This revision should be rolled into chromium. + """ + latest_version = self.GetLatestVersion() + + # The latest release. + latest_hash = self.GitLog(n=1, format="%H", branch=latest_version) + assert latest_hash + return latest_hash + + def GetLatestReleaseBase(self, version=None): + """The latest release base is the latest revision that is covered in the + last change log file. It doesn't include cherry-picked patches. + """ + latest_version = version or self.GetLatestVersion() + + # Strip patch level if it exists. + latest_version = ".".join(latest_version.split(".")[:3]) + + # The latest release base. + latest_hash = self.GitLog(n=1, format="%H", branch=latest_version) + assert latest_hash + + title = self.GitLog(n=1, format="%s", git_hash=latest_hash) + match = PUSH_MSG_GIT_RE.match(title) + if match: + # Legacy: In the old process there's one level of indirection. The + # version is on the candidates branch and points to the real release + # base on master through the commit message. + return match.group("git_rev") + match = PUSH_MSG_NEW_RE.match(title) + if match: + # This is a new-style v8 version branched from master. The commit + # "latest_hash" is the version-file change. Its parent is the release + # base on master. + return self.GitLog(n=1, format="%H", git_hash="%s^" % latest_hash) + + self.Die("Unknown latest release: %s" % latest_hash) + + def ArrayToVersion(self, prefix): + return ".".join([self[prefix + "major"], + self[prefix + "minor"], + self[prefix + "build"], + self[prefix + "patch"]]) + + def StoreVersion(self, version, prefix): + version_parts = version.split(".") + if len(version_parts) == 3: + version_parts.append("0") + major, minor, build, patch = version_parts + self[prefix + "major"] = major + self[prefix + "minor"] = minor + self[prefix + "build"] = build + self[prefix + "patch"] = patch + + def SetVersion(self, version_file, prefix): + output = "" + for line in FileToText(version_file).splitlines(): + if line.startswith("#define MAJOR_VERSION"): + line = re.sub("\d+$", self[prefix + "major"], line) + elif line.startswith("#define MINOR_VERSION"): + line = re.sub("\d+$", self[prefix + "minor"], line) + elif line.startswith("#define BUILD_NUMBER"): + line = re.sub("\d+$", self[prefix + "build"], line) + elif line.startswith("#define PATCH_LEVEL"): + line = re.sub("\d+$", self[prefix + "patch"], line) + elif (self[prefix + "candidate"] and + line.startswith("#define IS_CANDIDATE_VERSION")): + line = re.sub("\d+$", self[prefix + "candidate"], line) + output += "%s\n" % line + TextToFile(output, version_file) + + +class BootstrapStep(Step): + MESSAGE = "Bootstapping v8 checkout." + + def RunStep(self): + if os.path.realpath(self.default_cwd) == os.path.realpath(V8_BASE): + self.Die("Can't use v8 checkout with calling script as work checkout.") + # Directory containing the working v8 checkout. + if not os.path.exists(self._options.work_dir): + os.makedirs(self._options.work_dir) + if not os.path.exists(self.default_cwd): + self.Command("fetch", "v8", cwd=self._options.work_dir) + + +class UploadStep(Step): + MESSAGE = "Upload for code review." + + def RunStep(self): + if self._options.reviewer: + print "Using account %s for review." % self._options.reviewer + reviewer = self._options.reviewer + else: + print "Please enter the email address of a V8 reviewer for your patch: ", + self.DieNoManualMode("A reviewer must be specified in forced mode.") + reviewer = self.ReadLine() + self.GitUpload(reviewer, self._options.author, self._options.force_upload, + bypass_hooks=self._options.bypass_upload_hooks, + cc=self._options.cc) + + +class DetermineV8Sheriff(Step): + MESSAGE = "Determine the V8 sheriff for code review." + + def RunStep(self): + self["sheriff"] = None + if not self._options.sheriff: # pragma: no cover + return + + try: + # The googlers mapping maps @google.com accounts to @chromium.org + # accounts. + googlers = imp.load_source('googlers_mapping', + self._options.googlers_mapping) + googlers = googlers.list_to_dict(googlers.get_list()) + except: # pragma: no cover + print "Skip determining sheriff without googler mapping." + return + + # The sheriff determined by the rotation on the waterfall has a + # @google.com account. + url = "https://chromium-build.appspot.com/p/chromium/sheriff_v8.js" + match = re.match(r"document\.write\('(\w+)'\)", self.ReadURL(url)) + + # If "channel is sheriff", we can't match an account. + if match: + g_name = match.group(1) + self["sheriff"] = googlers.get(g_name + "@google.com", + g_name + "@chromium.org") + self._options.reviewer = self["sheriff"] + print "Found active sheriff: %s" % self["sheriff"] + else: + print "No active sheriff found." + + +def MakeStep(step_class=Step, number=0, state=None, config=None, + options=None, side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER): + # Allow to pass in empty dictionaries. + state = state if state is not None else {} + config = config if config is not None else {} + + try: + message = step_class.MESSAGE + except AttributeError: + message = step_class.__name__ + + return step_class(message, number=number, config=config, + state=state, options=options, + handler=side_effect_handler) + + +class ScriptsBase(object): + def __init__(self, + config=None, + side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER, + state=None): + self._config = config or self._Config() + self._side_effect_handler = side_effect_handler + self._state = state if state is not None else {} + + def _Description(self): + return None + + def _PrepareOptions(self, parser): + pass + + def _ProcessOptions(self, options): + return True + + def _Steps(self): # pragma: no cover + raise Exception("Not implemented.") + + def _Config(self): + return {} + + def MakeOptions(self, args=None): + parser = argparse.ArgumentParser(description=self._Description()) + parser.add_argument("-a", "--author", default="", + help="The author email used for rietveld.") + parser.add_argument("--dry-run", default=False, action="store_true", + help="Perform only read-only actions.") + parser.add_argument("-g", "--googlers-mapping", + help="Path to the script mapping google accounts.") + parser.add_argument("-r", "--reviewer", default="", + help="The account name to be used for reviews.") + parser.add_argument("--sheriff", default=False, action="store_true", + help=("Determine current sheriff to review CLs. On " + "success, this will overwrite the reviewer " + "option.")) + parser.add_argument("-s", "--step", + help="Specify the step where to start work. Default: 0.", + default=0, type=int) + parser.add_argument("--work-dir", + help=("Location where to bootstrap a working v8 " + "checkout.")) + self._PrepareOptions(parser) + + if args is None: # pragma: no cover + options = parser.parse_args() + else: + options = parser.parse_args(args) + + # Process common options. + if options.step < 0: # pragma: no cover + print "Bad step number %d" % options.step + parser.print_help() + return None + if options.sheriff and not options.googlers_mapping: # pragma: no cover + print "To determine the current sheriff, requires the googler mapping" + parser.print_help() + return None + + # Defaults for options, common to all scripts. + options.manual = getattr(options, "manual", True) + options.force = getattr(options, "force", False) + options.bypass_upload_hooks = False + + # Derived options. + options.requires_editor = not options.force + options.wait_for_lgtm = not options.force + options.force_readline_defaults = not options.manual + options.force_upload = not options.manual + + # Process script specific options. + if not self._ProcessOptions(options): + parser.print_help() + return None + + if not options.work_dir: + options.work_dir = "/tmp/v8-release-scripts-work-dir" + return options + + def RunSteps(self, step_classes, args=None): + options = self.MakeOptions(args) + if not options: + return 1 + + state_file = "%s-state.json" % self._config["PERSISTFILE_BASENAME"] + if options.step == 0 and os.path.exists(state_file): + os.remove(state_file) + + steps = [] + for (number, step_class) in enumerate([BootstrapStep] + step_classes): + steps.append(MakeStep(step_class, number, self._state, self._config, + options, self._side_effect_handler)) + for step in steps[options.step:]: + if step.Run(): + return 0 + return 0 + + def Run(self, args=None): + return self.RunSteps(self._Steps(), args) diff --git a/deps/v8/tools/release/create_release.py b/deps/v8/tools/release/create_release.py new file mode 100755 index 0000000000..44c10d9b30 --- /dev/null +++ b/deps/v8/tools/release/create_release.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python +# Copyright 2015 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import os +import sys +import tempfile +import urllib2 + +from common_includes import * + + +class Preparation(Step): + MESSAGE = "Preparation." + + def RunStep(self): + fetchspecs = [ + "+refs/heads/*:refs/heads/*", + "+refs/pending/*:refs/pending/*", + "+refs/pending-tags/*:refs/pending-tags/*", + ] + self.Git("fetch origin %s" % " ".join(fetchspecs)) + self.GitCheckout("origin/master") + self.DeleteBranch("work-branch") + + +class PrepareBranchRevision(Step): + MESSAGE = "Check from which revision to branch off." + + def RunStep(self): + if self._options.revision: + self["push_hash"], tree_object = self.GitLog( + n=1, format="\"%H %T\"", git_hash=self._options.revision).split(" ") + else: + self["push_hash"], tree_object = self.GitLog( + n=1, format="\"%H %T\"", branch="origin/master").split(" ") + print "Release revision %s" % self["push_hash"] + assert self["push_hash"] + + pending_tuples = self.GitLog( + n=200, format="\"%H %T\"", branch="refs/pending/heads/master") + for hsh, tree in map(lambda s: s.split(" "), pending_tuples.splitlines()): + if tree == tree_object: + self["pending_hash"] = hsh + break + print "Pending release revision %s" % self["pending_hash"] + assert self["pending_hash"] + + +class IncrementVersion(Step): + MESSAGE = "Increment version number." + + def RunStep(self): + latest_version = self.GetLatestVersion() + + # The version file on master can be used to bump up major/minor at + # branch time. + self.GitCheckoutFile(VERSION_FILE, self.vc.RemoteMasterBranch()) + self.ReadAndPersistVersion("master_") + master_version = self.ArrayToVersion("master_") + + # Use the highest version from master or from tags to determine the new + # version. + authoritative_version = sorted( + [master_version, latest_version], key=SortingKey)[1] + self.StoreVersion(authoritative_version, "authoritative_") + + # Variables prefixed with 'new_' contain the new version numbers for the + # ongoing candidates push. + self["new_major"] = self["authoritative_major"] + self["new_minor"] = self["authoritative_minor"] + self["new_build"] = str(int(self["authoritative_build"]) + 1) + + # Make sure patch level is 0 in a new push. + self["new_patch"] = "0" + + # The new version is not a candidate. + self["new_candidate"] = "0" + + self["version"] = "%s.%s.%s" % (self["new_major"], + self["new_minor"], + self["new_build"]) + + print ("Incremented version to %s" % self["version"]) + + +class DetectLastRelease(Step): + MESSAGE = "Detect commit ID of last release base." + + def RunStep(self): + self["last_push_master"] = self.GetLatestReleaseBase() + + +class PrepareChangeLog(Step): + MESSAGE = "Prepare raw ChangeLog entry." + + def Reload(self, body): + """Attempts to reload the commit message from rietveld in order to allow + late changes to the LOG flag. Note: This is brittle to future changes of + the web page name or structure. + """ + match = re.search(r"^Review URL: https://codereview\.chromium\.org/(\d+)$", + body, flags=re.M) + if match: + cl_url = ("https://codereview.chromium.org/%s/description" + % match.group(1)) + try: + # Fetch from Rietveld but only retry once with one second delay since + # there might be many revisions. + body = self.ReadURL(cl_url, wait_plan=[1]) + except urllib2.URLError: # pragma: no cover + pass + return body + + def RunStep(self): + self["date"] = self.GetDate() + output = "%s: Version %s\n\n" % (self["date"], self["version"]) + TextToFile(output, self.Config("CHANGELOG_ENTRY_FILE")) + commits = self.GitLog(format="%H", + git_hash="%s..%s" % (self["last_push_master"], + self["push_hash"])) + + # Cache raw commit messages. + commit_messages = [ + [ + self.GitLog(n=1, format="%s", git_hash=commit), + self.Reload(self.GitLog(n=1, format="%B", git_hash=commit)), + self.GitLog(n=1, format="%an", git_hash=commit), + ] for commit in commits.splitlines() + ] + + # Auto-format commit messages. + body = MakeChangeLogBody(commit_messages, auto_format=True) + AppendToFile(body, self.Config("CHANGELOG_ENTRY_FILE")) + + msg = (" Performance and stability improvements on all platforms." + "\n#\n# The change log above is auto-generated. Please review if " + "all relevant\n# commit messages from the list below are included." + "\n# All lines starting with # will be stripped.\n#\n") + AppendToFile(msg, self.Config("CHANGELOG_ENTRY_FILE")) + + # Include unformatted commit messages as a reference in a comment. + comment_body = MakeComment(MakeChangeLogBody(commit_messages)) + AppendToFile(comment_body, self.Config("CHANGELOG_ENTRY_FILE")) + + +class EditChangeLog(Step): + MESSAGE = "Edit ChangeLog entry." + + def RunStep(self): + print ("Please press <Return> to have your EDITOR open the ChangeLog " + "entry, then edit its contents to your liking. When you're done, " + "save the file and exit your EDITOR. ") + self.ReadLine(default="") + self.Editor(self.Config("CHANGELOG_ENTRY_FILE")) + + # Strip comments and reformat with correct indentation. + changelog_entry = FileToText(self.Config("CHANGELOG_ENTRY_FILE")).rstrip() + changelog_entry = StripComments(changelog_entry) + changelog_entry = "\n".join(map(Fill80, changelog_entry.splitlines())) + changelog_entry = changelog_entry.lstrip() + + if changelog_entry == "": # pragma: no cover + self.Die("Empty ChangeLog entry.") + + # Safe new change log for adding it later to the candidates patch. + TextToFile(changelog_entry, self.Config("CHANGELOG_ENTRY_FILE")) + + +class MakeBranch(Step): + MESSAGE = "Create the branch." + + def RunStep(self): + self.Git("reset --hard origin/master") + self.Git("checkout -b work-branch %s" % self["pending_hash"]) + self.GitCheckoutFile(CHANGELOG_FILE, self["latest_version"]) + self.GitCheckoutFile(VERSION_FILE, self["latest_version"]) + + +class AddChangeLog(Step): + MESSAGE = "Add ChangeLog changes to release branch." + + def RunStep(self): + changelog_entry = FileToText(self.Config("CHANGELOG_ENTRY_FILE")) + old_change_log = FileToText(os.path.join(self.default_cwd, CHANGELOG_FILE)) + new_change_log = "%s\n\n\n%s" % (changelog_entry, old_change_log) + TextToFile(new_change_log, os.path.join(self.default_cwd, CHANGELOG_FILE)) + + +class SetVersion(Step): + MESSAGE = "Set correct version for candidates." + + def RunStep(self): + self.SetVersion(os.path.join(self.default_cwd, VERSION_FILE), "new_") + + +class CommitBranch(Step): + MESSAGE = "Commit version and changelog to new branch." + + def RunStep(self): + # Convert the ChangeLog entry to commit message format. + text = FileToText(self.Config("CHANGELOG_ENTRY_FILE")) + + # Remove date and trailing white space. + text = re.sub(r"^%s: " % self["date"], "", text.rstrip()) + + # Remove indentation and merge paragraphs into single long lines, keeping + # empty lines between them. + def SplitMapJoin(split_text, fun, join_text): + return lambda text: join_text.join(map(fun, text.split(split_text))) + text = SplitMapJoin( + "\n\n", SplitMapJoin("\n", str.strip, " "), "\n\n")(text) + + if not text: # pragma: no cover + self.Die("Commit message editing failed.") + self["commit_title"] = text.splitlines()[0] + TextToFile(text, self.Config("COMMITMSG_FILE")) + + self.GitCommit(file_name = self.Config("COMMITMSG_FILE")) + os.remove(self.Config("COMMITMSG_FILE")) + os.remove(self.Config("CHANGELOG_ENTRY_FILE")) + + +class PushBranch(Step): + MESSAGE = "Push changes." + + def RunStep(self): + pushspecs = [ + "refs/heads/work-branch:refs/pending/heads/%s" % self["version"], + "%s:refs/pending-tags/heads/%s" % + (self["pending_hash"], self["version"]), + "%s:refs/heads/%s" % (self["push_hash"], self["version"]), + ] + cmd = "push origin %s" % " ".join(pushspecs) + if self._options.dry_run: + print "Dry run. Command:\ngit %s" % cmd + else: + self.Git(cmd) + + +class TagRevision(Step): + MESSAGE = "Tag the new revision." + + def RunStep(self): + if self._options.dry_run: + print ("Dry run. Tagging \"%s\" with %s" % + (self["commit_title"], self["version"])) + else: + self.vc.Tag(self["version"], + "origin/%s" % self["version"], + self["commit_title"]) + + +class CleanUp(Step): + MESSAGE = "Done!" + + def RunStep(self): + print("Congratulations, you have successfully created version %s." + % self["version"]) + + self.GitCheckout("origin/master") + self.DeleteBranch("work-branch") + self.Git("gc") + + +class CreateRelease(ScriptsBase): + def _PrepareOptions(self, parser): + group = parser.add_mutually_exclusive_group() + group.add_argument("-f", "--force", + help="Don't prompt the user.", + default=True, action="store_true") + group.add_argument("-m", "--manual", + help="Prompt the user at every important step.", + default=False, action="store_true") + parser.add_argument("-R", "--revision", + help="The git commit ID to push (defaults to HEAD).") + + def _ProcessOptions(self, options): # pragma: no cover + if not options.author or not options.reviewer: + print "Reviewer (-r) and author (-a) are required." + return False + return True + + def _Config(self): + return { + "PERSISTFILE_BASENAME": "/tmp/create-releases-tempfile", + "CHANGELOG_ENTRY_FILE": + "/tmp/v8-create-releases-tempfile-changelog-entry", + "COMMITMSG_FILE": "/tmp/v8-create-releases-tempfile-commitmsg", + } + + def _Steps(self): + return [ + Preparation, + PrepareBranchRevision, + IncrementVersion, + DetectLastRelease, + PrepareChangeLog, + EditChangeLog, + MakeBranch, + AddChangeLog, + SetVersion, + CommitBranch, + PushBranch, + TagRevision, + CleanUp, + ] + + +if __name__ == "__main__": # pragma: no cover + sys.exit(CreateRelease().Run()) diff --git a/deps/v8/tools/release/git_recipes.py b/deps/v8/tools/release/git_recipes.py new file mode 100644 index 0000000000..3d2a9ef87d --- /dev/null +++ b/deps/v8/tools/release/git_recipes.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python +# Copyright 2014 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import re + +SHA1_RE = re.compile('^[a-fA-F0-9]{40}$') +ROLL_DEPS_GIT_SVN_ID_RE = re.compile('^git-svn-id: .*@([0-9]+) .*$') + +# Regular expression that matches a single commit footer line. +COMMIT_FOOTER_ENTRY_RE = re.compile(r'([^:]+):\s+(.+)') + +# Footer metadata key for commit position. +COMMIT_POSITION_FOOTER_KEY = 'Cr-Commit-Position' + +# Regular expression to parse a commit position +COMMIT_POSITION_RE = re.compile(r'(.+)@\{#(\d+)\}') + +# Key for the 'git-svn' ID metadata commit footer entry. +GIT_SVN_ID_FOOTER_KEY = 'git-svn-id' + +# e.g., git-svn-id: https://v8.googlecode.com/svn/trunk@23117 +# ce2b1a6d-e550-0410-aec6-3dcde31c8c00 +GIT_SVN_ID_RE = re.compile(r'[^@]+@(\d+)\s+(?:[a-zA-Z0-9\-]+)') + + +# Copied from bot_update.py. +def GetCommitMessageFooterMap(message): + """Returns: (dict) A dictionary of commit message footer entries. + """ + footers = {} + + # Extract the lines in the footer block. + lines = [] + for line in message.strip().splitlines(): + line = line.strip() + if len(line) == 0: + del(lines[:]) + continue + lines.append(line) + + # Parse the footer + for line in lines: + m = COMMIT_FOOTER_ENTRY_RE.match(line) + if not m: + # If any single line isn't valid, the entire footer is invalid. + footers.clear() + return footers + footers[m.group(1)] = m.group(2).strip() + return footers + + +class GitFailedException(Exception): + pass + + +def Strip(f): + def new_f(*args, **kwargs): + result = f(*args, **kwargs) + if result is None: + return result + else: + return result.strip() + return new_f + + +def MakeArgs(l): + """['-a', '', 'abc', ''] -> '-a abc'""" + return " ".join(filter(None, l)) + + +def Quoted(s): + return "\"%s\"" % s + + +class GitRecipesMixin(object): + def GitIsWorkdirClean(self, **kwargs): + return self.Git("status -s -uno", **kwargs).strip() == "" + + @Strip + def GitBranch(self, **kwargs): + return self.Git("branch", **kwargs) + + def GitCreateBranch(self, name, remote="", **kwargs): + assert name + remote_args = ["--upstream", remote] if remote else [] + self.Git(MakeArgs(["new-branch", name] + remote_args), **kwargs) + + def GitDeleteBranch(self, name, **kwargs): + assert name + self.Git(MakeArgs(["branch -D", name]), **kwargs) + + def GitReset(self, name, **kwargs): + assert name + self.Git(MakeArgs(["reset --hard", name]), **kwargs) + + def GitStash(self, **kwargs): + self.Git(MakeArgs(["stash"]), **kwargs) + + def GitRemotes(self, **kwargs): + return map(str.strip, + self.Git(MakeArgs(["branch -r"]), **kwargs).splitlines()) + + def GitCheckout(self, name, **kwargs): + assert name + self.Git(MakeArgs(["checkout -f", name]), **kwargs) + + def GitCheckoutFile(self, name, branch_or_hash, **kwargs): + assert name + assert branch_or_hash + self.Git(MakeArgs(["checkout -f", branch_or_hash, "--", name]), **kwargs) + + def GitCheckoutFileSafe(self, name, branch_or_hash, **kwargs): + try: + self.GitCheckoutFile(name, branch_or_hash, **kwargs) + except GitFailedException: # pragma: no cover + # The file doesn't exist in that revision. + return False + return True + + def GitChangedFiles(self, git_hash, **kwargs): + assert git_hash + try: + files = self.Git(MakeArgs(["diff --name-only", + git_hash, + "%s^" % git_hash]), **kwargs) + return map(str.strip, files.splitlines()) + except GitFailedException: # pragma: no cover + # Git fails using "^" at branch roots. + return [] + + + @Strip + def GitCurrentBranch(self, **kwargs): + for line in self.Git("status -s -b -uno", **kwargs).strip().splitlines(): + match = re.match(r"^## (.+)", line) + if match: return match.group(1) + raise Exception("Couldn't find curent branch.") # pragma: no cover + + @Strip + def GitLog(self, n=0, format="", grep="", git_hash="", parent_hash="", + branch="", reverse=False, **kwargs): + assert not (git_hash and parent_hash) + args = ["log"] + if n > 0: + args.append("-%d" % n) + if format: + args.append("--format=%s" % format) + if grep: + args.append("--grep=\"%s\"" % grep.replace("\"", "\\\"")) + if reverse: + args.append("--reverse") + if git_hash: + args.append(git_hash) + if parent_hash: + args.append("%s^" % parent_hash) + args.append(branch) + return self.Git(MakeArgs(args), **kwargs) + + def GitGetPatch(self, git_hash, **kwargs): + assert git_hash + return self.Git(MakeArgs(["log", "-1", "-p", git_hash]), **kwargs) + + # TODO(machenbach): Unused? Remove. + def GitAdd(self, name, **kwargs): + assert name + self.Git(MakeArgs(["add", Quoted(name)]), **kwargs) + + def GitApplyPatch(self, patch_file, reverse=False, **kwargs): + assert patch_file + args = ["apply --index --reject"] + if reverse: + args.append("--reverse") + args.append(Quoted(patch_file)) + self.Git(MakeArgs(args), **kwargs) + + def GitUpload(self, reviewer="", author="", force=False, cq=False, + bypass_hooks=False, cc="", **kwargs): + args = ["cl upload --send-mail"] + if author: + args += ["--email", Quoted(author)] + if reviewer: + args += ["-r", Quoted(reviewer)] + if force: + args.append("-f") + if cq: + args.append("--use-commit-queue") + if bypass_hooks: + args.append("--bypass-hooks") + if cc: + args += ["--cc", Quoted(cc)] + # TODO(machenbach): Check output in forced mode. Verify that all required + # base files were uploaded, if not retry. + self.Git(MakeArgs(args), pipe=False, **kwargs) + + def GitCommit(self, message="", file_name="", author=None, **kwargs): + assert message or file_name + args = ["commit"] + if file_name: + args += ["-aF", Quoted(file_name)] + if message: + args += ["-am", Quoted(message)] + if author: + args += ["--author", "\"%s <%s>\"" % (author, author)] + self.Git(MakeArgs(args), **kwargs) + + def GitPresubmit(self, **kwargs): + self.Git("cl presubmit", "PRESUBMIT_TREE_CHECK=\"skip\"", **kwargs) + + def GitCLLand(self, **kwargs): + self.Git( + "cl land -f --bypass-hooks", retry_on=lambda x: x is None, **kwargs) + + def GitDiff(self, loc1, loc2, **kwargs): + return self.Git(MakeArgs(["diff", loc1, loc2]), **kwargs) + + def GitPull(self, **kwargs): + self.Git("pull", **kwargs) + + def GitFetchOrigin(self, **kwargs): + self.Git("fetch origin", **kwargs) + + @Strip + # Copied from bot_update.py and modified for svn-like numbers only. + def GetCommitPositionNumber(self, git_hash, **kwargs): + """Dumps the 'git' log for a specific revision and parses out the commit + position number. + + If a commit position metadata key is found, its number will be returned. + + Otherwise, we will search for a 'git-svn' metadata entry. If one is found, + its SVN revision value is returned. + """ + git_log = self.GitLog(format='%B', n=1, git_hash=git_hash, **kwargs) + footer_map = GetCommitMessageFooterMap(git_log) + + # Search for commit position metadata + value = footer_map.get(COMMIT_POSITION_FOOTER_KEY) + if value: + match = COMMIT_POSITION_RE.match(value) + if match: + return match.group(2) + + # Extract the svn revision from 'git-svn' metadata + value = footer_map.get(GIT_SVN_ID_FOOTER_KEY) + if value: + match = GIT_SVN_ID_RE.match(value) + if match: + return match.group(1) + raise GitFailedException("Couldn't determine commit position for %s" % + git_hash) diff --git a/deps/v8/tools/release/merge_to_branch.py b/deps/v8/tools/release/merge_to_branch.py new file mode 100755 index 0000000000..7aa9fb6ab6 --- /dev/null +++ b/deps/v8/tools/release/merge_to_branch.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python +# Copyright 2014 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse +from collections import OrderedDict +import sys + +from common_includes import * + +def IsSvnNumber(rev): + return rev.isdigit() and len(rev) < 8 + +class Preparation(Step): + MESSAGE = "Preparation." + + def RunStep(self): + if os.path.exists(self.Config("ALREADY_MERGING_SENTINEL_FILE")): + if self._options.force: + os.remove(self.Config("ALREADY_MERGING_SENTINEL_FILE")) + elif self._options.step == 0: # pragma: no cover + self.Die("A merge is already in progress") + open(self.Config("ALREADY_MERGING_SENTINEL_FILE"), "a").close() + + self.InitialEnvironmentChecks(self.default_cwd) + if self._options.revert_master: + # FIXME(machenbach): Make revert master obsolete? + self["merge_to_branch"] = "master" + elif self._options.branch: + self["merge_to_branch"] = self._options.branch + else: # pragma: no cover + self.Die("Please specify a branch to merge to") + + self.CommonPrepare() + self.PrepareBranch() + + +class CreateBranch(Step): + MESSAGE = "Create a fresh branch for the patch." + + def RunStep(self): + self.GitCreateBranch(self.Config("BRANCHNAME"), + self.vc.RemoteBranch(self["merge_to_branch"])) + + +class SearchArchitecturePorts(Step): + MESSAGE = "Search for corresponding architecture ports." + + def RunStep(self): + self["full_revision_list"] = list(OrderedDict.fromkeys( + self._options.revisions)) + port_revision_list = [] + for revision in self["full_revision_list"]: + # Search for commits which matches the "Port XXX" pattern. + git_hashes = self.GitLog(reverse=True, format="%H", + grep="Port %s" % revision, + branch=self.vc.RemoteMasterBranch()) + for git_hash in git_hashes.splitlines(): + revision_title = self.GitLog(n=1, format="%s", git_hash=git_hash) + + # Is this revision included in the original revision list? + if git_hash in self["full_revision_list"]: + print("Found port of %s -> %s (already included): %s" + % (revision, git_hash, revision_title)) + else: + print("Found port of %s -> %s: %s" + % (revision, git_hash, revision_title)) + port_revision_list.append(git_hash) + + # Do we find any port? + if len(port_revision_list) > 0: + if self.Confirm("Automatically add corresponding ports (%s)?" + % ", ".join(port_revision_list)): + #: 'y': Add ports to revision list. + self["full_revision_list"].extend(port_revision_list) + + +class CreateCommitMessage(Step): + MESSAGE = "Create commit message." + + def RunStep(self): + + # Stringify: [123, 234] -> "r123, r234" + self["revision_list"] = ", ".join(map(lambda s: "r%s" % s, + self["full_revision_list"])) + + if not self["revision_list"]: # pragma: no cover + self.Die("Revision list is empty.") + + if self._options.revert and not self._options.revert_master: + action_text = "Rollback of %s" + else: + action_text = "Merged %s" + + # The commit message title is added below after the version is specified. + msg_pieces = [ + "\n".join(action_text % s for s in self["full_revision_list"]), + ] + msg_pieces.append("\n\n") + + for commit_hash in self["full_revision_list"]: + patch_merge_desc = self.GitLog(n=1, format="%s", git_hash=commit_hash) + msg_pieces.append("%s\n\n" % patch_merge_desc) + + bugs = [] + for commit_hash in self["full_revision_list"]: + msg = self.GitLog(n=1, git_hash=commit_hash) + for bug in re.findall(r"^[ \t]*BUG[ \t]*=[ \t]*(.*?)[ \t]*$", msg, re.M): + bugs.extend(s.strip() for s in bug.split(",")) + bug_aggregate = ",".join(sorted(filter(lambda s: s and s != "none", bugs))) + if bug_aggregate: + msg_pieces.append("BUG=%s\nLOG=N\n" % bug_aggregate) + + self["new_commit_msg"] = "".join(msg_pieces) + + +class ApplyPatches(Step): + MESSAGE = "Apply patches for selected revisions." + + def RunStep(self): + for commit_hash in self["full_revision_list"]: + print("Applying patch for %s to %s..." + % (commit_hash, self["merge_to_branch"])) + patch = self.GitGetPatch(commit_hash) + TextToFile(patch, self.Config("TEMPORARY_PATCH_FILE")) + self.ApplyPatch(self.Config("TEMPORARY_PATCH_FILE"), self._options.revert) + if self._options.patch: + self.ApplyPatch(self._options.patch, self._options.revert) + + +class PrepareVersion(Step): + MESSAGE = "Prepare version file." + + def RunStep(self): + if self._options.revert_master: + return + # This is used to calculate the patch level increment. + self.ReadAndPersistVersion() + + +class IncrementVersion(Step): + MESSAGE = "Increment version number." + + def RunStep(self): + if self._options.revert_master: + return + new_patch = str(int(self["patch"]) + 1) + if self.Confirm("Automatically increment PATCH_LEVEL? (Saying 'n' will " + "fire up your EDITOR on %s so you can make arbitrary " + "changes. When you're done, save the file and exit your " + "EDITOR.)" % VERSION_FILE): + text = FileToText(os.path.join(self.default_cwd, VERSION_FILE)) + text = MSub(r"(?<=#define PATCH_LEVEL)(?P<space>\s+)\d*$", + r"\g<space>%s" % new_patch, + text) + TextToFile(text, os.path.join(self.default_cwd, VERSION_FILE)) + else: + self.Editor(os.path.join(self.default_cwd, VERSION_FILE)) + self.ReadAndPersistVersion("new_") + self["version"] = "%s.%s.%s.%s" % (self["new_major"], + self["new_minor"], + self["new_build"], + self["new_patch"]) + + +class CommitLocal(Step): + MESSAGE = "Commit to local branch." + + def RunStep(self): + # Add a commit message title. + if self._options.revert and self._options.revert_master: + # TODO(machenbach): Find a better convention if multiple patches are + # reverted in one CL. + self["commit_title"] = "Revert on master" + else: + self["commit_title"] = "Version %s (cherry-pick)" % self["version"] + self["new_commit_msg"] = "%s\n\n%s" % (self["commit_title"], + self["new_commit_msg"]) + TextToFile(self["new_commit_msg"], self.Config("COMMITMSG_FILE")) + self.GitCommit(file_name=self.Config("COMMITMSG_FILE")) + + +class CommitRepository(Step): + MESSAGE = "Commit to the repository." + + def RunStep(self): + self.GitCheckout(self.Config("BRANCHNAME")) + self.WaitForLGTM() + self.GitPresubmit() + self.vc.CLLand() + + +class TagRevision(Step): + MESSAGE = "Create the tag." + + def RunStep(self): + if self._options.revert_master: + return + print "Creating tag %s" % self["version"] + self.vc.Tag(self["version"], + self.vc.RemoteBranch(self["merge_to_branch"]), + self["commit_title"]) + + +class CleanUp(Step): + MESSAGE = "Cleanup." + + def RunStep(self): + self.CommonCleanup() + if not self._options.revert_master: + print "*** SUMMARY ***" + print "version: %s" % self["version"] + print "branch: %s" % self["merge_to_branch"] + if self["revision_list"]: + print "patches: %s" % self["revision_list"] + + +class MergeToBranch(ScriptsBase): + def _Description(self): + return ("Performs the necessary steps to merge revisions from " + "master to other branches, including candidates.") + + def _PrepareOptions(self, parser): + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--branch", help="The branch to merge to.") + group.add_argument("-R", "--revert-master", + help="Revert specified patches from master.", + default=False, action="store_true") + parser.add_argument("revisions", nargs="*", + help="The revisions to merge.") + parser.add_argument("-f", "--force", + help="Delete sentinel file.", + default=False, action="store_true") + parser.add_argument("-m", "--message", + help="A commit message for the patch.") + parser.add_argument("--revert", + help="Revert specified patches.", + default=False, action="store_true") + parser.add_argument("-p", "--patch", + help="A patch file to apply as part of the merge.") + + def _ProcessOptions(self, options): + # TODO(machenbach): Add a test that covers revert from master + if len(options.revisions) < 1: + if not options.patch: + print "Either a patch file or revision numbers must be specified" + return False + if not options.message: + print "You must specify a merge comment if no patches are specified" + return False + options.bypass_upload_hooks = True + # CC ulan to make sure that fixes are merged to Google3. + options.cc = "ulan@chromium.org" + + # Make sure to use git hashes in the new workflows. + for revision in options.revisions: + if (IsSvnNumber(revision) or + (revision[0:1] == "r" and IsSvnNumber(revision[1:]))): + print "Please provide full git hashes of the patches to merge." + print "Got: %s" % revision + return False + return True + + def _Config(self): + return { + "BRANCHNAME": "prepare-merge", + "PERSISTFILE_BASENAME": "/tmp/v8-merge-to-branch-tempfile", + "ALREADY_MERGING_SENTINEL_FILE": + "/tmp/v8-merge-to-branch-tempfile-already-merging", + "TEMPORARY_PATCH_FILE": "/tmp/v8-prepare-merge-tempfile-temporary-patch", + "COMMITMSG_FILE": "/tmp/v8-prepare-merge-tempfile-commitmsg", + } + + def _Steps(self): + return [ + Preparation, + CreateBranch, + SearchArchitecturePorts, + CreateCommitMessage, + ApplyPatches, + PrepareVersion, + IncrementVersion, + CommitLocal, + UploadStep, + CommitRepository, + TagRevision, + CleanUp, + ] + + +if __name__ == "__main__": # pragma: no cover + sys.exit(MergeToBranch().Run()) diff --git a/deps/v8/tools/release/push_to_candidates.py b/deps/v8/tools/release/push_to_candidates.py new file mode 100755 index 0000000000..750794eabd --- /dev/null +++ b/deps/v8/tools/release/push_to_candidates.py @@ -0,0 +1,415 @@ +#!/usr/bin/env python +# Copyright 2013 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse +import os +import sys +import tempfile +import urllib2 + +from common_includes import * + +PUSH_MSG_GIT_SUFFIX = " (based on %s)" + + +class Preparation(Step): + MESSAGE = "Preparation." + + def RunStep(self): + self.InitialEnvironmentChecks(self.default_cwd) + self.CommonPrepare() + + if(self["current_branch"] == self.Config("CANDIDATESBRANCH") + or self["current_branch"] == self.Config("BRANCHNAME")): + print "Warning: Script started on branch %s" % self["current_branch"] + + self.PrepareBranch() + self.DeleteBranch(self.Config("CANDIDATESBRANCH")) + + +class FreshBranch(Step): + MESSAGE = "Create a fresh branch." + + def RunStep(self): + self.GitCreateBranch(self.Config("BRANCHNAME"), + self.vc.RemoteMasterBranch()) + + +class PreparePushRevision(Step): + MESSAGE = "Check which revision to push." + + def RunStep(self): + if self._options.revision: + self["push_hash"] = self._options.revision + else: + self["push_hash"] = self.GitLog(n=1, format="%H", git_hash="HEAD") + if not self["push_hash"]: # pragma: no cover + self.Die("Could not determine the git hash for the push.") + + +class IncrementVersion(Step): + MESSAGE = "Increment version number." + + def RunStep(self): + latest_version = self.GetLatestVersion() + + # The version file on master can be used to bump up major/minor at + # branch time. + self.GitCheckoutFile(VERSION_FILE, self.vc.RemoteMasterBranch()) + self.ReadAndPersistVersion("master_") + master_version = self.ArrayToVersion("master_") + + # Use the highest version from master or from tags to determine the new + # version. + authoritative_version = sorted( + [master_version, latest_version], key=SortingKey)[1] + self.StoreVersion(authoritative_version, "authoritative_") + + # Variables prefixed with 'new_' contain the new version numbers for the + # ongoing candidates push. + self["new_major"] = self["authoritative_major"] + self["new_minor"] = self["authoritative_minor"] + self["new_build"] = str(int(self["authoritative_build"]) + 1) + + # Make sure patch level is 0 in a new push. + self["new_patch"] = "0" + + self["version"] = "%s.%s.%s" % (self["new_major"], + self["new_minor"], + self["new_build"]) + + print ("Incremented version to %s" % self["version"]) + + +class DetectLastRelease(Step): + MESSAGE = "Detect commit ID of last release base." + + def RunStep(self): + if self._options.last_master: + self["last_push_master"] = self._options.last_master + else: + self["last_push_master"] = self.GetLatestReleaseBase() + + +class PrepareChangeLog(Step): + MESSAGE = "Prepare raw ChangeLog entry." + + def Reload(self, body): + """Attempts to reload the commit message from rietveld in order to allow + late changes to the LOG flag. Note: This is brittle to future changes of + the web page name or structure. + """ + match = re.search(r"^Review URL: https://codereview\.chromium\.org/(\d+)$", + body, flags=re.M) + if match: + cl_url = ("https://codereview.chromium.org/%s/description" + % match.group(1)) + try: + # Fetch from Rietveld but only retry once with one second delay since + # there might be many revisions. + body = self.ReadURL(cl_url, wait_plan=[1]) + except urllib2.URLError: # pragma: no cover + pass + return body + + def RunStep(self): + self["date"] = self.GetDate() + output = "%s: Version %s\n\n" % (self["date"], self["version"]) + TextToFile(output, self.Config("CHANGELOG_ENTRY_FILE")) + commits = self.GitLog(format="%H", + git_hash="%s..%s" % (self["last_push_master"], + self["push_hash"])) + + # Cache raw commit messages. + commit_messages = [ + [ + self.GitLog(n=1, format="%s", git_hash=commit), + self.Reload(self.GitLog(n=1, format="%B", git_hash=commit)), + self.GitLog(n=1, format="%an", git_hash=commit), + ] for commit in commits.splitlines() + ] + + # Auto-format commit messages. + body = MakeChangeLogBody(commit_messages, auto_format=True) + AppendToFile(body, self.Config("CHANGELOG_ENTRY_FILE")) + + msg = (" Performance and stability improvements on all platforms." + "\n#\n# The change log above is auto-generated. Please review if " + "all relevant\n# commit messages from the list below are included." + "\n# All lines starting with # will be stripped.\n#\n") + AppendToFile(msg, self.Config("CHANGELOG_ENTRY_FILE")) + + # Include unformatted commit messages as a reference in a comment. + comment_body = MakeComment(MakeChangeLogBody(commit_messages)) + AppendToFile(comment_body, self.Config("CHANGELOG_ENTRY_FILE")) + + +class EditChangeLog(Step): + MESSAGE = "Edit ChangeLog entry." + + def RunStep(self): + print ("Please press <Return> to have your EDITOR open the ChangeLog " + "entry, then edit its contents to your liking. When you're done, " + "save the file and exit your EDITOR. ") + self.ReadLine(default="") + self.Editor(self.Config("CHANGELOG_ENTRY_FILE")) + + # Strip comments and reformat with correct indentation. + changelog_entry = FileToText(self.Config("CHANGELOG_ENTRY_FILE")).rstrip() + changelog_entry = StripComments(changelog_entry) + changelog_entry = "\n".join(map(Fill80, changelog_entry.splitlines())) + changelog_entry = changelog_entry.lstrip() + + if changelog_entry == "": # pragma: no cover + self.Die("Empty ChangeLog entry.") + + # Safe new change log for adding it later to the candidates patch. + TextToFile(changelog_entry, self.Config("CHANGELOG_ENTRY_FILE")) + + +class StragglerCommits(Step): + MESSAGE = ("Fetch straggler commits that sneaked in since this script was " + "started.") + + def RunStep(self): + self.vc.Fetch() + self.GitCheckout(self.vc.RemoteMasterBranch()) + + +class SquashCommits(Step): + MESSAGE = "Squash commits into one." + + def RunStep(self): + # Instead of relying on "git rebase -i", we'll just create a diff, because + # that's easier to automate. + TextToFile(self.GitDiff(self.vc.RemoteCandidateBranch(), + self["push_hash"]), + self.Config("PATCH_FILE")) + + # Convert the ChangeLog entry to commit message format. + text = FileToText(self.Config("CHANGELOG_ENTRY_FILE")) + + # Remove date and trailing white space. + text = re.sub(r"^%s: " % self["date"], "", text.rstrip()) + + # Show the used master hash in the commit message. + suffix = PUSH_MSG_GIT_SUFFIX % self["push_hash"] + text = MSub(r"^(Version \d+\.\d+\.\d+)$", "\\1%s" % suffix, text) + + # Remove indentation and merge paragraphs into single long lines, keeping + # empty lines between them. + def SplitMapJoin(split_text, fun, join_text): + return lambda text: join_text.join(map(fun, text.split(split_text))) + strip = lambda line: line.strip() + text = SplitMapJoin("\n\n", SplitMapJoin("\n", strip, " "), "\n\n")(text) + + if not text: # pragma: no cover + self.Die("Commit message editing failed.") + self["commit_title"] = text.splitlines()[0] + TextToFile(text, self.Config("COMMITMSG_FILE")) + + +class NewBranch(Step): + MESSAGE = "Create a new branch from candidates." + + def RunStep(self): + self.GitCreateBranch(self.Config("CANDIDATESBRANCH"), + self.vc.RemoteCandidateBranch()) + + +class ApplyChanges(Step): + MESSAGE = "Apply squashed changes." + + def RunStep(self): + self.ApplyPatch(self.Config("PATCH_FILE")) + os.remove(self.Config("PATCH_FILE")) + # The change log has been modified by the patch. Reset it to the version + # on candidates and apply the exact changes determined by this + # PrepareChangeLog step above. + self.GitCheckoutFile(CHANGELOG_FILE, self.vc.RemoteCandidateBranch()) + # The version file has been modified by the patch. Reset it to the version + # on candidates. + self.GitCheckoutFile(VERSION_FILE, self.vc.RemoteCandidateBranch()) + + +class CommitSquash(Step): + MESSAGE = "Commit to local candidates branch." + + def RunStep(self): + # Make a first commit with a slightly different title to not confuse + # the tagging. + msg = FileToText(self.Config("COMMITMSG_FILE")).splitlines() + msg[0] = msg[0].replace("(based on", "(squashed - based on") + self.GitCommit(message = "\n".join(msg)) + + +class PrepareVersionBranch(Step): + MESSAGE = "Prepare new branch to commit version and changelog file." + + def RunStep(self): + self.GitCheckout("master") + self.Git("fetch") + self.GitDeleteBranch(self.Config("CANDIDATESBRANCH")) + self.GitCreateBranch(self.Config("CANDIDATESBRANCH"), + self.vc.RemoteCandidateBranch()) + + +class AddChangeLog(Step): + MESSAGE = "Add ChangeLog changes to candidates branch." + + def RunStep(self): + changelog_entry = FileToText(self.Config("CHANGELOG_ENTRY_FILE")) + old_change_log = FileToText(os.path.join(self.default_cwd, CHANGELOG_FILE)) + new_change_log = "%s\n\n\n%s" % (changelog_entry, old_change_log) + TextToFile(new_change_log, os.path.join(self.default_cwd, CHANGELOG_FILE)) + os.remove(self.Config("CHANGELOG_ENTRY_FILE")) + + +class SetVersion(Step): + MESSAGE = "Set correct version for candidates." + + def RunStep(self): + self.SetVersion(os.path.join(self.default_cwd, VERSION_FILE), "new_") + + +class CommitCandidate(Step): + MESSAGE = "Commit version and changelog to local candidates branch." + + def RunStep(self): + self.GitCommit(file_name = self.Config("COMMITMSG_FILE")) + os.remove(self.Config("COMMITMSG_FILE")) + + +class SanityCheck(Step): + MESSAGE = "Sanity check." + + def RunStep(self): + # TODO(machenbach): Run presubmit script here as it is now missing in the + # prepare push process. + if not self.Confirm("Please check if your local checkout is sane: Inspect " + "%s, compile, run tests. Do you want to commit this new candidates " + "revision to the repository?" % VERSION_FILE): + self.Die("Execution canceled.") # pragma: no cover + + +class Land(Step): + MESSAGE = "Land the patch." + + def RunStep(self): + self.vc.CLLand() + + +class TagRevision(Step): + MESSAGE = "Tag the new revision." + + def RunStep(self): + self.vc.Tag( + self["version"], self.vc.RemoteCandidateBranch(), self["commit_title"]) + + +class CleanUp(Step): + MESSAGE = "Done!" + + def RunStep(self): + print("Congratulations, you have successfully created the candidates " + "revision %s." + % self["version"]) + + self.CommonCleanup() + if self.Config("CANDIDATESBRANCH") != self["current_branch"]: + self.GitDeleteBranch(self.Config("CANDIDATESBRANCH")) + + +class PushToCandidates(ScriptsBase): + def _PrepareOptions(self, parser): + group = parser.add_mutually_exclusive_group() + group.add_argument("-f", "--force", + help="Don't prompt the user.", + default=False, action="store_true") + group.add_argument("-m", "--manual", + help="Prompt the user at every important step.", + default=False, action="store_true") + parser.add_argument("-b", "--last-master", + help=("The git commit ID of the last master " + "revision that was pushed to candidates. This is" + " used for the auto-generated ChangeLog entry.")) + parser.add_argument("-l", "--last-push", + help="The git commit ID of the last candidates push.") + parser.add_argument("-R", "--revision", + help="The git commit ID to push (defaults to HEAD).") + + def _ProcessOptions(self, options): # pragma: no cover + if not options.manual and not options.reviewer: + print "A reviewer (-r) is required in (semi-)automatic mode." + return False + if not options.manual and not options.author: + print "Specify your chromium.org email with -a in (semi-)automatic mode." + return False + + options.tbr_commit = not options.manual + return True + + def _Config(self): + return { + "BRANCHNAME": "prepare-push", + "CANDIDATESBRANCH": "candidates-push", + "PERSISTFILE_BASENAME": "/tmp/v8-push-to-candidates-tempfile", + "CHANGELOG_ENTRY_FILE": + "/tmp/v8-push-to-candidates-tempfile-changelog-entry", + "PATCH_FILE": "/tmp/v8-push-to-candidates-tempfile-patch-file", + "COMMITMSG_FILE": "/tmp/v8-push-to-candidates-tempfile-commitmsg", + } + + def _Steps(self): + return [ + Preparation, + FreshBranch, + PreparePushRevision, + IncrementVersion, + DetectLastRelease, + PrepareChangeLog, + EditChangeLog, + StragglerCommits, + SquashCommits, + NewBranch, + ApplyChanges, + CommitSquash, + SanityCheck, + Land, + PrepareVersionBranch, + AddChangeLog, + SetVersion, + CommitCandidate, + Land, + TagRevision, + CleanUp, + ] + + +if __name__ == "__main__": # pragma: no cover + sys.exit(PushToCandidates().Run()) diff --git a/deps/v8/tools/release/releases.py b/deps/v8/tools/release/releases.py new file mode 100755 index 0000000000..0f35e7c88f --- /dev/null +++ b/deps/v8/tools/release/releases.py @@ -0,0 +1,513 @@ +#!/usr/bin/env python +# Copyright 2014 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# This script retrieves the history of all V8 branches and +# their corresponding Chromium revisions. + +# Requires a chromium checkout with branch heads: +# gclient sync --with_branch_heads +# gclient fetch + +import argparse +import csv +import itertools +import json +import os +import re +import sys + +from common_includes import * + +CONFIG = { + "BRANCHNAME": "retrieve-v8-releases", + "PERSISTFILE_BASENAME": "/tmp/v8-releases-tempfile", +} + +# Expression for retrieving the bleeding edge revision from a commit message. +PUSH_MSG_SVN_RE = re.compile(r".* \(based on bleeding_edge revision r(\d+)\)$") +PUSH_MSG_GIT_RE = re.compile(r".* \(based on ([a-fA-F0-9]+)\)$") + +# Expression for retrieving the merged patches from a merge commit message +# (old and new format). +MERGE_MESSAGE_RE = re.compile(r"^.*[M|m]erged (.+)(\)| into).*$", re.M) + +CHERRY_PICK_TITLE_GIT_RE = re.compile(r"^.* \(cherry\-pick\)\.?$") + +# New git message for cherry-picked CLs. One message per line. +MERGE_MESSAGE_GIT_RE = re.compile(r"^Merged ([a-fA-F0-9]+)\.?$") + +# Expression for retrieving reverted patches from a commit message (old and +# new format). +ROLLBACK_MESSAGE_RE = re.compile(r"^.*[R|r]ollback of (.+)(\)| in).*$", re.M) + +# New git message for reverted CLs. One message per line. +ROLLBACK_MESSAGE_GIT_RE = re.compile(r"^Rollback of ([a-fA-F0-9]+)\.?$") + +# Expression for retrieving the code review link. +REVIEW_LINK_RE = re.compile(r"^Review URL: (.+)$", re.M) + +# Expression with three versions (historical) for extracting the v8 revision +# from the chromium DEPS file. +DEPS_RE = re.compile(r"""^\s*(?:["']v8_revision["']: ["']""" + """|\(Var\("googlecode_url"\) % "v8"\) \+ "\/trunk@""" + """|"http\:\/\/v8\.googlecode\.com\/svn\/trunk@)""" + """([^"']+)["'].*$""", re.M) + +# Expression to pick tag and revision for bleeding edge tags. To be used with +# output of 'svn log'. +BLEEDING_EDGE_TAGS_RE = re.compile( + r"A \/tags\/([^\s]+) \(from \/branches\/bleeding_edge\:(\d+)\)") + + +def SortBranches(branches): + """Sort branches with version number names.""" + return sorted(branches, key=SortingKey, reverse=True) + + +def FilterDuplicatesAndReverse(cr_releases): + """Returns the chromium releases in reverse order filtered by v8 revision + duplicates. + + cr_releases is a list of [cr_rev, v8_hsh] reverse-sorted by cr_rev. + """ + last = "" + result = [] + for release in reversed(cr_releases): + if last == release[1]: + continue + last = release[1] + result.append(release) + return result + + +def BuildRevisionRanges(cr_releases): + """Returns a mapping of v8 revision -> chromium ranges. + The ranges are comma-separated, each range has the form R1:R2. The newest + entry is the only one of the form R1, as there is no end range. + + cr_releases is a list of [cr_rev, v8_hsh] reverse-sorted by cr_rev. + cr_rev either refers to a chromium commit position or a chromium branch + number. + """ + range_lists = {} + cr_releases = FilterDuplicatesAndReverse(cr_releases) + + # Visit pairs of cr releases from oldest to newest. + for cr_from, cr_to in itertools.izip( + cr_releases, itertools.islice(cr_releases, 1, None)): + + # Assume the chromium revisions are all different. + assert cr_from[0] != cr_to[0] + + ran = "%s:%d" % (cr_from[0], int(cr_to[0]) - 1) + + # Collect the ranges in lists per revision. + range_lists.setdefault(cr_from[1], []).append(ran) + + # Add the newest revision. + if cr_releases: + range_lists.setdefault(cr_releases[-1][1], []).append(cr_releases[-1][0]) + + # Stringify and comma-separate the range lists. + return dict((hsh, ", ".join(ran)) for hsh, ran in range_lists.iteritems()) + + +def MatchSafe(match): + if match: + return match.group(1) + else: + return "" + + +class Preparation(Step): + MESSAGE = "Preparation." + + def RunStep(self): + self.CommonPrepare() + self.PrepareBranch() + + +class RetrieveV8Releases(Step): + MESSAGE = "Retrieve all V8 releases." + + def ExceedsMax(self, releases): + return (self._options.max_releases > 0 + and len(releases) > self._options.max_releases) + + def GetMasterHashFromPush(self, title): + return MatchSafe(PUSH_MSG_GIT_RE.match(title)) + + def GetMergedPatches(self, body): + patches = MatchSafe(MERGE_MESSAGE_RE.search(body)) + if not patches: + patches = MatchSafe(ROLLBACK_MESSAGE_RE.search(body)) + if patches: + # Indicate reverted patches with a "-". + patches = "-%s" % patches + return patches + + def GetMergedPatchesGit(self, body): + patches = [] + for line in body.splitlines(): + patch = MatchSafe(MERGE_MESSAGE_GIT_RE.match(line)) + if patch: + patches.append(patch) + patch = MatchSafe(ROLLBACK_MESSAGE_GIT_RE.match(line)) + if patch: + patches.append("-%s" % patch) + return ", ".join(patches) + + + def GetReleaseDict( + self, git_hash, master_position, master_hash, branch, version, + patches, cl_body): + revision = self.GetCommitPositionNumber(git_hash) + return { + # The cr commit position number on the branch. + "revision": revision, + # The git revision on the branch. + "revision_git": git_hash, + # The cr commit position number on master. + "master_position": master_position, + # The same for git. + "master_hash": master_hash, + # The branch name. + "branch": branch, + # The version for displaying in the form 3.26.3 or 3.26.3.12. + "version": version, + # The date of the commit. + "date": self.GitLog(n=1, format="%ci", git_hash=git_hash), + # Merged patches if available in the form 'r1234, r2345'. + "patches_merged": patches, + # Default for easier output formatting. + "chromium_revision": "", + # Default for easier output formatting. + "chromium_branch": "", + # Link to the CL on code review. Candiates pushes are not uploaded, + # so this field will be populated below with the recent roll CL link. + "review_link": MatchSafe(REVIEW_LINK_RE.search(cl_body)), + # Link to the commit message on google code. + "revision_link": ("https://code.google.com/p/v8/source/detail?r=%s" + % revision), + } + + def GetRelease(self, git_hash, branch): + self.ReadAndPersistVersion() + base_version = [self["major"], self["minor"], self["build"]] + version = ".".join(base_version) + body = self.GitLog(n=1, format="%B", git_hash=git_hash) + + patches = "" + if self["patch"] != "0": + version += ".%s" % self["patch"] + if CHERRY_PICK_TITLE_GIT_RE.match(body.splitlines()[0]): + patches = self.GetMergedPatchesGit(body) + else: + patches = self.GetMergedPatches(body) + + if SortingKey("4.2.69") <= SortingKey(version): + master_hash = self.GetLatestReleaseBase(version=version) + else: + # Legacy: Before version 4.2.69, the master revision was determined + # by commit message. + title = self.GitLog(n=1, format="%s", git_hash=git_hash) + master_hash = self.GetMasterHashFromPush(title) + master_position = "" + if master_hash: + master_position = self.GetCommitPositionNumber(master_hash) + return self.GetReleaseDict( + git_hash, master_position, master_hash, branch, version, + patches, body), self["patch"] + + def GetReleasesFromBranch(self, branch): + self.GitReset(self.vc.RemoteBranch(branch)) + if branch == self.vc.MasterBranch(): + return self.GetReleasesFromMaster() + + releases = [] + try: + for git_hash in self.GitLog(format="%H").splitlines(): + if VERSION_FILE not in self.GitChangedFiles(git_hash): + continue + if self.ExceedsMax(releases): + break # pragma: no cover + if not self.GitCheckoutFileSafe(VERSION_FILE, git_hash): + break # pragma: no cover + + release, patch_level = self.GetRelease(git_hash, branch) + releases.append(release) + + # Follow branches only until their creation point. + # TODO(machenbach): This omits patches if the version file wasn't + # manipulated correctly. Find a better way to detect the point where + # the parent of the branch head leads to the trunk branch. + if branch != self.vc.CandidateBranch() and patch_level == "0": + break + + # Allow Ctrl-C interrupt. + except (KeyboardInterrupt, SystemExit): # pragma: no cover + pass + + # Clean up checked-out version file. + self.GitCheckoutFileSafe(VERSION_FILE, "HEAD") + return releases + + def GetReleaseFromRevision(self, revision): + releases = [] + try: + if (VERSION_FILE not in self.GitChangedFiles(revision) or + not self.GitCheckoutFileSafe(VERSION_FILE, revision)): + print "Skipping revision %s" % revision + return [] # pragma: no cover + + branches = map( + str.strip, + self.Git("branch -r --contains %s" % revision).strip().splitlines(), + ) + branch = "" + for b in branches: + if b.startswith("origin/"): + branch = b.split("origin/")[1] + break + if b.startswith("branch-heads/"): + branch = b.split("branch-heads/")[1] + break + else: + print "Could not determine branch for %s" % revision + + release, _ = self.GetRelease(revision, branch) + releases.append(release) + + # Allow Ctrl-C interrupt. + except (KeyboardInterrupt, SystemExit): # pragma: no cover + pass + + # Clean up checked-out version file. + self.GitCheckoutFileSafe(VERSION_FILE, "HEAD") + return releases + + + def RunStep(self): + self.GitCreateBranch(self._config["BRANCHNAME"]) + releases = [] + if self._options.branch == 'recent': + # List every release from the last 7 days. + revisions = self.GetRecentReleases(max_age=7 * 24 * 60 * 60) + for revision in revisions: + releases += self.GetReleaseFromRevision(revision) + elif self._options.branch == 'all': # pragma: no cover + # Retrieve the full release history. + for branch in self.vc.GetBranches(): + releases += self.GetReleasesFromBranch(branch) + releases += self.GetReleasesFromBranch(self.vc.CandidateBranch()) + releases += self.GetReleasesFromBranch(self.vc.MasterBranch()) + else: # pragma: no cover + # Retrieve history for a specified branch. + assert self._options.branch in (self.vc.GetBranches() + + [self.vc.CandidateBranch(), self.vc.MasterBranch()]) + releases += self.GetReleasesFromBranch(self._options.branch) + + self["releases"] = sorted(releases, + key=lambda r: SortingKey(r["version"]), + reverse=True) + + +class SwitchChromium(Step): + MESSAGE = "Switch to Chromium checkout." + + def RunStep(self): + cwd = self._options.chromium + # Check for a clean workdir. + if not self.GitIsWorkdirClean(cwd=cwd): # pragma: no cover + self.Die("Workspace is not clean. Please commit or undo your changes.") + # Assert that the DEPS file is there. + if not os.path.exists(os.path.join(cwd, "DEPS")): # pragma: no cover + self.Die("DEPS file not present.") + + +class UpdateChromiumCheckout(Step): + MESSAGE = "Update the checkout and create a new branch." + + def RunStep(self): + cwd = self._options.chromium + self.GitCheckout("master", cwd=cwd) + self.GitPull(cwd=cwd) + self.GitCreateBranch(self.Config("BRANCHNAME"), cwd=cwd) + + +def ConvertToCommitNumber(step, revision): + # Simple check for git hashes. + if revision.isdigit() and len(revision) < 8: + return revision + return step.GetCommitPositionNumber( + revision, cwd=os.path.join(step._options.chromium, "v8")) + + +class RetrieveChromiumV8Releases(Step): + MESSAGE = "Retrieve V8 releases from Chromium DEPS." + + def RunStep(self): + cwd = self._options.chromium + + # Update v8 checkout in chromium. + self.GitFetchOrigin(cwd=os.path.join(cwd, "v8")) + + # All v8 revisions we are interested in. + releases_dict = dict((r["revision_git"], r) for r in self["releases"]) + + cr_releases = [] + try: + for git_hash in self.GitLog( + format="%H", grep="V8", cwd=cwd).splitlines(): + if "DEPS" not in self.GitChangedFiles(git_hash, cwd=cwd): + continue + if not self.GitCheckoutFileSafe("DEPS", git_hash, cwd=cwd): + break # pragma: no cover + deps = FileToText(os.path.join(cwd, "DEPS")) + match = DEPS_RE.search(deps) + if match: + cr_rev = self.GetCommitPositionNumber(git_hash, cwd=cwd) + if cr_rev: + v8_hsh = match.group(1) + cr_releases.append([cr_rev, v8_hsh]) + + # Stop as soon as we find a v8 revision that we didn't fetch in the + # v8-revision-retrieval part above (i.e. a revision that's too old). + if v8_hsh not in releases_dict: + break # pragma: no cover + + # Allow Ctrl-C interrupt. + except (KeyboardInterrupt, SystemExit): # pragma: no cover + pass + + # Clean up. + self.GitCheckoutFileSafe("DEPS", "HEAD", cwd=cwd) + + # Add the chromium ranges to the v8 candidates and master releases. + all_ranges = BuildRevisionRanges(cr_releases) + + for hsh, ranges in all_ranges.iteritems(): + releases_dict.get(hsh, {})["chromium_revision"] = ranges + + +# TODO(machenbach): Unify common code with method above. +class RietrieveChromiumBranches(Step): + MESSAGE = "Retrieve Chromium branch information." + + def RunStep(self): + cwd = self._options.chromium + + # All v8 revisions we are interested in. + releases_dict = dict((r["revision_git"], r) for r in self["releases"]) + + # Filter out irrelevant branches. + branches = filter(lambda r: re.match(r"branch-heads/\d+", r), + self.GitRemotes(cwd=cwd)) + + # Transform into pure branch numbers. + branches = map(lambda r: int(re.match(r"branch-heads/(\d+)", r).group(1)), + branches) + + branches = sorted(branches, reverse=True) + + cr_branches = [] + try: + for branch in branches: + if not self.GitCheckoutFileSafe("DEPS", + "branch-heads/%d" % branch, + cwd=cwd): + break # pragma: no cover + deps = FileToText(os.path.join(cwd, "DEPS")) + match = DEPS_RE.search(deps) + if match: + v8_hsh = match.group(1) + cr_branches.append([str(branch), v8_hsh]) + + # Stop as soon as we find a v8 revision that we didn't fetch in the + # v8-revision-retrieval part above (i.e. a revision that's too old). + if v8_hsh not in releases_dict: + break # pragma: no cover + + # Allow Ctrl-C interrupt. + except (KeyboardInterrupt, SystemExit): # pragma: no cover + pass + + # Clean up. + self.GitCheckoutFileSafe("DEPS", "HEAD", cwd=cwd) + + # Add the chromium branches to the v8 candidate releases. + all_ranges = BuildRevisionRanges(cr_branches) + for revision, ranges in all_ranges.iteritems(): + releases_dict.get(revision, {})["chromium_branch"] = ranges + + +class CleanUp(Step): + MESSAGE = "Clean up." + + def RunStep(self): + self.GitCheckout("master", cwd=self._options.chromium) + self.GitDeleteBranch(self.Config("BRANCHNAME"), cwd=self._options.chromium) + self.CommonCleanup() + + +class WriteOutput(Step): + MESSAGE = "Print output." + + def Run(self): + if self._options.csv: + with open(self._options.csv, "w") as f: + writer = csv.DictWriter(f, + ["version", "branch", "revision", + "chromium_revision", "patches_merged"], + restval="", + extrasaction="ignore") + for release in self["releases"]: + writer.writerow(release) + if self._options.json: + with open(self._options.json, "w") as f: + f.write(json.dumps(self["releases"])) + if not self._options.csv and not self._options.json: + print self["releases"] # pragma: no cover + + +class Releases(ScriptsBase): + def _PrepareOptions(self, parser): + parser.add_argument("-b", "--branch", default="recent", + help=("The branch to analyze. If 'all' is specified, " + "analyze all branches. If 'recent' (default) " + "is specified, track beta, stable and " + "candidates.")) + parser.add_argument("-c", "--chromium", + help=("The path to your Chromium src/ " + "directory to automate the V8 roll.")) + parser.add_argument("--csv", help="Path to a CSV file for export.") + parser.add_argument("-m", "--max-releases", type=int, default=0, + help="The maximum number of releases to track.") + parser.add_argument("--json", help="Path to a JSON file for export.") + + def _ProcessOptions(self, options): # pragma: no cover + return True + + def _Config(self): + return { + "BRANCHNAME": "retrieve-v8-releases", + "PERSISTFILE_BASENAME": "/tmp/v8-releases-tempfile", + } + + def _Steps(self): + return [ + Preparation, + RetrieveV8Releases, + SwitchChromium, + UpdateChromiumCheckout, + RetrieveChromiumV8Releases, + RietrieveChromiumBranches, + CleanUp, + WriteOutput, + ] + + +if __name__ == "__main__": # pragma: no cover + sys.exit(Releases().Run()) diff --git a/deps/v8/tools/release/script_test.py b/deps/v8/tools/release/script_test.py new file mode 100755 index 0000000000..cbb2134f6d --- /dev/null +++ b/deps/v8/tools/release/script_test.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# Copyright 2014 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Wraps test execution with a coverage analysis. To get the best speed, the +# native python coverage version >= 3.7.1 should be installed. + +import coverage +import os +import unittest +import sys + + +def Main(argv): + script_path = os.path.dirname(os.path.abspath(__file__)) + cov = coverage.coverage(include=([os.path.join(script_path, '*.py')])) + cov.start() + import test_scripts + alltests = map(unittest.TestLoader().loadTestsFromTestCase, [ + test_scripts.ToplevelTest, + test_scripts.ScriptTest, + test_scripts.SystemTest, + ]) + unittest.TextTestRunner(verbosity=2).run(unittest.TestSuite(alltests)) + cov.stop() + print cov.report() + + +if __name__ == '__main__': + sys.exit(Main(sys.argv)) diff --git a/deps/v8/tools/release/test_scripts.py b/deps/v8/tools/release/test_scripts.py new file mode 100644 index 0000000000..3beddfd936 --- /dev/null +++ b/deps/v8/tools/release/test_scripts.py @@ -0,0 +1,1558 @@ +#!/usr/bin/env python +# Copyright 2013 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import shutil +import tempfile +import traceback +import unittest + +import auto_push +from auto_push import LastReleaseBailout +import auto_roll +import common_includes +from common_includes import * +import create_release +from create_release import CreateRelease +import merge_to_branch +from merge_to_branch import * +import push_to_candidates +from push_to_candidates import * +import chromium_roll +from chromium_roll import ChromiumRoll +import releases +from releases import Releases +from auto_tag import AutoTag + + +TEST_CONFIG = { + "DEFAULT_CWD": None, + "BRANCHNAME": "test-prepare-push", + "CANDIDATESBRANCH": "test-candidates-push", + "PERSISTFILE_BASENAME": "/tmp/test-v8-push-to-candidates-tempfile", + "CHANGELOG_ENTRY_FILE": + "/tmp/test-v8-push-to-candidates-tempfile-changelog-entry", + "PATCH_FILE": "/tmp/test-v8-push-to-candidates-tempfile-patch", + "COMMITMSG_FILE": "/tmp/test-v8-push-to-candidates-tempfile-commitmsg", + "CHROMIUM": "/tmp/test-v8-push-to-candidates-tempfile-chromium", + "SETTINGS_LOCATION": None, + "ALREADY_MERGING_SENTINEL_FILE": + "/tmp/test-merge-to-branch-tempfile-already-merging", + "TEMPORARY_PATCH_FILE": "/tmp/test-merge-to-branch-tempfile-temporary-patch", + "CLUSTERFUZZ_API_KEY_FILE": "/tmp/test-fake-cf-api-key", +} + + +AUTO_PUSH_ARGS = [ + "-a", "author@chromium.org", + "-r", "reviewer@chromium.org", +] + + +class ToplevelTest(unittest.TestCase): + def testSortBranches(self): + S = releases.SortBranches + self.assertEquals(["3.1", "2.25"], S(["2.25", "3.1"])[0:2]) + self.assertEquals(["3.0", "2.25"], S(["2.25", "3.0", "2.24"])[0:2]) + self.assertEquals(["3.11", "3.2"], S(["3.11", "3.2", "2.24"])[0:2]) + + def testFilterDuplicatesAndReverse(self): + F = releases.FilterDuplicatesAndReverse + self.assertEquals([], F([])) + self.assertEquals([["100", "10"]], F([["100", "10"]])) + self.assertEquals([["99", "9"], ["100", "10"]], + F([["100", "10"], ["99", "9"]])) + self.assertEquals([["98", "9"], ["100", "10"]], + F([["100", "10"], ["99", "9"], ["98", "9"]])) + self.assertEquals([["98", "9"], ["99", "10"]], + F([["100", "10"], ["99", "10"], ["98", "9"]])) + + def testBuildRevisionRanges(self): + B = releases.BuildRevisionRanges + self.assertEquals({}, B([])) + self.assertEquals({"10": "100"}, B([["100", "10"]])) + self.assertEquals({"10": "100", "9": "99:99"}, + B([["100", "10"], ["99", "9"]])) + self.assertEquals({"10": "100", "9": "97:99"}, + B([["100", "10"], ["98", "9"], ["97", "9"]])) + self.assertEquals({"10": "100", "9": "99:99", "3": "91:98"}, + B([["100", "10"], ["99", "9"], ["91", "3"]])) + self.assertEquals({"13": "101", "12": "100:100", "9": "94:97", + "3": "91:93, 98:99"}, + B([["101", "13"], ["100", "12"], ["98", "3"], + ["94", "9"], ["91", "3"]])) + + def testMakeComment(self): + self.assertEquals("# Line 1\n# Line 2\n#", + MakeComment(" Line 1\n Line 2\n")) + self.assertEquals("#Line 1\n#Line 2", + MakeComment("Line 1\n Line 2")) + + def testStripComments(self): + self.assertEquals(" Line 1\n Line 3\n", + StripComments(" Line 1\n# Line 2\n Line 3\n#\n")) + self.assertEquals("\nLine 2 ### Test\n #", + StripComments("###\n# \n\n# Line 1\nLine 2 ### Test\n #")) + + def testMakeChangeLogBodySimple(self): + commits = [ + ["Title text 1", + "Title text 1\n\nBUG=\n", + "author1@chromium.org"], + ["Title text 2.", + "Title text 2\n\nBUG=1234\n", + "author2@chromium.org"], + ] + self.assertEquals(" Title text 1.\n" + " (author1@chromium.org)\n\n" + " Title text 2 (Chromium issue 1234).\n" + " (author2@chromium.org)\n\n", + MakeChangeLogBody(commits)) + + def testMakeChangeLogBodyEmpty(self): + self.assertEquals("", MakeChangeLogBody([])) + + def testMakeChangeLogBodyAutoFormat(self): + commits = [ + ["Title text 1!", + "Title text 1\nLOG=y\nBUG=\n", + "author1@chromium.org"], + ["Title text 2", + "Title text 2\n\nBUG=1234\n", + "author2@chromium.org"], + ["Title text 3", + "Title text 3\n\nBUG=1234\nLOG = Yes\n", + "author3@chromium.org"], + ["Title text 3", + "Title text 4\n\nBUG=1234\nLOG=\n", + "author4@chromium.org"], + ] + self.assertEquals(" Title text 1.\n\n" + " Title text 3 (Chromium issue 1234).\n\n", + MakeChangeLogBody(commits, True)) + + def testRegressWrongLogEntryOnTrue(self): + body = """ +Check elimination: Learn from if(CompareMap(x)) on true branch. + +BUG= +R=verwaest@chromium.org + +Committed: https://code.google.com/p/v8/source/detail?r=18210 +""" + self.assertEquals("", MakeChangeLogBody([["title", body, "author"]], True)) + + def testMakeChangeLogBugReferenceEmpty(self): + self.assertEquals("", MakeChangeLogBugReference("")) + self.assertEquals("", MakeChangeLogBugReference("LOG=")) + self.assertEquals("", MakeChangeLogBugReference(" BUG =")) + self.assertEquals("", MakeChangeLogBugReference("BUG=none\t")) + + def testMakeChangeLogBugReferenceSimple(self): + self.assertEquals("(issue 987654)", + MakeChangeLogBugReference("BUG = v8:987654")) + self.assertEquals("(Chromium issue 987654)", + MakeChangeLogBugReference("BUG=987654 ")) + + def testMakeChangeLogBugReferenceFromBody(self): + self.assertEquals("(Chromium issue 1234567)", + MakeChangeLogBugReference("Title\n\nTBR=\nBUG=\n" + " BUG=\tchromium:1234567\t\n" + "R=somebody\n")) + + def testMakeChangeLogBugReferenceMultiple(self): + # All issues should be sorted and grouped. Multiple references to the same + # issue should be filtered. + self.assertEquals("(issues 123, 234, Chromium issue 345)", + MakeChangeLogBugReference("Title\n\n" + "BUG=v8:234\n" + " BUG\t= 345, \tv8:234,\n" + "BUG=v8:123\n" + "R=somebody\n")) + self.assertEquals("(Chromium issues 123, 234)", + MakeChangeLogBugReference("Title\n\n" + "BUG=234,,chromium:123 \n" + "R=somebody\n")) + self.assertEquals("(Chromium issues 123, 234)", + MakeChangeLogBugReference("Title\n\n" + "BUG=chromium:234, , 123\n" + "R=somebody\n")) + self.assertEquals("(issues 345, 456)", + MakeChangeLogBugReference("Title\n\n" + "\t\tBUG=v8:345,v8:456\n" + "R=somebody\n")) + self.assertEquals("(issue 123, Chromium issues 345, 456)", + MakeChangeLogBugReference("Title\n\n" + "BUG=chromium:456\n" + "BUG = none\n" + "R=somebody\n" + "BUG=456,v8:123, 345")) + + # TODO(machenbach): These test don't make much sense when the formatting is + # done later. + def testMakeChangeLogBugReferenceLong(self): + # -----------------00--------10--------20--------30-------- + self.assertEquals("(issues 234, 1234567890, 1234567" + "8901234567890, Chromium issues 12345678," + " 123456789)", + MakeChangeLogBugReference("BUG=v8:234\n" + "BUG=v8:1234567890\n" + "BUG=v8:12345678901234567890\n" + "BUG=123456789\n" + "BUG=12345678\n")) + # -----------------00--------10--------20--------30-------- + self.assertEquals("(issues 234, 1234567890, 1234567" + "8901234567890, Chromium issues" + " 123456789, 1234567890)", + MakeChangeLogBugReference("BUG=v8:234\n" + "BUG=v8:12345678901234567890\n" + "BUG=v8:1234567890\n" + "BUG=123456789\n" + "BUG=1234567890\n")) + # -----------------00--------10--------20--------30-------- + self.assertEquals("(Chromium issues 234, 1234567890" + ", 12345678901234567, " + "1234567890123456789)", + MakeChangeLogBugReference("BUG=234\n" + "BUG=12345678901234567\n" + "BUG=1234567890123456789\n" + "BUG=1234567890\n")) + + +def Cmd(*args, **kwargs): + """Convenience function returning a shell command test expectation.""" + return { + "name": "command", + "args": args, + "ret": args[-1], + "cb": kwargs.get("cb"), + "cwd": kwargs.get("cwd", TEST_CONFIG["DEFAULT_CWD"]), + } + + +def RL(text, cb=None): + """Convenience function returning a readline test expectation.""" + return { + "name": "readline", + "args": [], + "ret": text, + "cb": cb, + "cwd": None, + } + + +def URL(*args, **kwargs): + """Convenience function returning a readurl test expectation.""" + return { + "name": "readurl", + "args": args[:-1], + "ret": args[-1], + "cb": kwargs.get("cb"), + "cwd": None, + } + + +class SimpleMock(object): + def __init__(self): + self._recipe = [] + self._index = -1 + + def Expect(self, recipe): + self._recipe = recipe + + def Call(self, name, *args, **kwargs): # pragma: no cover + self._index += 1 + try: + expected_call = self._recipe[self._index] + except IndexError: + raise NoRetryException("Calling %s %s" % (name, " ".join(args))) + + if not isinstance(expected_call, dict): + raise NoRetryException("Found wrong expectation type for %s %s" % + (name, " ".join(args))) + + if expected_call["name"] != name: + raise NoRetryException("Expected action: %s %s - Actual: %s" % + (expected_call["name"], expected_call["args"], name)) + + # Check if the given working directory matches the expected one. + if expected_call["cwd"] != kwargs.get("cwd"): + raise NoRetryException("Expected cwd: %s in %s %s - Actual: %s" % + (expected_call["cwd"], + expected_call["name"], + expected_call["args"], + kwargs.get("cwd"))) + + # The number of arguments in the expectation must match the actual + # arguments. + if len(args) > len(expected_call['args']): + raise NoRetryException("When calling %s with arguments, the " + "expectations must consist of at least as many arguments." % + name) + + # Compare expected and actual arguments. + for (expected_arg, actual_arg) in zip(expected_call['args'], args): + if expected_arg != actual_arg: + raise NoRetryException("Expected: %s - Actual: %s" % + (expected_arg, actual_arg)) + + # The expected call contains an optional callback for checking the context + # at the time of the call. + if expected_call['cb']: + try: + expected_call['cb']() + except: + tb = traceback.format_exc() + raise NoRetryException("Caught exception from callback: %s" % tb) + + # If the return value is an exception, raise it instead of returning. + if isinstance(expected_call['ret'], Exception): + raise expected_call['ret'] + return expected_call['ret'] + + def AssertFinished(self): # pragma: no cover + if self._index < len(self._recipe) -1: + raise NoRetryException("Called mock too seldom: %d vs. %d" % + (self._index, len(self._recipe))) + + +class ScriptTest(unittest.TestCase): + def MakeEmptyTempFile(self): + handle, name = tempfile.mkstemp() + os.close(handle) + self._tmp_files.append(name) + return name + + def MakeEmptyTempDirectory(self): + name = tempfile.mkdtemp() + self._tmp_files.append(name) + return name + + + def WriteFakeVersionFile(self, major=3, minor=22, build=4, patch=0): + version_file = os.path.join(TEST_CONFIG["DEFAULT_CWD"], VERSION_FILE) + if not os.path.exists(os.path.dirname(version_file)): + os.makedirs(os.path.dirname(version_file)) + with open(version_file, "w") as f: + f.write(" // Some line...\n") + f.write("\n") + f.write("#define MAJOR_VERSION %s\n" % major) + f.write("#define MINOR_VERSION %s\n" % minor) + f.write("#define BUILD_NUMBER %s\n" % build) + f.write("#define PATCH_LEVEL %s\n" % patch) + f.write(" // Some line...\n") + f.write("#define IS_CANDIDATE_VERSION 0\n") + + def MakeStep(self): + """Convenience wrapper.""" + options = ScriptsBase(TEST_CONFIG, self, self._state).MakeOptions([]) + return MakeStep(step_class=Step, state=self._state, + config=TEST_CONFIG, side_effect_handler=self, + options=options) + + def RunStep(self, script=PushToCandidates, step_class=Step, args=None): + """Convenience wrapper.""" + args = args if args is not None else ["-m"] + return script(TEST_CONFIG, self, self._state).RunSteps([step_class], args) + + def Call(self, fun, *args, **kwargs): + print "Calling %s with %s and %s" % (str(fun), str(args), str(kwargs)) + + def Command(self, cmd, args="", prefix="", pipe=True, cwd=None): + print "%s %s" % (cmd, args) + print "in %s" % cwd + return self._mock.Call("command", cmd + " " + args, cwd=cwd) + + def ReadLine(self): + return self._mock.Call("readline") + + def ReadURL(self, url, params): + if params is not None: + return self._mock.Call("readurl", url, params) + else: + return self._mock.Call("readurl", url) + + def ReadClusterFuzzAPI(self, api_key, **params): + # TODO(machenbach): Use a mock for this and add a test that stops rolling + # due to clustefuzz results. + return [] + + def Sleep(self, seconds): + pass + + def GetDate(self): + return "1999-07-31" + + def GetUTCStamp(self): + return "1000000" + + def Expect(self, *args): + """Convenience wrapper.""" + self._mock.Expect(*args) + + def setUp(self): + self._mock = SimpleMock() + self._tmp_files = [] + self._state = {} + TEST_CONFIG["DEFAULT_CWD"] = self.MakeEmptyTempDirectory() + + def tearDown(self): + if os.path.exists(TEST_CONFIG["PERSISTFILE_BASENAME"]): + shutil.rmtree(TEST_CONFIG["PERSISTFILE_BASENAME"]) + + # Clean up temps. Doesn't work automatically. + for name in self._tmp_files: + if os.path.isfile(name): + os.remove(name) + if os.path.isdir(name): + shutil.rmtree(name) + + self._mock.AssertFinished() + + def testGitMock(self): + self.Expect([Cmd("git --version", "git version 1.2.3"), + Cmd("git dummy", "")]) + self.assertEquals("git version 1.2.3", self.MakeStep().Git("--version")) + self.assertEquals("", self.MakeStep().Git("dummy")) + + def testCommonPrepareDefault(self): + self.Expect([ + Cmd("git status -s -uno", ""), + Cmd("git status -s -b -uno", "## some_branch"), + Cmd("git fetch", ""), + Cmd("git branch", " branch1\n* %s" % TEST_CONFIG["BRANCHNAME"]), + RL("Y"), + Cmd("git branch -D %s" % TEST_CONFIG["BRANCHNAME"], ""), + ]) + self.MakeStep().CommonPrepare() + self.MakeStep().PrepareBranch() + self.assertEquals("some_branch", self._state["current_branch"]) + + def testCommonPrepareNoConfirm(self): + self.Expect([ + Cmd("git status -s -uno", ""), + Cmd("git status -s -b -uno", "## some_branch"), + Cmd("git fetch", ""), + Cmd("git branch", " branch1\n* %s" % TEST_CONFIG["BRANCHNAME"]), + RL("n"), + ]) + self.MakeStep().CommonPrepare() + self.assertRaises(Exception, self.MakeStep().PrepareBranch) + self.assertEquals("some_branch", self._state["current_branch"]) + + def testCommonPrepareDeleteBranchFailure(self): + self.Expect([ + Cmd("git status -s -uno", ""), + Cmd("git status -s -b -uno", "## some_branch"), + Cmd("git fetch", ""), + Cmd("git branch", " branch1\n* %s" % TEST_CONFIG["BRANCHNAME"]), + RL("Y"), + Cmd("git branch -D %s" % TEST_CONFIG["BRANCHNAME"], None), + ]) + self.MakeStep().CommonPrepare() + self.assertRaises(Exception, self.MakeStep().PrepareBranch) + self.assertEquals("some_branch", self._state["current_branch"]) + + def testInitialEnvironmentChecks(self): + TextToFile("", os.path.join(TEST_CONFIG["DEFAULT_CWD"], ".git")) + os.environ["EDITOR"] = "vi" + self.Expect([ + Cmd("which vi", "/usr/bin/vi"), + ]) + self.MakeStep().InitialEnvironmentChecks(TEST_CONFIG["DEFAULT_CWD"]) + + def testTagTimeout(self): + self.Expect([ + Cmd("git fetch", ""), + Cmd("git log -1 --format=%H --grep=\"Title\" origin/candidates", ""), + Cmd("git fetch", ""), + Cmd("git log -1 --format=%H --grep=\"Title\" origin/candidates", ""), + Cmd("git fetch", ""), + Cmd("git log -1 --format=%H --grep=\"Title\" origin/candidates", ""), + Cmd("git fetch", ""), + Cmd("git log -1 --format=%H --grep=\"Title\" origin/candidates", ""), + ]) + args = ["--branch", "candidates", "ab12345"] + self._state["version"] = "tag_name" + self._state["commit_title"] = "Title" + self.assertRaises(Exception, + lambda: self.RunStep(MergeToBranch, TagRevision, args)) + + def testReadAndPersistVersion(self): + self.WriteFakeVersionFile(build=5) + step = self.MakeStep() + step.ReadAndPersistVersion() + self.assertEquals("3", step["major"]) + self.assertEquals("22", step["minor"]) + self.assertEquals("5", step["build"]) + self.assertEquals("0", step["patch"]) + + def testRegex(self): + self.assertEqual("(issue 321)", + re.sub(r"BUG=v8:(.*)$", r"(issue \1)", "BUG=v8:321")) + self.assertEqual("(Chromium issue 321)", + re.sub(r"BUG=(.*)$", r"(Chromium issue \1)", "BUG=321")) + + cl = " too little\n\ttab\ttab\n too much\n trailing " + cl = MSub(r"\t", r" ", cl) + cl = MSub(r"^ {1,7}([^ ])", r" \1", cl) + cl = MSub(r"^ {9,80}([^ ])", r" \1", cl) + cl = MSub(r" +$", r"", cl) + self.assertEqual(" too little\n" + " tab tab\n" + " too much\n" + " trailing", cl) + + self.assertEqual("//\n#define BUILD_NUMBER 3\n", + MSub(r"(?<=#define BUILD_NUMBER)(?P<space>\s+)\d*$", + r"\g<space>3", + "//\n#define BUILD_NUMBER 321\n")) + + def testPreparePushRevision(self): + # Tests the default push hash used when the --revision option is not set. + self.Expect([ + Cmd("git log -1 --format=%H HEAD", "push_hash") + ]) + + self.RunStep(PushToCandidates, PreparePushRevision) + self.assertEquals("push_hash", self._state["push_hash"]) + + def testPrepareChangeLog(self): + self.WriteFakeVersionFile() + TEST_CONFIG["CHANGELOG_ENTRY_FILE"] = self.MakeEmptyTempFile() + + self.Expect([ + Cmd("git log --format=%H 1234..push_hash", "rev1\nrev2\nrev3\nrev4"), + Cmd("git log -1 --format=%s rev1", "Title text 1"), + Cmd("git log -1 --format=%B rev1", "Title\n\nBUG=\nLOG=y\n"), + Cmd("git log -1 --format=%an rev1", "author1@chromium.org"), + Cmd("git log -1 --format=%s rev2", "Title text 2."), + Cmd("git log -1 --format=%B rev2", "Title\n\nBUG=123\nLOG= \n"), + Cmd("git log -1 --format=%an rev2", "author2@chromium.org"), + Cmd("git log -1 --format=%s rev3", "Title text 3"), + Cmd("git log -1 --format=%B rev3", "Title\n\nBUG=321\nLOG=true\n"), + Cmd("git log -1 --format=%an rev3", "author3@chromium.org"), + Cmd("git log -1 --format=%s rev4", "Title text 4"), + Cmd("git log -1 --format=%B rev4", + ("Title\n\nBUG=456\nLOG=Y\n\n" + "Review URL: https://codereview.chromium.org/9876543210\n")), + URL("https://codereview.chromium.org/9876543210/description", + "Title\n\nBUG=456\nLOG=N\n\n"), + Cmd("git log -1 --format=%an rev4", "author4@chromium.org"), + ]) + + self._state["last_push_master"] = "1234" + self._state["push_hash"] = "push_hash" + self._state["version"] = "3.22.5" + self.RunStep(PushToCandidates, PrepareChangeLog) + + actual_cl = FileToText(TEST_CONFIG["CHANGELOG_ENTRY_FILE"]) + + expected_cl = """1999-07-31: Version 3.22.5 + + Title text 1. + + Title text 3 (Chromium issue 321). + + Performance and stability improvements on all platforms. +# +# The change log above is auto-generated. Please review if all relevant +# commit messages from the list below are included. +# All lines starting with # will be stripped. +# +# Title text 1. +# (author1@chromium.org) +# +# Title text 2 (Chromium issue 123). +# (author2@chromium.org) +# +# Title text 3 (Chromium issue 321). +# (author3@chromium.org) +# +# Title text 4 (Chromium issue 456). +# (author4@chromium.org) +# +#""" + + self.assertEquals(expected_cl, actual_cl) + + def testEditChangeLog(self): + TEST_CONFIG["CHANGELOG_ENTRY_FILE"] = self.MakeEmptyTempFile() + TextToFile(" New \n\tLines \n", TEST_CONFIG["CHANGELOG_ENTRY_FILE"]) + os.environ["EDITOR"] = "vi" + self.Expect([ + RL(""), # Open editor. + Cmd("vi %s" % TEST_CONFIG["CHANGELOG_ENTRY_FILE"], ""), + ]) + + self.RunStep(PushToCandidates, EditChangeLog) + + self.assertEquals("New\n Lines", + FileToText(TEST_CONFIG["CHANGELOG_ENTRY_FILE"])) + + TAGS = """ +4425.0 +0.0.0.0 +3.9.6 +3.22.4 +test_tag +""" + + # Version as tag: 3.22.4.0. Version on master: 3.22.6. + # Make sure that the latest version is 3.22.6.0. + def testIncrementVersion(self): + self.Expect([ + Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""), + Cmd("git tag", self.TAGS), + Cmd("git checkout -f origin/master -- src/version.cc", + "", cb=lambda: self.WriteFakeVersionFile(3, 22, 6)), + ]) + + self.RunStep(PushToCandidates, IncrementVersion) + + self.assertEquals("3", self._state["new_major"]) + self.assertEquals("22", self._state["new_minor"]) + self.assertEquals("7", self._state["new_build"]) + self.assertEquals("0", self._state["new_patch"]) + + def _TestSquashCommits(self, change_log, expected_msg): + TEST_CONFIG["CHANGELOG_ENTRY_FILE"] = self.MakeEmptyTempFile() + with open(TEST_CONFIG["CHANGELOG_ENTRY_FILE"], "w") as f: + f.write(change_log) + + self.Expect([ + Cmd("git diff origin/candidates hash1", "patch content"), + ]) + + self._state["push_hash"] = "hash1" + self._state["date"] = "1999-11-11" + + self.RunStep(PushToCandidates, SquashCommits) + self.assertEquals(FileToText(TEST_CONFIG["COMMITMSG_FILE"]), expected_msg) + + patch = FileToText(TEST_CONFIG["PATCH_FILE"]) + self.assertTrue(re.search(r"patch content", patch)) + + def testSquashCommitsUnformatted(self): + change_log = """1999-11-11: Version 3.22.5 + + Log text 1. + Chromium issue 12345 + + Performance and stability improvements on all platforms.\n""" + commit_msg = """Version 3.22.5 (based on hash1) + +Log text 1. Chromium issue 12345 + +Performance and stability improvements on all platforms.""" + self._TestSquashCommits(change_log, commit_msg) + + def testSquashCommitsFormatted(self): + change_log = """1999-11-11: Version 3.22.5 + + Long commit message that fills more than 80 characters (Chromium issue + 12345). + + Performance and stability improvements on all platforms.\n""" + commit_msg = """Version 3.22.5 (based on hash1) + +Long commit message that fills more than 80 characters (Chromium issue 12345). + +Performance and stability improvements on all platforms.""" + self._TestSquashCommits(change_log, commit_msg) + + def testSquashCommitsQuotationMarks(self): + change_log = """Line with "quotation marks".\n""" + commit_msg = """Line with "quotation marks".""" + self._TestSquashCommits(change_log, commit_msg) + + def testBootstrapper(self): + work_dir = self.MakeEmptyTempDirectory() + class FakeScript(ScriptsBase): + def _Steps(self): + return [] + + # Use the test configuration without the fake testing default work dir. + fake_config = dict(TEST_CONFIG) + del(fake_config["DEFAULT_CWD"]) + + self.Expect([ + Cmd("fetch v8", "", cwd=work_dir), + ]) + FakeScript(fake_config, self).Run(["--work-dir", work_dir]) + + def _PushToCandidates(self, force=False, manual=False): + TextToFile("", os.path.join(TEST_CONFIG["DEFAULT_CWD"], ".git")) + + # The version file on master has build level 5, while the version + # file from candidates has build level 4. + self.WriteFakeVersionFile(build=5) + + TEST_CONFIG["CHANGELOG_ENTRY_FILE"] = self.MakeEmptyTempFile() + master_change_log = "2014-03-17: Sentinel\n" + TextToFile(master_change_log, + os.path.join(TEST_CONFIG["DEFAULT_CWD"], CHANGELOG_FILE)) + os.environ["EDITOR"] = "vi" + + commit_msg_squashed = """Version 3.22.5 (squashed - based on push_hash) + +Log text 1 (issue 321). + +Performance and stability improvements on all platforms.""" + + commit_msg = """Version 3.22.5 (based on push_hash) + +Log text 1 (issue 321). + +Performance and stability improvements on all platforms.""" + + def ResetChangeLog(): + """On 'git co -b new_branch origin/candidates', + and 'git checkout -- ChangeLog', + the ChangLog will be reset to its content on candidates.""" + candidates_change_log = """1999-04-05: Version 3.22.4 + + Performance and stability improvements on all platforms.\n""" + TextToFile(candidates_change_log, + os.path.join(TEST_CONFIG["DEFAULT_CWD"], CHANGELOG_FILE)) + + def ResetToCandidates(): + ResetChangeLog() + self.WriteFakeVersionFile() + + def CheckVersionCommit(): + commit = FileToText(TEST_CONFIG["COMMITMSG_FILE"]) + self.assertEquals(commit_msg, commit) + version = FileToText( + os.path.join(TEST_CONFIG["DEFAULT_CWD"], VERSION_FILE)) + self.assertTrue(re.search(r"#define MINOR_VERSION\s+22", version)) + self.assertTrue(re.search(r"#define BUILD_NUMBER\s+5", version)) + self.assertFalse(re.search(r"#define BUILD_NUMBER\s+6", version)) + self.assertTrue(re.search(r"#define PATCH_LEVEL\s+0", version)) + self.assertTrue(re.search(r"#define IS_CANDIDATE_VERSION\s+0", version)) + + # Check that the change log on the candidates branch got correctly + # modified. + change_log = FileToText( + os.path.join(TEST_CONFIG["DEFAULT_CWD"], CHANGELOG_FILE)) + self.assertEquals( +"""1999-07-31: Version 3.22.5 + + Log text 1 (issue 321). + + Performance and stability improvements on all platforms. + + +1999-04-05: Version 3.22.4 + + Performance and stability improvements on all platforms.\n""", + change_log) + + force_flag = " -f" if not manual else "" + expectations = [] + if not force: + expectations.append(Cmd("which vi", "/usr/bin/vi")) + expectations += [ + Cmd("git status -s -uno", ""), + Cmd("git status -s -b -uno", "## some_branch\n"), + Cmd("git fetch", ""), + Cmd("git branch", " branch1\n* branch2\n"), + Cmd("git branch", " branch1\n* branch2\n"), + Cmd(("git new-branch %s --upstream origin/master" % + TEST_CONFIG["BRANCHNAME"]), ""), + Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""), + Cmd("git tag", self.TAGS), + Cmd("git checkout -f origin/master -- src/version.cc", + "", cb=self.WriteFakeVersionFile), + Cmd("git log -1 --format=%H 3.22.4", "release_hash\n"), + Cmd("git log -1 --format=%s release_hash", + "Version 3.22.4 (based on abc3)\n"), + Cmd("git log --format=%H abc3..push_hash", "rev1\n"), + Cmd("git log -1 --format=%s rev1", "Log text 1.\n"), + Cmd("git log -1 --format=%B rev1", "Text\nLOG=YES\nBUG=v8:321\nText\n"), + Cmd("git log -1 --format=%an rev1", "author1@chromium.org\n"), + ] + if manual: + expectations.append(RL("")) # Open editor. + if not force: + expectations.append( + Cmd("vi %s" % TEST_CONFIG["CHANGELOG_ENTRY_FILE"], "")) + expectations += [ + Cmd("git fetch", ""), + Cmd("git checkout -f origin/master", ""), + Cmd("git diff origin/candidates push_hash", "patch content\n"), + Cmd(("git new-branch %s --upstream origin/candidates" % + TEST_CONFIG["CANDIDATESBRANCH"]), "", cb=ResetToCandidates), + Cmd("git apply --index --reject \"%s\"" % TEST_CONFIG["PATCH_FILE"], ""), + Cmd("git checkout -f origin/candidates -- ChangeLog", "", + cb=ResetChangeLog), + Cmd("git checkout -f origin/candidates -- src/version.cc", "", + cb=self.WriteFakeVersionFile), + Cmd("git commit -am \"%s\"" % commit_msg_squashed, ""), + ] + if manual: + expectations.append(RL("Y")) # Sanity check. + expectations += [ + Cmd("git cl land -f --bypass-hooks", ""), + Cmd("git checkout -f master", ""), + Cmd("git fetch", ""), + Cmd("git branch -D %s" % TEST_CONFIG["CANDIDATESBRANCH"], ""), + Cmd(("git new-branch %s --upstream origin/candidates" % + TEST_CONFIG["CANDIDATESBRANCH"]), "", cb=ResetToCandidates), + Cmd("git commit -aF \"%s\"" % TEST_CONFIG["COMMITMSG_FILE"], "", + cb=CheckVersionCommit), + Cmd("git cl land -f --bypass-hooks", ""), + Cmd("git fetch", ""), + Cmd("git log -1 --format=%H --grep=" + "\"Version 3.22.5 (based on push_hash)\"" + " origin/candidates", "hsh_to_tag"), + Cmd("git tag 3.22.5 hsh_to_tag", ""), + Cmd("git push origin 3.22.5", ""), + Cmd("git checkout -f some_branch", ""), + Cmd("git branch -D %s" % TEST_CONFIG["BRANCHNAME"], ""), + Cmd("git branch -D %s" % TEST_CONFIG["CANDIDATESBRANCH"], ""), + ] + self.Expect(expectations) + + args = ["-a", "author@chromium.org", "--revision", "push_hash"] + if force: args.append("-f") + if manual: args.append("-m") + else: args += ["-r", "reviewer@chromium.org"] + PushToCandidates(TEST_CONFIG, self).Run(args) + + cl = FileToText(os.path.join(TEST_CONFIG["DEFAULT_CWD"], CHANGELOG_FILE)) + self.assertTrue(re.search(r"^\d\d\d\d\-\d+\-\d+: Version 3\.22\.5", cl)) + self.assertTrue(re.search(r" Log text 1 \(issue 321\).", cl)) + self.assertTrue(re.search(r"1999\-04\-05: Version 3\.22\.4", cl)) + + # Note: The version file is on build number 5 again in the end of this test + # since the git command that merges to master is mocked out. + + def testPushToCandidatesManual(self): + self._PushToCandidates(manual=True) + + def testPushToCandidatesSemiAutomatic(self): + self._PushToCandidates() + + def testPushToCandidatesForced(self): + self._PushToCandidates(force=True) + + def testCreateRelease(self): + TextToFile("", os.path.join(TEST_CONFIG["DEFAULT_CWD"], ".git")) + + # The version file on master has build level 5. + self.WriteFakeVersionFile(build=5) + + master_change_log = "2014-03-17: Sentinel\n" + TextToFile(master_change_log, + os.path.join(TEST_CONFIG["DEFAULT_CWD"], CHANGELOG_FILE)) + + commit_msg = """Version 3.22.5 + +Log text 1 (issue 321). + +Performance and stability improvements on all platforms.""" + + def ResetChangeLog(): + last_change_log = """1999-04-05: Version 3.22.4 + + Performance and stability improvements on all platforms.\n""" + TextToFile(last_change_log, + os.path.join(TEST_CONFIG["DEFAULT_CWD"], CHANGELOG_FILE)) + + + def CheckVersionCommit(): + commit = FileToText(TEST_CONFIG["COMMITMSG_FILE"]) + self.assertEquals(commit_msg, commit) + version = FileToText( + os.path.join(TEST_CONFIG["DEFAULT_CWD"], VERSION_FILE)) + self.assertTrue(re.search(r"#define MINOR_VERSION\s+22", version)) + self.assertTrue(re.search(r"#define BUILD_NUMBER\s+5", version)) + self.assertFalse(re.search(r"#define BUILD_NUMBER\s+6", version)) + self.assertTrue(re.search(r"#define PATCH_LEVEL\s+0", version)) + self.assertTrue(re.search(r"#define IS_CANDIDATE_VERSION\s+0", version)) + + # Check that the change log on the candidates branch got correctly + # modified. + change_log = FileToText( + os.path.join(TEST_CONFIG["DEFAULT_CWD"], CHANGELOG_FILE)) + self.assertEquals( +"""1999-07-31: Version 3.22.5 + + Log text 1 (issue 321). + + Performance and stability improvements on all platforms. + + +1999-04-05: Version 3.22.4 + + Performance and stability improvements on all platforms.\n""", + change_log) + + expectations = [ + Cmd("git fetch origin " + "+refs/heads/*:refs/heads/* " + "+refs/pending/*:refs/pending/* " + "+refs/pending-tags/*:refs/pending-tags/*", ""), + Cmd("git checkout -f origin/master", ""), + Cmd("git branch", ""), + Cmd("git log -1 --format=\"%H %T\" push_hash", "push_hash tree_hash"), + Cmd("git log -200 --format=\"%H %T\" refs/pending/heads/master", + "not_right wrong\npending_hash tree_hash\nsome other\n"), + Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""), + Cmd("git tag", self.TAGS), + Cmd("git checkout -f origin/master -- src/version.cc", + "", cb=self.WriteFakeVersionFile), + Cmd("git log -1 --format=%H 3.22.4", "release_hash\n"), + Cmd("git log -1 --format=%s release_hash", "Version 3.22.4\n"), + Cmd("git log -1 --format=%H release_hash^", "abc3\n"), + Cmd("git log --format=%H abc3..push_hash", "rev1\n"), + Cmd("git log -1 --format=%s rev1", "Log text 1.\n"), + Cmd("git log -1 --format=%B rev1", "Text\nLOG=YES\nBUG=v8:321\nText\n"), + Cmd("git log -1 --format=%an rev1", "author1@chromium.org\n"), + Cmd("git reset --hard origin/master", ""), + Cmd("git checkout -b work-branch pending_hash", ""), + Cmd("git checkout -f 3.22.4 -- ChangeLog", "", cb=ResetChangeLog), + Cmd("git checkout -f 3.22.4 -- src/version.cc", "", + cb=self.WriteFakeVersionFile), + Cmd("git commit -aF \"%s\"" % TEST_CONFIG["COMMITMSG_FILE"], "", + cb=CheckVersionCommit), + Cmd("git push origin " + "refs/heads/work-branch:refs/pending/heads/3.22.5 " + "pending_hash:refs/pending-tags/heads/3.22.5 " + "push_hash:refs/heads/3.22.5", ""), + Cmd("git fetch", ""), + Cmd("git log -1 --format=%H --grep=" + "\"Version 3.22.5\" origin/3.22.5", "hsh_to_tag"), + Cmd("git tag 3.22.5 hsh_to_tag", ""), + Cmd("git push origin 3.22.5", ""), + Cmd("git checkout -f origin/master", ""), + Cmd("git branch", "* master\n work-branch\n"), + Cmd("git branch -D work-branch", ""), + Cmd("git gc", ""), + ] + self.Expect(expectations) + + args = ["-a", "author@chromium.org", + "-r", "reviewer@chromium.org", + "--revision", "push_hash"] + CreateRelease(TEST_CONFIG, self).Run(args) + + cl = FileToText(os.path.join(TEST_CONFIG["DEFAULT_CWD"], CHANGELOG_FILE)) + self.assertTrue(re.search(r"^\d\d\d\d\-\d+\-\d+: Version 3\.22\.5", cl)) + self.assertTrue(re.search(r" Log text 1 \(issue 321\).", cl)) + self.assertTrue(re.search(r"1999\-04\-05: Version 3\.22\.4", cl)) + + # Note: The version file is on build number 5 again in the end of this test + # since the git command that merges to master is mocked out. + + C_V8_22624_LOG = """V8 CL. + +git-svn-id: https://v8.googlecode.com/svn/branches/bleeding_edge@22624 123 + +""" + + C_V8_123455_LOG = """V8 CL. + +git-svn-id: https://v8.googlecode.com/svn/branches/bleeding_edge@123455 123 + +""" + + C_V8_123456_LOG = """V8 CL. + +git-svn-id: https://v8.googlecode.com/svn/branches/bleeding_edge@123456 123 + +""" + + def testChromiumRoll(self): + googlers_mapping_py = "%s-mapping.py" % TEST_CONFIG["PERSISTFILE_BASENAME"] + with open(googlers_mapping_py, "w") as f: + f.write(""" +def list_to_dict(entries): + return {"g_name@google.com": "c_name@chromium.org"} +def get_list(): + pass""") + + # Setup fake directory structures. + TEST_CONFIG["CHROMIUM"] = self.MakeEmptyTempDirectory() + TextToFile("", os.path.join(TEST_CONFIG["CHROMIUM"], ".git")) + chrome_dir = TEST_CONFIG["CHROMIUM"] + os.makedirs(os.path.join(chrome_dir, "v8")) + + # Write fake deps file. + TextToFile("Some line\n \"v8_revision\": \"123444\",\n some line", + os.path.join(chrome_dir, "DEPS")) + def WriteDeps(): + TextToFile("Some line\n \"v8_revision\": \"22624\",\n some line", + os.path.join(chrome_dir, "DEPS")) + + expectations = [ + Cmd("git fetch origin", ""), + Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""), + Cmd("git tag", self.TAGS), + Cmd("git log -1 --format=%H 3.22.4", "push_hash\n"), + Cmd("git log -1 --format=%s push_hash", + "Version 3.22.4 (based on abc)\n"), + Cmd("git log -1 --format=%H 3.22.4", "push_hash\n"), + Cmd("git log -1 --format=%s push_hash", + "Version 3.22.4 (based on abc)"), + Cmd("git describe --tags last_roll_hsh", "3.22.2.1"), + Cmd("git log -1 --format=%H 3.22.2", "last_roll_base_hash"), + Cmd("git log -1 --format=%s last_roll_base_hash", "Version 3.22.2"), + Cmd("git log -1 --format=%H last_roll_base_hash^", + "last_roll_master_hash"), + URL("https://chromium-build.appspot.com/p/chromium/sheriff_v8.js", + "document.write('g_name')"), + Cmd("git status -s -uno", "", cwd=chrome_dir), + Cmd("git checkout -f master", "", cwd=chrome_dir), + Cmd("gclient sync --nohooks", "syncing...", cwd=chrome_dir), + Cmd("git pull", "", cwd=chrome_dir), + Cmd("git fetch origin", ""), + Cmd("git new-branch v8-roll-push_hash", "", cwd=chrome_dir), + Cmd("roll-dep v8 push_hash", "rolled", cb=WriteDeps, cwd=chrome_dir), + Cmd(("git commit -am \"Update V8 to version 3.22.4 " + "(based on abc).\n\n" + "Summary of changes available at:\n" + "https://chromium.googlesource.com/v8/v8/+log/last_rol..abc\n\n" + "Please reply to the V8 sheriff c_name@chromium.org in " + "case of problems.\n\nTBR=c_name@chromium.org\" " + "--author \"author@chromium.org <author@chromium.org>\""), + "", cwd=chrome_dir), + Cmd("git cl upload --send-mail --email \"author@chromium.org\" -f", "", + cwd=chrome_dir), + ] + self.Expect(expectations) + + args = ["-a", "author@chromium.org", "-c", chrome_dir, + "--sheriff", "--googlers-mapping", googlers_mapping_py, + "-r", "reviewer@chromium.org", + "--last-roll", "last_roll_hsh"] + ChromiumRoll(TEST_CONFIG, self).Run(args) + + deps = FileToText(os.path.join(chrome_dir, "DEPS")) + self.assertTrue(re.search("\"v8_revision\": \"22624\"", deps)) + + def testCheckLastPushRecently(self): + self.Expect([ + Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""), + Cmd("git tag", self.TAGS), + Cmd("git log -1 --format=%H 3.22.4", "release_hash\n"), + Cmd("git log -1 --format=%s release_hash", + "Version 3.22.4 (based on abc3)\n"), + Cmd("git log --format=%H abc3..abc123", "\n"), + ]) + + self._state["candidate"] = "abc123" + self.assertEquals(0, self.RunStep( + auto_push.AutoPush, LastReleaseBailout, AUTO_PUSH_ARGS)) + + def testAutoPush(self): + TextToFile("", os.path.join(TEST_CONFIG["DEFAULT_CWD"], ".git")) + + self.Expect([ + Cmd("git status -s -uno", ""), + Cmd("git status -s -b -uno", "## some_branch\n"), + Cmd("git fetch", ""), + Cmd("git fetch origin +refs/heads/roll:refs/heads/roll", ""), + Cmd("git show-ref -s refs/heads/roll", "abc123\n"), + Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""), + Cmd("git tag", self.TAGS), + Cmd("git log -1 --format=%H 3.22.4", "release_hash\n"), + Cmd("git log -1 --format=%s release_hash", + "Version 3.22.4 (based on abc3)\n"), + Cmd("git log --format=%H abc3..abc123", "some_stuff\n"), + ]) + + auto_push.AutoPush(TEST_CONFIG, self).Run(AUTO_PUSH_ARGS + ["--push"]) + + state = json.loads(FileToText("%s-state.json" + % TEST_CONFIG["PERSISTFILE_BASENAME"])) + + self.assertEquals("abc123", state["candidate"]) + + def testAutoRollExistingRoll(self): + self.Expect([ + URL("https://codereview.chromium.org/search", + "owner=author%40chromium.org&limit=30&closed=3&format=json", + ("{\"results\": [{\"subject\": \"different\"}," + "{\"subject\": \"Update V8 to Version...\"}]}")), + ]) + + result = auto_roll.AutoRoll(TEST_CONFIG, self).Run( + AUTO_PUSH_ARGS + ["-c", TEST_CONFIG["CHROMIUM"]]) + self.assertEquals(0, result) + + # Snippet from the original DEPS file. + FAKE_DEPS = """ +vars = { + "v8_revision": "abcd123455", +} +deps = { + "src/v8": + (Var("googlecode_url") % "v8") + "/" + Var("v8_branch") + "@" + + Var("v8_revision"), +} +""" + + def testAutoRollUpToDate(self): + TEST_CONFIG["CHROMIUM"] = self.MakeEmptyTempDirectory() + TextToFile(self.FAKE_DEPS, os.path.join(TEST_CONFIG["CHROMIUM"], "DEPS")) + self.Expect([ + URL("https://codereview.chromium.org/search", + "owner=author%40chromium.org&limit=30&closed=3&format=json", + ("{\"results\": [{\"subject\": \"different\"}]}")), + Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""), + Cmd("git tag", self.TAGS), + Cmd("git log -1 --format=%H 3.22.4", "push_hash\n"), + ]) + + result = auto_roll.AutoRoll(TEST_CONFIG, self).Run( + AUTO_PUSH_ARGS + ["-c", TEST_CONFIG["CHROMIUM"]]) + self.assertEquals(0, result) + + def testAutoRoll(self): + TEST_CONFIG["CHROMIUM"] = self.MakeEmptyTempDirectory() + TextToFile(self.FAKE_DEPS, os.path.join(TEST_CONFIG["CHROMIUM"], "DEPS")) + TEST_CONFIG["CLUSTERFUZZ_API_KEY_FILE"] = self.MakeEmptyTempFile() + TextToFile("fake key", TEST_CONFIG["CLUSTERFUZZ_API_KEY_FILE"]) + + self.Expect([ + URL("https://codereview.chromium.org/search", + "owner=author%40chromium.org&limit=30&closed=3&format=json", + ("{\"results\": [{\"subject\": \"different\"}]}")), + Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""), + Cmd("git tag", self.TAGS), + Cmd("git log -1 --format=%H 3.22.4", "push_hash\n"), + ]) + + result = auto_roll.AutoRoll(TEST_CONFIG, self).Run( + AUTO_PUSH_ARGS + ["-c", TEST_CONFIG["CHROMIUM"], "--roll"]) + self.assertEquals(0, result) + + def testMergeToBranch(self): + TEST_CONFIG["ALREADY_MERGING_SENTINEL_FILE"] = self.MakeEmptyTempFile() + TextToFile("", os.path.join(TEST_CONFIG["DEFAULT_CWD"], ".git")) + self.WriteFakeVersionFile(build=5) + os.environ["EDITOR"] = "vi" + extra_patch = self.MakeEmptyTempFile() + + def VerifyPatch(patch): + return lambda: self.assertEquals(patch, + FileToText(TEST_CONFIG["TEMPORARY_PATCH_FILE"])) + + msg = """Version 3.22.5.1 (cherry-pick) + +Merged ab12345 +Merged ab23456 +Merged ab34567 +Merged ab45678 +Merged ab56789 + +Title4 + +Title2 + +Title3 + +Title1 + +Revert "Something" + +BUG=123,234,345,456,567,v8:123 +LOG=N +""" + + def VerifyLand(): + commit = FileToText(TEST_CONFIG["COMMITMSG_FILE"]) + self.assertEquals(msg, commit) + version = FileToText( + os.path.join(TEST_CONFIG["DEFAULT_CWD"], VERSION_FILE)) + self.assertTrue(re.search(r"#define MINOR_VERSION\s+22", version)) + self.assertTrue(re.search(r"#define BUILD_NUMBER\s+5", version)) + self.assertTrue(re.search(r"#define PATCH_LEVEL\s+1", version)) + self.assertTrue(re.search(r"#define IS_CANDIDATE_VERSION\s+0", version)) + + self.Expect([ + Cmd("git status -s -uno", ""), + Cmd("git status -s -b -uno", "## some_branch\n"), + Cmd("git fetch", ""), + Cmd("git branch", " branch1\n* branch2\n"), + Cmd("git new-branch %s --upstream refs/remotes/origin/candidates" % + TEST_CONFIG["BRANCHNAME"], ""), + Cmd(("git log --format=%H --grep=\"Port ab12345\" " + "--reverse origin/master"), + "ab45678\nab23456"), + Cmd("git log -1 --format=%s ab45678", "Title1"), + Cmd("git log -1 --format=%s ab23456", "Title2"), + Cmd(("git log --format=%H --grep=\"Port ab23456\" " + "--reverse origin/master"), + ""), + Cmd(("git log --format=%H --grep=\"Port ab34567\" " + "--reverse origin/master"), + "ab56789"), + Cmd("git log -1 --format=%s ab56789", "Title3"), + RL("Y"), # Automatically add corresponding ports (ab34567, ab56789)? + # Simulate git being down which stops the script. + Cmd("git log -1 --format=%s ab12345", None), + # Restart script in the failing step. + Cmd("git log -1 --format=%s ab12345", "Title4"), + Cmd("git log -1 --format=%s ab23456", "Title2"), + Cmd("git log -1 --format=%s ab34567", "Title3"), + Cmd("git log -1 --format=%s ab45678", "Title1"), + Cmd("git log -1 --format=%s ab56789", "Revert \"Something\""), + Cmd("git log -1 ab12345", "Title4\nBUG=123\nBUG=234"), + Cmd("git log -1 ab23456", "Title2\n BUG = v8:123,345"), + Cmd("git log -1 ab34567", "Title3\nLOG=n\nBUG=567, 456"), + Cmd("git log -1 ab45678", "Title1\nBUG="), + Cmd("git log -1 ab56789", "Revert \"Something\"\nBUG=none"), + Cmd("git log -1 -p ab12345", "patch4"), + Cmd(("git apply --index --reject \"%s\"" % + TEST_CONFIG["TEMPORARY_PATCH_FILE"]), + "", cb=VerifyPatch("patch4")), + Cmd("git log -1 -p ab23456", "patch2"), + Cmd(("git apply --index --reject \"%s\"" % + TEST_CONFIG["TEMPORARY_PATCH_FILE"]), + "", cb=VerifyPatch("patch2")), + Cmd("git log -1 -p ab34567", "patch3"), + Cmd(("git apply --index --reject \"%s\"" % + TEST_CONFIG["TEMPORARY_PATCH_FILE"]), + "", cb=VerifyPatch("patch3")), + Cmd("git log -1 -p ab45678", "patch1"), + Cmd(("git apply --index --reject \"%s\"" % + TEST_CONFIG["TEMPORARY_PATCH_FILE"]), + "", cb=VerifyPatch("patch1")), + Cmd("git log -1 -p ab56789", "patch5\n"), + Cmd(("git apply --index --reject \"%s\"" % + TEST_CONFIG["TEMPORARY_PATCH_FILE"]), + "", cb=VerifyPatch("patch5\n")), + Cmd("git apply --index --reject \"%s\"" % extra_patch, ""), + RL("Y"), # Automatically increment patch level? + Cmd("git commit -aF \"%s\"" % TEST_CONFIG["COMMITMSG_FILE"], ""), + RL("reviewer@chromium.org"), # V8 reviewer. + Cmd("git cl upload --send-mail -r \"reviewer@chromium.org\" " + "--bypass-hooks --cc \"ulan@chromium.org\"", ""), + Cmd("git checkout -f %s" % TEST_CONFIG["BRANCHNAME"], ""), + RL("LGTM"), # Enter LGTM for V8 CL. + Cmd("git cl presubmit", "Presubmit successfull\n"), + Cmd("git cl land -f --bypass-hooks", "Closing issue\n", + cb=VerifyLand), + Cmd("git fetch", ""), + Cmd("git log -1 --format=%H --grep=\"" + "Version 3.22.5.1 (cherry-pick)" + "\" refs/remotes/origin/candidates", + ""), + Cmd("git fetch", ""), + Cmd("git log -1 --format=%H --grep=\"" + "Version 3.22.5.1 (cherry-pick)" + "\" refs/remotes/origin/candidates", + "hsh_to_tag"), + Cmd("git tag 3.22.5.1 hsh_to_tag", ""), + Cmd("git push origin 3.22.5.1", ""), + Cmd("git checkout -f some_branch", ""), + Cmd("git branch -D %s" % TEST_CONFIG["BRANCHNAME"], ""), + ]) + + # ab12345 and ab34567 are patches. ab23456 (included) and ab45678 are the + # MIPS ports of ab12345. ab56789 is the MIPS port of ab34567. + args = ["-f", "-p", extra_patch, "--branch", "candidates", + "ab12345", "ab23456", "ab34567"] + + # The first run of the script stops because of git being down. + self.assertRaises(GitFailedException, + lambda: MergeToBranch(TEST_CONFIG, self).Run(args)) + + # Test that state recovery after restarting the script works. + args += ["-s", "4"] + MergeToBranch(TEST_CONFIG, self).Run(args) + + def testReleases(self): + c_hash1_commit_log = """Update V8 to Version 4.2.71. + +Cr-Commit-Position: refs/heads/master@{#5678} +""" + c_hash2_commit_log = """Revert something. + +BUG=12345 + +Reason: +> Some reason. +> Cr-Commit-Position: refs/heads/master@{#12345} +> git-svn-id: svn://svn.chromium.org/chrome/trunk/src@12345 003-1c4 + +Review URL: https://codereview.chromium.org/12345 + +Cr-Commit-Position: refs/heads/master@{#4567} +git-svn-id: svn://svn.chromium.org/chrome/trunk/src@4567 0039-1c4b + +""" + c_hash3_commit_log = """Simple. + +git-svn-id: svn://svn.chromium.org/chrome/trunk/src@3456 0039-1c4b + +""" + c_hash_234_commit_log = """Version 3.3.1.1 (cherry-pick). + +Merged abc12. + +Review URL: fake.com + +Cr-Commit-Position: refs/heads/candidates@{#234} +""" + c_hash_123_commit_log = """Version 3.3.1.0 + +git-svn-id: googlecode@123 0039-1c4b +""" + c_hash_345_commit_log = """Version 3.4.0. + +Cr-Commit-Position: refs/heads/candidates@{#345} +""" + c_hash_456_commit_log = """Version 4.2.71. + +Cr-Commit-Position: refs/heads/4.2.71@{#1} +""" + + json_output = self.MakeEmptyTempFile() + csv_output = self.MakeEmptyTempFile() + self.WriteFakeVersionFile() + + TEST_CONFIG["CHROMIUM"] = self.MakeEmptyTempDirectory() + chrome_dir = TEST_CONFIG["CHROMIUM"] + chrome_v8_dir = os.path.join(chrome_dir, "v8") + os.makedirs(chrome_v8_dir) + def WriteDEPS(revision): + TextToFile("Line\n \"v8_revision\": \"%s\",\n line\n" % revision, + os.path.join(chrome_dir, "DEPS")) + WriteDEPS(567) + + def ResetVersion(major, minor, build, patch=0): + return lambda: self.WriteFakeVersionFile(major=major, + minor=minor, + build=build, + patch=patch) + + def ResetDEPS(revision): + return lambda: WriteDEPS(revision) + + self.Expect([ + Cmd("git status -s -uno", ""), + Cmd("git status -s -b -uno", "## some_branch\n"), + Cmd("git fetch", ""), + Cmd("git branch", " branch1\n* branch2\n"), + Cmd("git new-branch %s" % TEST_CONFIG["BRANCHNAME"], ""), + Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""), + Cmd("git rev-list --max-age=395200 --tags", + "bad_tag\nhash_234\nhash_123\nhash_345\nhash_456\n"), + Cmd("git describe --tags bad_tag", "3.23.42-1-deadbeef"), + Cmd("git describe --tags hash_234", "3.3.1.1"), + Cmd("git describe --tags hash_123", "3.21.2"), + Cmd("git describe --tags hash_345", "3.22.3"), + Cmd("git describe --tags hash_456", "4.2.71"), + Cmd("git diff --name-only hash_234 hash_234^", VERSION_FILE), + Cmd("git checkout -f hash_234 -- %s" % VERSION_FILE, "", + cb=ResetVersion(3, 3, 1, 1)), + Cmd("git branch -r --contains hash_234", " branch-heads/3.3\n"), + Cmd("git log -1 --format=%B hash_234", c_hash_234_commit_log), + Cmd("git log -1 --format=%s hash_234", ""), + Cmd("git log -1 --format=%B hash_234", c_hash_234_commit_log), + Cmd("git log -1 --format=%ci hash_234", "18:15"), + Cmd("git checkout -f HEAD -- %s" % VERSION_FILE, "", + cb=ResetVersion(3, 22, 5)), + Cmd("git diff --name-only hash_123 hash_123^", VERSION_FILE), + Cmd("git checkout -f hash_123 -- %s" % VERSION_FILE, "", + cb=ResetVersion(3, 21, 2)), + Cmd("git branch -r --contains hash_123", " branch-heads/3.21\n"), + Cmd("git log -1 --format=%B hash_123", c_hash_123_commit_log), + Cmd("git log -1 --format=%s hash_123", ""), + Cmd("git log -1 --format=%B hash_123", c_hash_123_commit_log), + Cmd("git log -1 --format=%ci hash_123", "03:15"), + Cmd("git checkout -f HEAD -- %s" % VERSION_FILE, "", + cb=ResetVersion(3, 22, 5)), + Cmd("git diff --name-only hash_345 hash_345^", VERSION_FILE), + Cmd("git checkout -f hash_345 -- %s" % VERSION_FILE, "", + cb=ResetVersion(3, 22, 3)), + Cmd("git branch -r --contains hash_345", " origin/candidates\n"), + Cmd("git log -1 --format=%B hash_345", c_hash_345_commit_log), + Cmd("git log -1 --format=%s hash_345", ""), + Cmd("git log -1 --format=%B hash_345", c_hash_345_commit_log), + Cmd("git log -1 --format=%ci hash_345", ""), + Cmd("git checkout -f HEAD -- %s" % VERSION_FILE, "", + cb=ResetVersion(3, 22, 5)), + Cmd("git diff --name-only hash_456 hash_456^", VERSION_FILE), + Cmd("git checkout -f hash_456 -- %s" % VERSION_FILE, "", + cb=ResetVersion(4, 2, 71)), + Cmd("git branch -r --contains hash_456", " origin/4.2.71\n"), + Cmd("git log -1 --format=%B hash_456", c_hash_456_commit_log), + Cmd("git log -1 --format=%H 4.2.71", "hash_456"), + Cmd("git log -1 --format=%s hash_456", "Version 4.2.71"), + Cmd("git log -1 --format=%H hash_456^", "master_456"), + Cmd("git log -1 --format=%B master_456", + "Cr-Commit-Position: refs/heads/master@{#456}"), + Cmd("git log -1 --format=%B hash_456", c_hash_456_commit_log), + Cmd("git log -1 --format=%ci hash_456", "02:15"), + Cmd("git checkout -f HEAD -- %s" % VERSION_FILE, "", + cb=ResetVersion(3, 22, 5)), + Cmd("git status -s -uno", "", cwd=chrome_dir), + Cmd("git checkout -f master", "", cwd=chrome_dir), + Cmd("git pull", "", cwd=chrome_dir), + Cmd("git new-branch %s" % TEST_CONFIG["BRANCHNAME"], "", + cwd=chrome_dir), + Cmd("git fetch origin", "", cwd=chrome_v8_dir), + Cmd("git log --format=%H --grep=\"V8\"", + "c_hash0\nc_hash1\nc_hash2\nc_hash3\n", + cwd=chrome_dir), + Cmd("git diff --name-only c_hash0 c_hash0^", "", cwd=chrome_dir), + Cmd("git diff --name-only c_hash1 c_hash1^", "DEPS", cwd=chrome_dir), + Cmd("git checkout -f c_hash1 -- DEPS", "", + cb=ResetDEPS("hash_456"), + cwd=chrome_dir), + Cmd("git log -1 --format=%B c_hash1", c_hash1_commit_log, + cwd=chrome_dir), + Cmd("git diff --name-only c_hash2 c_hash2^", "DEPS", cwd=chrome_dir), + Cmd("git checkout -f c_hash2 -- DEPS", "", + cb=ResetDEPS("hash_345"), + cwd=chrome_dir), + Cmd("git log -1 --format=%B c_hash2", c_hash2_commit_log, + cwd=chrome_dir), + Cmd("git diff --name-only c_hash3 c_hash3^", "DEPS", cwd=chrome_dir), + Cmd("git checkout -f c_hash3 -- DEPS", "", cb=ResetDEPS("deadbeef"), + cwd=chrome_dir), + Cmd("git log -1 --format=%B c_hash3", c_hash3_commit_log, + cwd=chrome_dir), + Cmd("git checkout -f HEAD -- DEPS", "", cb=ResetDEPS("hash_567"), + cwd=chrome_dir), + Cmd("git branch -r", " weird/123\n branch-heads/7\n", cwd=chrome_dir), + Cmd("git checkout -f branch-heads/7 -- DEPS", "", + cb=ResetDEPS("hash_345"), + cwd=chrome_dir), + Cmd("git checkout -f HEAD -- DEPS", "", cb=ResetDEPS("hash_567"), + cwd=chrome_dir), + Cmd("git checkout -f master", "", cwd=chrome_dir), + Cmd("git branch -D %s" % TEST_CONFIG["BRANCHNAME"], "", cwd=chrome_dir), + Cmd("git checkout -f some_branch", ""), + Cmd("git branch -D %s" % TEST_CONFIG["BRANCHNAME"], ""), + ]) + + args = ["-c", TEST_CONFIG["CHROMIUM"], + "--json", json_output, + "--csv", csv_output, + "--max-releases", "1"] + Releases(TEST_CONFIG, self).Run(args) + + # Check expected output. + csv = ("4.2.71,4.2.71,1,5678,\r\n" + "3.22.3,candidates,345,4567:5677,\r\n" + "3.21.2,3.21,123,,\r\n" + "3.3.1.1,3.3,234,,abc12\r\n") + self.assertEquals(csv, FileToText(csv_output)) + + expected_json = [ + { + "revision": "1", + "revision_git": "hash_456", + "master_position": "456", + "master_hash": "master_456", + "patches_merged": "", + "version": "4.2.71", + "chromium_revision": "5678", + "branch": "4.2.71", + "review_link": "", + "date": "02:15", + "chromium_branch": "", + # FIXME(machenbach): Fix revisions link for git. + "revision_link": "https://code.google.com/p/v8/source/detail?r=1", + }, + { + "revision": "345", + "revision_git": "hash_345", + "master_position": "", + "master_hash": "", + "patches_merged": "", + "version": "3.22.3", + "chromium_revision": "4567:5677", + "branch": "candidates", + "review_link": "", + "date": "", + "chromium_branch": "7", + "revision_link": "https://code.google.com/p/v8/source/detail?r=345", + }, + { + "revision": "123", + "revision_git": "hash_123", + "patches_merged": "", + "master_position": "", + "master_hash": "", + "version": "3.21.2", + "chromium_revision": "", + "branch": "3.21", + "review_link": "", + "date": "03:15", + "chromium_branch": "", + "revision_link": "https://code.google.com/p/v8/source/detail?r=123", + }, + { + "revision": "234", + "revision_git": "hash_234", + "patches_merged": "abc12", + "master_position": "", + "master_hash": "", + "version": "3.3.1.1", + "chromium_revision": "", + "branch": "3.3", + "review_link": "fake.com", + "date": "18:15", + "chromium_branch": "", + "revision_link": "https://code.google.com/p/v8/source/detail?r=234", + }, + ] + self.assertEquals(expected_json, json.loads(FileToText(json_output))) + + +class SystemTest(unittest.TestCase): + def testReload(self): + options = ScriptsBase( + TEST_CONFIG, DEFAULT_SIDE_EFFECT_HANDLER, {}).MakeOptions([]) + step = MakeStep(step_class=PrepareChangeLog, number=0, state={}, config={}, + options=options, + side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER) + body = step.Reload( +"""------------------------------------------------------------------------ +r17997 | machenbach@chromium.org | 2013-11-22 11:04:04 +0100 (...) | 6 lines + +Prepare push to trunk. Now working on version 3.23.11. + +R=danno@chromium.org + +Review URL: https://codereview.chromium.org/83173002 + +------------------------------------------------------------------------""") + self.assertEquals( +"""Prepare push to trunk. Now working on version 3.23.11. + +R=danno@chromium.org + +Committed: https://code.google.com/p/v8/source/detail?r=17997""", body) |