tools: update GYP to 324dd166

PR-URL: https://github.com/nodejs/node/pull/14718
Refs: https://chromium.googlesource.com/external/gyp/+/324dd166b7c0b39d513026fa52d6280ac6d56770
Refs: https://github.com/refack/GYP/commit/324dd166b7c0b39d513026fa52d6280ac6d56770
Reviewed-By: James M Snell <jasnell@gmail.com>

10 files changed, 279 insertions, 383 deletions

diff --git a/tools/gyp/gyptest.py b/tools/gyp/gyptest.py
index 98957e1633..9930e78c7b 100755
--- a/tools/gyp/gyptest.py
+++ b/tools/gyp/gyptest.py
@@ -1,134 +1,19 @@
 #!/usr/bin/env python
-
 # Copyright (c) 2012 Google Inc. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-__doc__ = """
-gyptest.py -- test runner for GYP tests.
-"""
+"""gyptest.py -- test runner for GYP tests."""
+
+from __future__ import print_function
 
+import argparse
+import math
 import os
-import optparse
-import shlex
+import platform
 import subprocess
 import sys
-
-class CommandRunner(object):
-  """
-  Executor class for commands, including "commands" implemented by
-  Python functions.
-  """
-  verbose = True
-  active = True
-
-  def __init__(self, dictionary={}):
-    self.subst_dictionary(dictionary)
-
-  def subst_dictionary(self, dictionary):
-    self._subst_dictionary = dictionary
-
-  def subst(self, string, dictionary=None):
-    """
-    Substitutes (via the format operator) the values in the specified
-    dictionary into the specified command.
-
-    The command can be an (action, string) tuple.  In all cases, we
-    perform substitution on strings and don't worry if something isn't
-    a string.  (It's probably a Python function to be executed.)
-    """
-    if dictionary is None:
-      dictionary = self._subst_dictionary
-    if dictionary:
-      try:
-        string = string % dictionary
-      except TypeError:
-        pass
-    return string
-
-  def display(self, command, stdout=None, stderr=None):
-    if not self.verbose:
-      return
-    if type(command) == type(()):
-      func = command[0]
-      args = command[1:]
-      s = '%s(%s)' % (func.__name__, ', '.join(map(repr, args)))
-    if type(command) == type([]):
-      # TODO:  quote arguments containing spaces
-      # TODO:  handle meta characters?
-      s = ' '.join(command)
-    else:
-      s = self.subst(command)
-    if not s.endswith('\n'):
-      s += '\n'
-    sys.stdout.write(s)
-    sys.stdout.flush()
-
-  def execute(self, command, stdout=None, stderr=None):
-    """
-    Executes a single command.
-    """
-    if not self.active:
-      return 0
-    if type(command) == type(''):
-      command = self.subst(command)
-      cmdargs = shlex.split(command)
-      if cmdargs[0] == 'cd':
-         command = (os.chdir,) + tuple(cmdargs[1:])
-    if type(command) == type(()):
-      func = command[0]
-      args = command[1:]
-      return func(*args)
-    else:
-      if stdout is sys.stdout:
-        # Same as passing sys.stdout, except python2.4 doesn't fail on it.
-        subout = None
-      else:
-        # Open pipe for anything else so Popen works on python2.4.
-        subout = subprocess.PIPE
-      if stderr is sys.stderr:
-        # Same as passing sys.stderr, except python2.4 doesn't fail on it.
-        suberr = None
-      elif stderr is None:
-        # Merge with stdout if stderr isn't specified.
-        suberr = subprocess.STDOUT
-      else:
-        # Open pipe for anything else so Popen works on python2.4.
-        suberr = subprocess.PIPE
-      p = subprocess.Popen(command,
-                           shell=(sys.platform == 'win32'),
-                           stdout=subout,
-                           stderr=suberr)
-      p.wait()
-      if stdout is None:
-        self.stdout = p.stdout.read()
-      elif stdout is not sys.stdout:
-        stdout.write(p.stdout.read())
-      if stderr not in (None, sys.stderr):
-        stderr.write(p.stderr.read())
-      return p.returncode
-
-  def run(self, command, display=None, stdout=None, stderr=None):
-    """
-    Runs a single command, displaying it first.
-    """
-    if display is None:
-      display = command
-    self.display(display)
-    return self.execute(command, stdout, stderr)
-
-
-class Unbuffered(object):
-  def __init__(self, fp):
-    self.fp = fp
-  def write(self, arg):
-    self.fp.write(arg)
-    self.fp.flush()
-  def __getattr__(self, attr):
-    return getattr(self.fp, attr)
-
-sys.stdout = Unbuffered(sys.stdout)
-sys.stderr = Unbuffered(sys.stderr)
+import time
 
 
 def is_test_name(f):
@@ -138,8 +23,6 @@ def is_test_name(f):
 def find_all_gyptest_files(directory):
   result = []
   for root, dirs, files in os.walk(directory):
-    if '.svn' in dirs:
-      dirs.remove('.svn')
     result.extend([ os.path.join(root, f) for f in files if is_test_name(f) ])
   result.sort()
   return result
@@ -149,73 +32,68 @@ def main(argv=None):
   if argv is None:
     argv = sys.argv
 
-  usage = "gyptest.py [-ahlnq] [-f formats] [test ...]"
-  parser = optparse.OptionParser(usage=usage)
-  parser.add_option("-a", "--all", action="store_true",
-            help="run all tests")
-  parser.add_option("-C", "--chdir", action="store", default=None,
-            help="chdir to the specified directory")
-  parser.add_option("-f", "--format", action="store", default='',
-            help="run tests with the specified formats")
-  parser.add_option("-G", '--gyp_option', action="append", default=[],
-            help="Add -G options to the gyp command line")
-  parser.add_option("-l", "--list", action="store_true",
-            help="list available tests and exit")
-  parser.add_option("-n", "--no-exec", action="store_true",
-            help="no execute, just print the command line")
-  parser.add_option("--passed", action="store_true",
-            help="report passed tests")
-  parser.add_option("--path", action="append", default=[],
-            help="additional $PATH directory")
-  parser.add_option("-q", "--quiet", action="store_true",
-            help="quiet, don't print test command lines")
-  opts, args = parser.parse_args(argv[1:])
-
-  if opts.chdir:
-    os.chdir(opts.chdir)
-
-  if opts.path:
+  parser = argparse.ArgumentParser()
+  parser.add_argument("-a", "--all", action="store_true",
+      help="run all tests")
+  parser.add_argument("-C", "--chdir", action="store",
+      help="change to directory")
+  parser.add_argument("-f", "--format", action="store", default='',
+      help="run tests with the specified formats")
+  parser.add_argument("-G", '--gyp_option', action="append", default=[],
+      help="Add -G options to the gyp command line")
+  parser.add_argument("-l", "--list", action="store_true",
+      help="list available tests and exit")
+  parser.add_argument("-n", "--no-exec", action="store_true",
+      help="no execute, just print the command line")
+  parser.add_argument("--path", action="append", default=[],
+      help="additional $PATH directory")
+  parser.add_argument("-q", "--quiet", action="store_true",
+      help="quiet, don't print anything unless there are failures")
+  parser.add_argument("-v", "--verbose", action="store_true",
+      help="print configuration info and test results.")
+  parser.add_argument('tests', nargs='*')
+  args = parser.parse_args(argv[1:])
+
+  if args.chdir:
+    os.chdir(args.chdir)
+
+  if args.path:
     extra_path = [os.path.abspath(p) for p in opts.path]
     extra_path = os.pathsep.join(extra_path)
     os.environ['PATH'] = extra_path + os.pathsep + os.environ['PATH']
 
-  if not args:
-    if not opts.all:
+  if not args.tests:
+    if not args.all:
       sys.stderr.write('Specify -a to get all tests.\n')
       return 1
-    args = ['test']
+    args.tests = ['test']
 
   tests = []
-  for arg in args:
+  for arg in args.tests:
     if os.path.isdir(arg):
       tests.extend(find_all_gyptest_files(os.path.normpath(arg)))
     else:
       if not is_test_name(os.path.basename(arg)):
-        print >>sys.stderr, arg, 'is not a valid gyp test name.'
+        print(arg, 'is not a valid gyp test name.', file=sys.stderr)
         sys.exit(1)
       tests.append(arg)
 
-  if opts.list:
+  if args.list:
     for test in tests:
-      print test
+      print(test)
     sys.exit(0)
 
-  CommandRunner.verbose = not opts.quiet
-  CommandRunner.active = not opts.no_exec
-  cr = CommandRunner()
-
   os.environ['PYTHONPATH'] = os.path.abspath('test/lib')
-  if not opts.quiet:
-    sys.stdout.write('PYTHONPATH=%s\n' % os.environ['PYTHONPATH'])
 
-  passed = []
-  failed = []
-  no_result = []
+  if args.verbose:
+    print_configuration_info()
+
+  if args.gyp_option and not args.quiet:
+    print('Extra Gyp options: %s\n' % args.gyp_option)
 
-  if opts.format:
-    format_list = opts.format.split(',')
+  if args.format:
+    format_list = args.format.split(',')
   else:
-    # TODO:  not duplicate this mapping from pylib/gyp/__init__.py
     format_list = {
       'aix5':     ['make'],
       'freebsd7': ['make'],
@@ -223,53 +101,143 @@ def main(argv=None):
       'openbsd5': ['make'],
       'cygwin':   ['msvs'],
       'win32':    ['msvs', 'ninja'],
+      'linux':    ['make', 'ninja'],
       'linux2':   ['make', 'ninja'],
       'linux3':   ['make', 'ninja'],
-      'darwin':   ['make', 'ninja', 'xcode', 'xcode-ninja'],
+
+      # TODO: Re-enable xcode-ninja.
+      # https://bugs.chromium.org/p/gyp/issues/detail?id=530
+      # 'darwin':   ['make', 'ninja', 'xcode', 'xcode-ninja'],
+      'darwin':   ['make', 'ninja', 'xcode'],
     }[sys.platform]
 
-  for format in format_list:
-    os.environ['TESTGYP_FORMAT'] = format
-    if not opts.quiet:
-      sys.stdout.write('TESTGYP_FORMAT=%s\n' % format)
+  gyp_options = []
+  for option in args.gyp_option:
+    gyp_options += ['-G', option]
 
-    gyp_options = []
-    for option in opts.gyp_option:
-      gyp_options += ['-G', option]
-    if gyp_options and not opts.quiet:
-      sys.stdout.write('Extra Gyp options: %s\n' % gyp_options)
+  runner = Runner(format_list, tests, gyp_options, args.verbose)
+  runner.run()
 
-    for test in tests:
-      status = cr.run([sys.executable, test] + gyp_options,
-                      stdout=sys.stdout,
-                      stderr=sys.stderr)
-      if status == 2:
-        no_result.append(test)
-      elif status:
-        failed.append(test)
-      else:
-        passed.append(test)
-
-  if not opts.quiet:
-    def report(description, tests):
-      if tests:
-        if len(tests) == 1:
-          sys.stdout.write("\n%s the following test:\n" % description)
-        else:
-          fmt = "\n%s the following %d tests:\n"
-          sys.stdout.write(fmt % (description, len(tests)))
-        sys.stdout.write("\t" + "\n\t".join(tests) + "\n")
-
-    if opts.passed:
-      report("Passed", passed)
-    report("Failed", failed)
-    report("No result from", no_result)
-
-  if failed:
+  if not args.quiet:
+    runner.print_results()
+
+  if runner.failures:
     return 1
   else:
     return 0
 
 
+def print_configuration_info():
+  print('Test configuration:')
+  if sys.platform == 'darwin':
+    sys.path.append(os.path.abspath('test/lib'))
+    import TestMac
+    print('  Mac %s %s' % (platform.mac_ver()[0], platform.mac_ver()[2]))
+    print('  Xcode %s' % TestMac.Xcode.Version())
+  elif sys.platform == 'win32':
+    sys.path.append(os.path.abspath('pylib'))
+    import gyp.MSVSVersion
+    print('  Win %s %s\n' % platform.win32_ver()[0:2])
+    print('  MSVS %s' %
+          gyp.MSVSVersion.SelectVisualStudioVersion().Description())
+  elif sys.platform in ('linux', 'linux2'):
+    print('  Linux %s' % ' '.join(platform.linux_distribution()))
+  print('  Python %s' % platform.python_version())
+  print('  PYTHONPATH=%s' % os.environ['PYTHONPATH'])
+  print()
+
+
+class Runner(object):
+  def __init__(self, formats, tests, gyp_options, verbose):
+    self.formats = formats
+    self.tests = tests
+    self.verbose = verbose
+    self.gyp_options = gyp_options
+    self.failures = []
+    self.num_tests = len(formats) * len(tests)
+    num_digits = len(str(self.num_tests))
+    self.fmt_str = '[%%%dd/%%%dd] (%%s) %%s' % (num_digits, num_digits)
+    self.isatty = sys.stdout.isatty() and not self.verbose
+    self.env = os.environ.copy()
+    self.hpos = 0
+
+  def run(self):
+    run_start = time.time()
+
+    i = 1
+    for fmt in self.formats:
+      for test in self.tests:
+        self.run_test(test, fmt, i)
+        i += 1
+
+    if self.isatty:
+      self.erase_current_line()
+
+    self.took = time.time() - run_start
+
+  def run_test(self, test, fmt, i):
+    if self.isatty:
+      self.erase_current_line()
+
+    msg = self.fmt_str % (i, self.num_tests, fmt, test)
+    self.print_(msg)
+
+    start = time.time()
+    cmd = [sys.executable, test] + self.gyp_options
+    self.env['TESTGYP_FORMAT'] = fmt
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT, env=self.env)
+    proc.wait()
+    took = time.time() - start
+
+    stdout = proc.stdout.read().decode('utf8')
+    if proc.returncode == 2:
+      res = 'skipped'
+    elif proc.returncode:
+      res = 'failed'
+      self.failures.append('(%s) %s' % (test, fmt))
+    else:
+      res = 'passed'
+    res_msg = ' %s %.3fs' % (res, took)
+    self.print_(res_msg)
+
+    if (stdout and
+        not stdout.endswith('PASSED\n') and
+        not (stdout.endswith('NO RESULT\n'))):
+      print()
+      for l in stdout.splitlines():
+        print('    %s' % l)
+    elif not self.isatty:
+      print()
+
+  def print_(self, msg):
+    print(msg, end='')
+    index = msg.rfind('\n')
+    if index == -1:
+      self.hpos += len(msg)
+    else:
+      self.hpos = len(msg) - index
+    sys.stdout.flush()
+
+  def erase_current_line(self):
+    print('\b' * self.hpos + ' ' * self.hpos + '\b' * self.hpos, end='')
+    sys.stdout.flush()
+    self.hpos = 0
+
+  def print_results(self):
+    num_failures = len(self.failures)
+    if num_failures:
+      print()
+      if num_failures == 1:
+        print("Failed the following test:")
+      else:
+        print("Failed the following %d tests:" % num_failures)
+      print("\t" + "\n\t".join(sorted(self.failures)))
+      print()
+    print('Ran %d tests in %.3fs, %d failed.' % (self.num_tests, self.took,
+                                                 num_failures))
+    print()
+
+
 if __name__ == "__main__":
   sys.exit(main())
diff --git a/tools/gyp/pylib/gyp/MSVSVersion.py b/tools/gyp/pylib/gyp/MSVSVersion.py
index c981e64b87..44b958d5b3 100644
--- a/tools/gyp/pylib/gyp/MSVSVersion.py
+++ b/tools/gyp/pylib/gyp/MSVSVersion.py
@@ -13,6 +13,10 @@ import gyp
 import glob
 
 
+def JoinPath(*args):
+  return os.path.normpath(os.path.join(*args))
+
+
 class VisualStudioVersion(object):
   """Information regarding a version of Visual Studio."""
 
@@ -71,45 +75,59 @@ class VisualStudioVersion(object):
     of a user override."""
     return self.default_toolset
 
+
   def _SetupScriptInternal(self, target_arch):
     """Returns a command (with arguments) to be used to set up the
     environment."""
+    assert target_arch in ('x86', 'x64'), "target_arch not supported"
     # If WindowsSDKDir is set and SetEnv.Cmd exists then we are using the
     # depot_tools build tools and should run SetEnv.Cmd to set up the
     # environment. The check for WindowsSDKDir alone is not sufficient because
     # this is set by running vcvarsall.bat.
-    assert target_arch in ('x86', 'x64')
-    sdk_dir = os.environ.get('WindowsSDKDir')
-    if sdk_dir:
-      setup_path = os.path.normpath(os.path.join(sdk_dir, 'Bin/SetEnv.Cmd'))
+    sdk_dir = os.environ.get('WindowsSDKDir', '')
+    setup_path = JoinPath(sdk_dir, 'Bin', 'SetEnv.Cmd')
     if self.sdk_based and sdk_dir and os.path.exists(setup_path):
       return [setup_path, '/' + target_arch]
-    else:
-      # We don't use VC/vcvarsall.bat for x86 because vcvarsall calls
-      # vcvars32, which it can only find if VS??COMNTOOLS is set, which it
-      # isn't always.
-      if target_arch == 'x86':
-        if self.short_name >= '2013' and self.short_name[-1] != 'e' and (
-            os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or
-            os.environ.get('PROCESSOR_ARCHITEW6432') == 'AMD64'):
-          # VS2013 and later, non-Express have a x64-x86 cross that we want
-          # to prefer.
-          return [os.path.normpath(
-             os.path.join(self.path, 'VC/vcvarsall.bat')), 'amd64_x86']
-        # Otherwise, the standard x86 compiler.
-        return [os.path.normpath(
-          os.path.join(self.path, 'Common7/Tools/vsvars32.bat'))]
+
+    is_host_arch_x64 = (
+      os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or
+      os.environ.get('PROCESSOR_ARCHITEW6432') == 'AMD64'
+    )
+
+    # For VS2017 (and newer) it's fairly easy
+    if self.short_name >= '2017':
+      script_path = JoinPath(self.path,
+                             'VC', 'Auxiliary', 'Build', 'vcvarsall.bat')
+
+      # Always use a native executable, cross-compiling if necessary.
+      host_arch = 'amd64' if is_host_arch_x64 else 'x86'
+      msvc_target_arch = 'amd64' if target_arch == 'x64' else 'x86'
+      arg = host_arch
+      if host_arch != msvc_target_arch:
+        arg += '_' + msvc_target_arch
+
+      return [script_path, arg]
+
+    # We try to find the best version of the env setup batch.
+    vcvarsall = JoinPath(self.path, 'VC', 'vcvarsall.bat')
+    if target_arch == 'x86':
+      if self.short_name >= '2013' and self.short_name[-1] != 'e' and \
+         is_host_arch_x64:
+        # VS2013 and later, non-Express have a x64-x86 cross that we want
+        # to prefer.
+        return [vcvarsall, 'amd64_x86']
       else:
-        assert target_arch == 'x64'
-        arg = 'x86_amd64'
-        # Use the 64-on-64 compiler if we're not using an express
-        # edition and we're running on a 64bit OS.
-        if self.short_name[-1] != 'e' and (
-            os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or
-            os.environ.get('PROCESSOR_ARCHITEW6432') == 'AMD64'):
-          arg = 'amd64'
-        return [os.path.normpath(
-            os.path.join(self.path, 'VC/vcvarsall.bat')), arg]
+        # Otherwise, the standard x86 compiler. We don't use VC/vcvarsall.bat
+        # for x86 because vcvarsall calls vcvars32, which it can only find if
+        # VS??COMNTOOLS is set, which isn't guaranteed.
+        return [JoinPath(self.path, 'Common7', 'Tools', 'vsvars32.bat')]
+    elif target_arch == 'x64':
+      arg = 'x86_amd64'
+      # Use the 64-on-64 compiler if we're not using an express edition and
+      # we're running on a 64bit OS.
+      if self.short_name[-1] != 'e' and is_host_arch_x64:
+        arg = 'amd64'
+      return [vcvarsall, arg]
 
   def SetupScript(self, target_arch):
     script_data = self._SetupScriptInternal(target_arch)
diff --git a/tools/gyp/pylib/gyp/easy_xml.py b/tools/gyp/pylib/gyp/easy_xml.py
index bf949b6ac9..2522efb244 100644
--- a/tools/gyp/pylib/gyp/easy_xml.py
+++ b/tools/gyp/pylib/gyp/easy_xml.py
@@ -4,6 +4,7 @@
 
 import re
 import os
+import locale
 
 
 def XmlToString(content, encoding='utf-8', pretty=False):
@@ -116,6 +117,10 @@ def WriteXmlIfChanged(content, path, encoding='utf-8', pretty=False,
   if win32 and os.linesep != '\r\n':
     xml_string = xml_string.replace('\n', '\r\n')
 
+  default_encoding = locale.getdefaultlocale()[1]
+  if default_encoding and default_encoding.upper() != encoding.upper():
+    xml_string = xml_string.decode(default_encoding).encode(encoding)
+
   # Get the old content
   try:
     f = open(path, 'r')
diff --git a/tools/gyp/pylib/gyp/generator/compile_commands_json.py b/tools/gyp/pylib/gyp/generator/compile_commands_json.py
deleted file mode 100644
index 575db63c4e..0000000000
--- a/tools/gyp/pylib/gyp/generator/compile_commands_json.py
+++ /dev/null
@@ -1,115 +0,0 @@
-# Copyright (c) 2016 Ben Noordhuis <info@bnoordhuis.nl>. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import gyp.common
-import gyp.xcode_emulation
-import json
-import os
-
-generator_additional_non_configuration_keys = []
-generator_additional_path_sections = []
-generator_extra_sources_for_rules = []
-generator_filelist_paths = None
-generator_supports_multiple_toolsets = True
-generator_wants_sorted_dependencies = False
-
-# Lifted from make.py.  The actual values don't matter much.
-generator_default_variables = {
-  'CONFIGURATION_NAME': '$(BUILDTYPE)',
-  'EXECUTABLE_PREFIX': '',
-  'EXECUTABLE_SUFFIX': '',
-  'INTERMEDIATE_DIR': '$(obj).$(TOOLSET)/$(TARGET)/geni',
-  'PRODUCT_DIR': '$(builddir)',
-  'RULE_INPUT_DIRNAME': '%(INPUT_DIRNAME)s',
-  'RULE_INPUT_EXT': '$(suffix $<)',
-  'RULE_INPUT_NAME': '$(notdir $<)',
-  'RULE_INPUT_PATH': '$(abspath $<)',
-  'RULE_INPUT_ROOT': '%(INPUT_ROOT)s',
-  'SHARED_INTERMEDIATE_DIR': '$(obj)/gen',
-  'SHARED_LIB_PREFIX': 'lib',
-  'STATIC_LIB_PREFIX': 'lib',
-  'STATIC_LIB_SUFFIX': '.a',
-}
-
-
-def IsMac(params):
-  return 'mac' == gyp.common.GetFlavor(params)
-
-
-def CalculateVariables(default_variables, params):
-  default_variables.setdefault('OS', gyp.common.GetFlavor(params))
-
-
-def AddCommandsForTarget(cwd, target, params, per_config_commands):
-  output_dir = params['generator_flags']['output_dir']
-  for configuration_name, configuration in target['configurations'].iteritems():
-    builddir_name = os.path.join(output_dir, configuration_name)
-
-    if IsMac(params):
-      xcode_settings = gyp.xcode_emulation.XcodeSettings(target)
-      cflags = xcode_settings.GetCflags(configuration_name)
-      cflags_c = xcode_settings.GetCflagsC(configuration_name)
-      cflags_cc = xcode_settings.GetCflagsCC(configuration_name)
-    else:
-      cflags = configuration.get('cflags', [])
-      cflags_c = configuration.get('cflags_c', [])
-      cflags_cc = configuration.get('cflags_cc', [])
-
-    cflags_c = cflags + cflags_c
-    cflags_cc = cflags + cflags_cc
-
-    defines = configuration.get('defines', [])
-    defines = ['-D' + s for s in defines]
-
-    # TODO(bnoordhuis) Handle generated source files.
-    sources = target.get('sources', [])
-    sources = [s for s in sources if s.endswith('.c') or s.endswith('.cc')]
-
-    def resolve(filename):
-      return os.path.abspath(os.path.join(cwd, filename))
-
-    # TODO(bnoordhuis) Handle generated header files.
-    include_dirs = configuration.get('include_dirs', [])
-    include_dirs = [s for s in include_dirs if not s.startswith('$(obj)')]
-    includes = ['-I' + resolve(s) for s in include_dirs]
-
-    defines = gyp.common.EncodePOSIXShellList(defines)
-    includes = gyp.common.EncodePOSIXShellList(includes)
-    cflags_c = gyp.common.EncodePOSIXShellList(cflags_c)
-    cflags_cc = gyp.common.EncodePOSIXShellList(cflags_cc)
-
-    commands = per_config_commands.setdefault(configuration_name, [])
-    for source in sources:
-      file = resolve(source)
-      isc = source.endswith('.c')
-      cc = 'cc' if isc else 'c++'
-      cflags = cflags_c if isc else cflags_cc
-      command = ' '.join((cc, defines, includes, cflags,
-                          '-c', gyp.common.EncodePOSIXShellArgument(file)))
-      commands.append(dict(command=command, directory=output_dir, file=file))
-
-
-def GenerateOutput(target_list, target_dicts, data, params):
-  per_config_commands = {}
-  for qualified_target, target in target_dicts.iteritems():
-    build_file, target_name, toolset = (
-        gyp.common.ParseQualifiedTarget(qualified_target))
-    if IsMac(params):
-      settings = data[build_file]
-      gyp.xcode_emulation.MergeGlobalXcodeSettingsToSpec(settings, target)
-    cwd = os.path.dirname(build_file)
-    AddCommandsForTarget(cwd, target, params, per_config_commands)
-
-  output_dir = params['generator_flags']['output_dir']
-  for configuration_name, commands in per_config_commands.iteritems():
-    filename = os.path.join(output_dir,
-                            configuration_name,
-                            'compile_commands.json')
-    gyp.common.EnsureDirExists(filename)
-    fp = open(filename, 'w')
-    json.dump(commands, fp=fp, indent=0, check_circular=False)
-
-
-def PerformBuild(data, configurations, params):
-  pass
diff --git a/tools/gyp/pylib/gyp/generator/make.py b/tools/gyp/pylib/gyp/generator/make.py
index f7f519b3e6..e80ebaed78 100644
--- a/tools/gyp/pylib/gyp/generator/make.py
+++ b/tools/gyp/pylib/gyp/generator/make.py
@@ -147,7 +147,7 @@ cmd_alink_thin = rm -f $@ && $(AR.$(TOOLSET)) crsT $@ $(filter %.o,$^)
 # special "figure out circular dependencies" flags around the entire
 # input list during linking.
 quiet_cmd_link = LINK($(TOOLSET)) $@
-cmd_link = $(LINK.$(TOOLSET)) $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ -Wl,--start-group $(LD_INPUTS) $(LIBS) -Wl,--end-group
+cmd_link = $(LINK.$(TOOLSET)) $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ -Wl,--start-group $(LD_INPUTS) -Wl,--end-group $(LIBS)
 
 # We support two kinds of shared objects (.so):
 # 1) shared_library, which is just bundling together many dependent libraries
@@ -2074,10 +2074,10 @@ def GenerateOutput(target_list, target_dicts, data, params):
     'AR.target':   GetEnvironFallback(('AR_target', 'AR'), '$(AR)'),
     'CXX.target':  GetEnvironFallback(('CXX_target', 'CXX'), '$(CXX)'),
     'LINK.target': GetEnvironFallback(('LINK_target', 'LINK'), '$(LINK)'),
-    'CC.host':     GetEnvironFallback(('CC_host', 'CC'), 'gcc'),
-    'AR.host':     GetEnvironFallback(('AR_host', 'AR'), 'ar'),
-    'CXX.host':    GetEnvironFallback(('CXX_host', 'CXX'), 'g++'),
-    'LINK.host':   GetEnvironFallback(('LINK_host', 'LINK'), '$(CXX.host)'),
+    'CC.host':     GetEnvironFallback(('CC_host',), 'gcc'),
+    'AR.host':     GetEnvironFallback(('AR_host',), 'ar'),
+    'CXX.host':    GetEnvironFallback(('CXX_host',), 'g++'),
+    'LINK.host':   GetEnvironFallback(('LINK_host',), '$(CXX.host)'),
   })
 
   build_file, _, _ = gyp.common.ParseQualifiedTarget(target_list[0])
diff --git a/tools/gyp/pylib/gyp/generator/msvs.py b/tools/gyp/pylib/gyp/generator/msvs.py
index ab92979e5c..8fe9e5af23 100644
--- a/tools/gyp/pylib/gyp/generator/msvs.py
+++ b/tools/gyp/pylib/gyp/generator/msvs.py
@@ -306,9 +306,19 @@ def _ConfigWindowsTargetPlatformVersion(config_data, version):
         continue
       version = MSVSVersion._RegistryGetValue(key % ver, 'ProductVersion') or ''
       # Find a matching entry in sdk_dir\include.
-      names = sorted([x for x in os.listdir(r'%s\include' % sdk_dir)
+      expected_sdk_dir=r'%s\include' % sdk_dir
+      names = sorted([x for x in (os.listdir(expected_sdk_dir)
+                                  if os.path.isdir(expected_sdk_dir)
+                                  else []
+                                  )
                       if x.startswith(version)], reverse=True)
-      return names[0]
+      if names:
+        return names[0]
+      else:
+        print >> sys.stdout, (
+          'Warning: No include files found for '
+          'detected Windows SDK version %s' % (version)
+        )
 
 
 def _BuildCommandLineForRuleRaw(spec, cmd, cygwin_shell, has_input_path,
@@ -1717,14 +1727,17 @@ def _GetCopies(spec):
         src_bare = src[:-1]
         base_dir = posixpath.split(src_bare)[0]
         outer_dir = posixpath.split(src_bare)[1]
-        cmd = 'cd "%s" && xcopy /e /f /y "%s" "%s\\%s\\"' % (
-            _FixPath(base_dir), outer_dir, _FixPath(dst), outer_dir)
+        fixed_dst = _FixPath(dst)
+        full_dst = '"%s\\%s\\"' % (fixed_dst, outer_dir)
+        cmd = 'mkdir %s 2>nul & cd "%s" && xcopy /e /f /y "%s" %s' % (
+            full_dst, _FixPath(base_dir), outer_dir, full_dst)
         copies.append(([src], ['dummy_copies', dst], cmd,
-                       'Copying %s to %s' % (src, dst)))
+                       'Copying %s to %s' % (src, fixed_dst)))
       else:
+        fix_dst = _FixPath(cpy['destination'])
         cmd = 'mkdir "%s" 2>nul & set ERRORLEVEL=0 & copy /Y "%s" "%s"' % (
-            _FixPath(cpy['destination']), _FixPath(src), _FixPath(dst))
-        copies.append(([src], [dst], cmd, 'Copying %s to %s' % (src, dst)))
+            fix_dst, _FixPath(src), _FixPath(dst))
+        copies.append(([src], [dst], cmd, 'Copying %s to %s' % (src, fix_dst)))
   return copies
 
 
@@ -2718,7 +2731,7 @@ def _GetMSBuildGlobalProperties(spec, version, guid, gyp_file_name):
     properties[0].append(['WindowsTargetPlatformVersion',
                           str(msvs_windows_sdk_version)])
   elif version.compatible_sdks:
-    raise GypError('%s requires any SDK of %o version, but non were found' %
+    raise GypError('%s requires any SDK of %s version, but none were found' %
                    (version.description, version.compatible_sdks))
 
   if platform_name == 'ARM':
diff --git a/tools/gyp/pylib/gyp/generator/ninja.py b/tools/gyp/pylib/gyp/generator/ninja.py
index 163281b3e3..0555a4a90d 100644
--- a/tools/gyp/pylib/gyp/generator/ninja.py
+++ b/tools/gyp/pylib/gyp/generator/ninja.py
@@ -1931,10 +1931,6 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
       ld = os.path.join(build_to_root, value)
     if key == 'LD.host':
       ld_host = os.path.join(build_to_root, value)
-    if key == 'LDXX':
-      ldxx = os.path.join(build_to_root, value)
-    if key == 'LDXX.host':
-      ldxx_host = os.path.join(build_to_root, value)
     if key == 'NM':
       nm = os.path.join(build_to_root, value)
     if key == 'NM.host':
@@ -2028,7 +2024,6 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
                           CommandWithWrapper('CXX.host', wrappers, cxx_host))
     if flavor == 'win':
       master_ninja.variable('ld_host', ld_host)
-      master_ninja.variable('ldxx_host', ldxx_host)
     else:
       master_ninja.variable('ld_host', CommandWithWrapper(
           'LINK', wrappers, ld_host))
@@ -2153,13 +2148,13 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
       restat=True,
       command=mtime_preserving_solink_base % {'suffix': '@$link_file_list'},
       rspfile='$link_file_list',
-      rspfile_content='-Wl,--start-group $in $solibs $libs -Wl,--end-group',
+      rspfile_content='-Wl,--start-group $in -Wl,--end-group $solibs $libs',
       pool='link_pool')
     master_ninja.rule(
       'link',
       description='LINK $out',
       command=('$ld $ldflags -o $out '
-               '-Wl,--start-group $in $solibs $libs -Wl,--end-group'),
+               '-Wl,--start-group $in -Wl,--end-group $solibs $libs'),
       pool='link_pool')
   elif flavor == 'win':
     master_ninja.rule(
diff --git a/tools/gyp/pylib/gyp/mac_tool.py b/tools/gyp/pylib/gyp/mac_tool.py
index b0363cc393..0ad7e7a1b6 100755
--- a/tools/gyp/pylib/gyp/mac_tool.py
+++ b/tools/gyp/pylib/gyp/mac_tool.py
@@ -105,17 +105,21 @@ class MacTool(object):
 
     ibtool_section_re = re.compile(r'/\*.*\*/')
     ibtool_re = re.compile(r'.*note:.*is clipping its content')
-    ibtoolout = subprocess.Popen(args, stdout=subprocess.PIPE)
+    try:
+      stdout = subprocess.check_output(args)
+    except subprocess.CalledProcessError as e:
+      print(e.output)
+      raise
     current_section_header = None
-    for line in ibtoolout.stdout:
+    for line in stdout.splitlines():
       if ibtool_section_re.match(line):
         current_section_header = line
       elif not ibtool_re.match(line):
         if current_section_header:
-          sys.stdout.write(current_section_header)
+          print(current_section_header)
           current_section_header = None
-        sys.stdout.write(line)
-    return ibtoolout.returncode
+        print(line)
+    return 0
 
   def _ConvertToBinary(self, dest):
     subprocess.check_call([
diff --git a/tools/gyp/pylib/gyp/msvs_emulation.py b/tools/gyp/pylib/gyp/msvs_emulation.py
index 14daaec4c7..6d5b5bd234 100644
--- a/tools/gyp/pylib/gyp/msvs_emulation.py
+++ b/tools/gyp/pylib/gyp/msvs_emulation.py
@@ -30,6 +30,10 @@ def QuoteForRspFile(arg):
   # works more or less because most programs (including the compiler, etc.)
   # use that function to handle command line arguments.
 
+  # Use a heuristic to try to find args that are paths, and normalize them
+  if arg.find('/') > 0 or arg.count('/') > 1:
+    arg = os.path.normpath(arg)
+
   # For a literal quote, CommandLineToArgvW requires 2n+1 backslashes
   # preceding it, and results in n backslashes + the quote. So we substitute
   # in 2* what we match, +1 more, plus the quote.
@@ -269,8 +273,8 @@ class MsvsSettings(object):
   def AdjustLibraries(self, libraries):
     """Strip -l from library if it's specified with that."""
     libs = [lib[2:] if lib.startswith('-l') else lib for lib in libraries]
-    return [lib + '.lib' if not lib.endswith('.lib') \
-        and not lib.endswith('.obj') else lib for lib in libs]
+    return [lib + '.lib' if not lib.lower().endswith('.lib') else lib
+            for lib in libs]
 
   def _GetAndMunge(self, field, path, default, prefix, append, map):
     """Retrieve a value from |field| at |path| or return |default|. If
@@ -307,7 +311,10 @@ class MsvsSettings(object):
     # There's two levels of architecture/platform specification in VS. The
     # first level is globally for the configuration (this is what we consider
     # "the" config at the gyp level, which will be something like 'Debug' or
-    # 'Release_x64'), and a second target-specific configuration, which is an
+    # 'Release'), VS2015 and later only use this level
+    if self.vs_version.short_name >= 2015:
+      return config
+    # and a second target-specific configuration, which is an
     # override for the global one. |config| is remapped here to take into
     # account the local target-specific overrides to the global configuration.
     arch = self.GetArch(config)
@@ -469,8 +476,10 @@ class MsvsSettings(object):
         prefix='/arch:')
     cflags.extend(['/FI' + f for f in self._Setting(
         ('VCCLCompilerTool', 'ForcedIncludeFiles'), config, default=[])])
-    if self.vs_version.short_name in ('2013', '2013e', '2015'):
-      # New flag required in 2013 to maintain previous PDB behavior.
+    if self.vs_version.project_version >= 12.0:
+      # New flag introduced in VS2013 (project version 12.0) Forces writes to
+      # the program database (PDB) to be serialized through MSPDBSRV.EXE.
+      # https://msdn.microsoft.com/en-us/library/dn502518.aspx
       cflags.append('/FS')
     # ninja handles parallelism by itself, don't have the compiler do it too.
     cflags = filter(lambda x: not x.startswith('/MP'), cflags)
@@ -530,7 +539,8 @@ class MsvsSettings(object):
     """Returns the .def file from sources, if any.  Otherwise returns None."""
     spec = self.spec
     if spec['type'] in ('shared_library', 'loadable_module', 'executable'):
-      def_files = [s for s in spec.get('sources', []) if s.endswith('.def')]
+      def_files = [s for s in spec.get('sources', [])
+                   if s.lower().endswith('.def')]
       if len(def_files) == 1:
         return gyp_to_build_path(def_files[0])
       elif len(def_files) > 1:
diff --git a/tools/gyp/pylib/gyp/xcode_emulation.py b/tools/gyp/pylib/gyp/xcode_emulation.py
index 667c53695a..dba8e7699e 100644
--- a/tools/gyp/pylib/gyp/xcode_emulation.py
+++ b/tools/gyp/pylib/gyp/xcode_emulation.py
@@ -1662,8 +1662,6 @@ def _GetXcodeEnv(xcode_settings, built_products_dir, srcroot, configuration,
     sdk_root = xcode_settings._SdkRoot(configuration)
     if not sdk_root:
       sdk_root = xcode_settings._XcodeSdkPath('')
-    if sdk_root is None:
-      sdk_root = ''
     env['SDKROOT'] = sdk_root
 
   if not additional_settings: