diff options
Diffstat (limited to 'deps/npm/lib/install')
21 files changed, 921 insertions, 807 deletions
diff --git a/deps/npm/lib/install/action/extract.js b/deps/npm/lib/install/action/extract.js index fd9562c184..7839177850 100644 --- a/deps/npm/lib/install/action/extract.js +++ b/deps/npm/lib/install/action/extract.js @@ -1,67 +1,56 @@ 'use strict' -var path = require('path') -var iferr = require('iferr') -var asyncMap = require('slide').asyncMap -var fs = require('graceful-fs') -var mkdirp = require('mkdirp') -var move = require('../../utils/move.js') -var gentlyRm = require('../../utils/gently-rm.js') -var updatePackageJson = require('../update-package-json') -var npm = require('../../npm.js') -var moduleName = require('../../utils/module-name.js') -var packageId = require('../../utils/package-id.js') -var cache = require('../../cache.js') -var moduleStagingPath = require('../module-staging-path.js') -var readPackageJson = require('read-package-json') -module.exports = function (staging, pkg, log, next) { - log.silly('extract', packageId(pkg)) - var up = npm.config.get('unsafe-perm') - var user = up ? null : npm.config.get('user') - var group = up ? null : npm.config.get('group') - var extractTo = moduleStagingPath(staging, pkg) - cache.unpack(pkg.package.name, pkg.package.version, extractTo, null, null, user, group, - andUpdatePackageJson(pkg, staging, extractTo, - andStageBundledChildren(pkg, staging, extractTo, log, - andRemoveExtraneousBundles(extractTo, next)))) -} - -function andUpdatePackageJson (pkg, staging, extractTo, next) { - return iferr(next, function () { - readPackageJson(path.join(extractTo, 'package.json'), false, function (err, metadata) { - if (!err) { - // Copy _ keys (internal to npm) and any missing keys from the possibly incomplete - // registry metadata over to the full package metadata read off of disk. - Object.keys(pkg.package).forEach(function (key) { - if (key[0] === '_' || !(key in metadata)) metadata[key] = pkg.package[key] - }) - metadata.name = pkg.package.name // things go wrong if these don't match - pkg.package = metadata - } - updatePackageJson(pkg, extractTo, next) - }) - }) -} +const BB = require('bluebird') -function andStageBundledChildren (pkg, staging, extractTo, log, next) { - return iferr(next, function () { - if (!pkg.package.bundleDependencies) return next() +const fs = BB.promisifyAll(require('graceful-fs')) +const gentlyRm = BB.promisify(require('../../utils/gently-rm.js')) +const log = require('npmlog') +const mkdirp = BB.promisify(require('mkdirp')) +const moduleName = require('../../utils/module-name.js') +const moduleStagingPath = require('../module-staging-path.js') +const move = BB.promisify(require('../../utils/move.js')) +const npa = require('npm-package-arg') +const npm = require('../../npm.js') +const packageId = require('../../utils/package-id.js') +const pacote = require('pacote') +const pacoteOpts = require('../../config/pacote') +const path = require('path') - asyncMap(pkg.children, andStageBundledModule(pkg, staging, extractTo), next) +module.exports = extract +function extract (staging, pkg, log) { + log.silly('extract', packageId(pkg)) + const up = npm.config.get('unsafe-perm') + const user = up ? null : npm.config.get('user') + const group = up ? null : npm.config.get('group') + const extractTo = moduleStagingPath(staging, pkg) + const opts = pacoteOpts({ + uid: user, + gid: group, + integrity: pkg.package._integrity }) -} - -function andRemoveExtraneousBundles (extractTo, next) { - return iferr(next, function () { - gentlyRm(path.join(extractTo, 'node_modules'), next) + return pacote.extract( + pkg.package._resolved + ? npa.resolve(pkg.package.name, pkg.package._resolved) + : pkg.package._requested, + extractTo, + opts + ).then(() => { + if (pkg.package.bundleDependencies) { + return readBundled(pkg, staging, extractTo) + } + }).then(() => { + return gentlyRm(path.join(extractTo, 'node_modules')) }) } -function andStageBundledModule (bundler, staging, parentPath) { - return function (child, next) { - if (child.error) return next(child.error) - stageBundledModule(bundler, child, staging, parentPath, next) - } +function readBundled (pkg, staging, extractTo) { + return BB.map(pkg.children, (child) => { + if (child.error) { + throw child.error + } else { + return stageBundledModule(pkg, child, staging, extractTo) + } + }, {concurrency: 10}) } function getTree (pkg) { @@ -70,47 +59,43 @@ function getTree (pkg) { } function warn (pkg, code, msg) { - var tree = getTree(pkg) - var err = new Error(msg) + const tree = getTree(pkg) + const err = new Error(msg) err.code = code tree.warnings.push(err) } -function stageBundledModule (bundler, child, staging, parentPath, next) { - var stageFrom = path.join(parentPath, 'node_modules', child.package.name) - var stageTo = moduleStagingPath(staging, child) - - return asyncMap(child.children, andStageBundledModule(bundler, staging, stageFrom), iferr(next, finishModule)) +function stageBundledModule (bundler, child, staging, parentPath) { + const stageFrom = path.join(parentPath, 'node_modules', child.package.name) + const stageTo = moduleStagingPath(staging, child) - function finishModule () { - // If we were the one's who bundled this module… - if (child.fromBundle === bundler) { - return moveModule() + return BB.map(child.children, (child) => { + if (child.error) { + throw child.error } else { - return checkForReplacement() + return stageBundledModule(bundler, child, staging, stageFrom) } - } - - function moveModule () { - return mkdirp(path.dirname(stageTo), iferr(next, function () { - return move(stageFrom, stageTo, iferr(next, updateMovedPackageJson)) - })) - } + }).then(() => { + return finishModule(bundler, child, stageTo, stageFrom) + }) +} - function checkForReplacement () { - return fs.stat(stageFrom, function (notExists, exists) { - if (exists) { - warn(bundler, 'EBUNDLEOVERRIDE', 'In ' + packageId(bundler) + - ' replacing bundled version of ' + moduleName(child) + - ' with ' + packageId(child)) - return gentlyRm(stageFrom, next) - } else { - return next() - } +function finishModule (bundler, child, stageTo, stageFrom) { + // If we were the one's who bundled this module… + if (child.fromBundle === bundler) { + return mkdirp(path.dirname(stageTo)).then(() => { + return move(stageFrom, stageTo) }) - } - - function updateMovedPackageJson () { - updatePackageJson(child, stageTo, next) + } else { + return fs.statAsync(stageFrom).then(() => { + const bundlerId = packageId(bundler) + if (!getTree(bundler).warnings.some((w) => { + return w.code === 'EBUNDLEOVERRIDE' + })) { + warn(bundler, 'EBUNDLEOVERRIDE', `${bundlerId} had bundled packages that do not match the required version(s). They have been replaced with non-bundled versions.`) + } + log.verbose('bundle', `EBUNDLEOVERRIDE: Replacing ${bundlerId}'s bundled version of ${moduleName(child)} with ${packageId(child)}.`) + return gentlyRm(stageFrom) + }, () => {}) } } diff --git a/deps/npm/lib/install/action/fetch.js b/deps/npm/lib/install/action/fetch.js index 0e9146a0d5..474e00b05c 100644 --- a/deps/npm/lib/install/action/fetch.js +++ b/deps/npm/lib/install/action/fetch.js @@ -1,29 +1,12 @@ 'use strict' -// var cache = require('../../cache.js') -// var packageId = require('../../utils/package-id.js') -// var moduleName = require('../../utils/module-name.js') -module.exports = function (staging, pkg, log, next) { - next() -/* -// FIXME: Unnecessary as long as we have to have the tarball to resolve all deps, which -// is progressively seeming to be likely for the indefinite future. -// ALSO fails for local deps specified with relative URLs outside of the top level. +const packageId = require('../../utils/package-id.js') +const pacote = require('pacote') +const pacoteOpts = require('../../config/pacote') - var name = moduleName(pkg) - var version - switch (pkg.package._requested.type) { - case 'version': - case 'range': - version = pkg.package.version - break - case 'hosted': - name = name + '@' + pkg.package._requested.spec - break - default: - name = pkg.package._requested.raw - } +module.exports = fetch +function fetch (staging, pkg, log, next) { log.silly('fetch', packageId(pkg)) - cache.add(name, version, pkg.parent.path, false, next) -*/ + const opts = pacoteOpts({integrity: pkg.package._integrity}) + pacote.prefetch(pkg.package._requested, opts).then(() => next(), next) } diff --git a/deps/npm/lib/install/action/finalize.js b/deps/npm/lib/install/action/finalize.js index 03a71f4cc0..1e86475710 100644 --- a/deps/npm/lib/install/action/finalize.js +++ b/deps/npm/lib/install/action/finalize.js @@ -1,85 +1,94 @@ 'use strict' -var path = require('path') -var rimraf = require('rimraf') -var fs = require('graceful-fs') -var mkdirp = require('mkdirp') -var asyncMap = require('slide').asyncMap -var move = require('../../utils/move.js') -var gentlyRm = require('../../utils/gently-rm') -var moduleStagingPath = require('../module-staging-path.js') +const path = require('path') +const fs = require('graceful-fs') +const Bluebird = require('bluebird') +const rimraf = Bluebird.promisify(require('rimraf')) +const mkdirp = Bluebird.promisify(require('mkdirp')) +const lstat = Bluebird.promisify(fs.lstat) +const readdir = Bluebird.promisify(fs.readdir) +const symlink = Bluebird.promisify(fs.symlink) +const gentlyRm = require('../../utils/gently-rm') +const moduleStagingPath = require('../module-staging-path.js') +const move = require('move-concurrently') +const moveOpts = {fs: fs, Promise: Bluebird, maxConcurrency: 4} +const getRequested = require('../get-requested.js') -module.exports = function (staging, pkg, log, next) { - log.silly('finalize', pkg.path) +module.exports = function (staging, pkg, log) { + log.silly('finalize', pkg.realpath) - var extractedTo = moduleStagingPath(staging, pkg) + const extractedTo = moduleStagingPath(staging, pkg) - var delpath = path.join(path.dirname(pkg.path), '.' + path.basename(pkg.path) + '.DELETE') + const delpath = path.join(path.dirname(pkg.realpath), '.' + path.basename(pkg.realpath) + '.DELETE') + let movedDestAway = false - mkdirp(path.resolve(pkg.path, '..'), whenParentExists) - - function whenParentExists (mkdirEr) { - if (mkdirEr) return next(mkdirEr) - // We stat first, because we can't rely on ENOTEMPTY from Windows. - // Windows, by contrast, gives the generic EPERM of a folder already exists. - fs.lstat(pkg.path, destStatted) - } - - function destStatted (doesNotExist) { - if (doesNotExist) { - move(extractedTo, pkg.path, whenMoved) - } else { - moveAway() - } - } - - function whenMoved (moveEr) { - if (!moveEr) return next() - if (moveEr.code !== 'ENOTEMPTY' && moveEr.code !== 'EEXIST') return next(moveEr) - moveAway() + const requested = pkg.package._requested || getRequested(pkg) + if (requested.type === 'directory') { + return makeParentPath(pkg.path) + .then(() => symlink(pkg.realpath, pkg.path, 'junction')) + .catch((ex) => { + return rimraf(pkg.path).then(() => symlink(pkg.realpath, pkg.path, 'junction')) + }) + } else { + return makeParentPath(pkg.realpath) + .then(moveStagingToDestination) + .then(restoreOldNodeModules) + .catch((err) => { + if (movedDestAway) { + return rimraf(pkg.realpath).then(moveOldDestinationBack).then(() => { + throw err + }) + } else { + throw err + } + }) + .then(() => rimraf(delpath)) } - function moveAway () { - move(pkg.path, delpath, whenOldMovedAway) + function makeParentPath (dir) { + return mkdirp(path.dirname(dir)) } - function whenOldMovedAway (moveEr) { - if (moveEr) return next(moveEr) - move(extractedTo, pkg.path, whenConflictMoved) + function moveStagingToDestination () { + return destinationIsClear() + .then(actuallyMoveStaging) + .catch(() => moveOldDestinationAway().then(actuallyMoveStaging)) } - function whenConflictMoved (moveEr) { - // if we got an error we'll try to put back the original module back, - // succeed or fail though we want the original error that caused this - if (moveEr) return move(delpath, pkg.path, function () { next(moveEr) }) - fs.readdir(path.join(delpath, 'node_modules'), makeTarget) + function destinationIsClear () { + return lstat(pkg.realpath).then(() => { + throw new Error('destination exists') + }, () => {}) } - function makeTarget (readdirEr, files) { - if (readdirEr) return cleanup() - if (!files.length) return cleanup() - mkdirp(path.join(pkg.path, 'node_modules'), function (mkdirEr) { moveModules(mkdirEr, files) }) + function actuallyMoveStaging () { + return move(extractedTo, pkg.realpath, moveOpts) } - function moveModules (mkdirEr, files) { - if (mkdirEr) return next(mkdirEr) - asyncMap(files, function (file, done) { - var from = path.join(delpath, 'node_modules', file) - var to = path.join(pkg.path, 'node_modules', file) - move(from, to, done) - }, cleanup) + function moveOldDestinationAway () { + return rimraf(delpath).then(() => { + return move(pkg.realpath, delpath, moveOpts) + }).then(() => { movedDestAway = true }) } - function cleanup (moveEr) { - if (moveEr) return next(moveEr) - rimraf(delpath, afterCleanup) + function moveOldDestinationBack () { + return move(delpath, pkg.realpath, moveOpts).then(() => { movedDestAway = false }) } - function afterCleanup (rimrafEr) { - if (rimrafEr) log.warn('finalize', rimrafEr) - next() + function restoreOldNodeModules () { + if (!movedDestAway) return + return readdir(path.join(delpath, 'node_modules')).catch(() => []).then((modules) => { + if (!modules.length) return + return mkdirp(path.join(pkg.realpath, 'node_modules')).then(() => Bluebird.map(modules, (file) => { + const from = path.join(delpath, 'node_modules', file) + const to = path.join(pkg.realpath, 'node_modules', file) + return move(from, to, moveOpts) + })) + }) } } module.exports.rollback = function (top, staging, pkg, next) { - gentlyRm(pkg.path, false, top, next) + const requested = pkg.package._requested || getRequested(pkg) + if (requested.type === 'directory') return next() + gentlyRm(pkg.realpath, false, top, next) } diff --git a/deps/npm/lib/install/action/global-install.js b/deps/npm/lib/install/action/global-install.js index e4fd8d11d1..bdc121b693 100644 --- a/deps/npm/lib/install/action/global-install.js +++ b/deps/npm/lib/install/action/global-install.js @@ -8,7 +8,7 @@ module.exports = function (staging, pkg, log, next) { log.silly('global-install', packageId(pkg)) var globalRoot = path.resolve(npm.globalDir, '..') npm.config.set('global', true) - var install = new Installer(globalRoot, false, [pkg.package.name + '@' + pkg.package._requested.spec]) + var install = new Installer(globalRoot, false, [pkg.package.name + '@' + pkg.package._requested.fetchSpec]) install.link = false install.run(function () { npm.config.set('global', false) diff --git a/deps/npm/lib/install/action/refresh-package-json.js b/deps/npm/lib/install/action/refresh-package-json.js new file mode 100644 index 0000000000..337be0caf2 --- /dev/null +++ b/deps/npm/lib/install/action/refresh-package-json.js @@ -0,0 +1,38 @@ +'use strict' +const path = require('path') +const Bluebird = require('bluebird') +const readJson = Bluebird.promisify(require('read-package-json')) +const updatePackageJson = Bluebird.promisify(require('../update-package-json')) +const getRequested = require('../get-requested.js') + +module.exports = function (staging, pkg, log) { + log.silly('refresh-package-json', pkg.realpath) + + return readJson(path.join(pkg.path, 'package.json'), false).then((metadata) => { + Object.keys(pkg.package).forEach(function (key) { + if (key !== '_injectedFromShrinkwrap' && !isEmpty(pkg.package[key])) { + metadata[key] = pkg.package[key] + if (key === '_resolved' && metadata[key] == null && pkg.package._injectedFromShrinkwrap) { + metadata[key] = pkg.package._injectedFromShrinkwrap.resolved + } + } + }) + // These two sneak in and it's awful + delete metadata.readme + delete metadata.readmeFilename + + pkg.package = metadata + }).catch(() => 'ignore').then(() => { + const requested = pkg.package._requested || getRequested(pkg) + if (requested.type !== 'directory') { + return updatePackageJson(pkg, pkg.path) + } + }) +} + +function isEmpty (value) { + if (value == null) return true + if (Array.isArray(value)) return !value.length + if (typeof value === 'object') return !Object.keys(value).length + return false +} diff --git a/deps/npm/lib/install/action/update-linked.js b/deps/npm/lib/install/action/update-linked.js deleted file mode 100644 index 0babe10fdf..0000000000 --- a/deps/npm/lib/install/action/update-linked.js +++ /dev/null @@ -1,16 +0,0 @@ -'use strict' -var path = require('path') - -function getTop (pkg) { - if (pkg.target && pkg.target.parent) return getTop(pkg.target.parent) - if (pkg.parent) return getTop(pkg.parent) - return pkg.path -} - -module.exports = function (staging, pkg, log, next) { - if (pkg.package.version !== pkg.oldPkg.package.version) { - log.warn('update-linked', path.relative(getTop(pkg), pkg.path), 'needs updating to', pkg.package.version, - 'from', pkg.oldPkg.package.version, "but we can't, as it's a symlink") - } - next() -} diff --git a/deps/npm/lib/install/actions.js b/deps/npm/lib/install/actions.js index cb41217c02..912985e2c7 100644 --- a/deps/npm/lib/install/actions.js +++ b/deps/npm/lib/install/actions.js @@ -1,18 +1,16 @@ 'use strict' -var validate = require('aproba') -var chain = require('slide').chain -var asyncMap = require('slide').asyncMap -var limit = require('call-limit') -var iferr = require('iferr') -var npm = require('../npm.js') -var andFinishTracker = require('./and-finish-tracker.js') -var andAddParentToErrors = require('./and-add-parent-to-errors.js') -var failedDependency = require('./deps.js').failedDependency -var moduleName = require('../utils/module-name.js') -var reportOptionalFailure = require('./report-optional-failure.js') -var isInstallable = require('./validate-args.js').isInstallable - -var actions = {} + +const BB = require('bluebird') + +const andAddParentToErrors = require('./and-add-parent-to-errors.js') +const failedDependency = require('./deps.js').failedDependency +const isInstallable = BB.promisify(require('./validate-args.js').isInstallable) +const moduleName = require('../utils/module-name.js') +const npm = require('../npm.js') +const reportOptionalFailure = require('./report-optional-failure.js') +const validate = require('aproba') + +const actions = {} actions.fetch = require('./action/fetch.js') actions.extract = require('./action/extract.js') @@ -24,20 +22,19 @@ actions.prepare = require('./action/prepare.js') actions.finalize = require('./action/finalize.js') actions.remove = require('./action/remove.js') actions.move = require('./action/move.js') -actions['update-linked'] = require('./action/update-linked.js') actions['global-install'] = require('./action/global-install.js') actions['global-link'] = require('./action/global-link.js') +actions['refresh-package-json'] = require('./action/refresh-package-json.js') // FIXME: We wrap actions like three ways to sunday here. // Rewrite this to only work one way. Object.keys(actions).forEach(function (actionName) { var action = actions[actionName] - actions[actionName] = limit(function (staging, pkg, log, next) { - // top, buildpath, pkg, log - validate('SOOF', arguments) + actions[actionName] = (staging, pkg, log) => { + validate('SOO', [staging, pkg, log]) // refuse to run actions for failed packages - if (pkg.failed) return next() + if (pkg.failed) return BB.resolve() if (action.rollback) { if (!pkg.rollback) pkg.rollback = [] pkg.rollback.unshift(action.rollback) @@ -46,98 +43,137 @@ Object.keys(actions).forEach(function (actionName) { if (!pkg.commit) pkg.commit = [] pkg.commit.push(action.commit) } + + let actionP if (pkg.knownInstallable) { - return thenRunAction() + actionP = runAction(action, staging, pkg, log) } else { - return isInstallable(pkg.package, iferr(andDone(next), andMarkInstallable(thenRunAction))) - } - function andMarkInstallable (cb) { - return function () { + actionP = isInstallable(pkg.package).then(() => { pkg.knownInstallable = true - cb() - } + return runAction(action, staging, pkg, log) + }) } - function thenRunAction () { - action(staging, pkg, log, andDone(next)) - } - function andDone (cb) { - return andFinishTracker(log, andAddParentToErrors(pkg.parent, andHandleOptionalDepErrors(pkg, cb))) - } - }, npm.limit.action) + + return actionP.then(() => { + log.finish() + }, (err) => { + return BB.fromNode((cb) => { + andAddParentToErrors(pkg.parent, cb)(err) + }).catch((err) => { + return handleOptionalDepErrors(pkg, err) + }) + }) + } }) +exports.actions = actions + +function runAction (action, staging, pkg, log) { + return BB.fromNode((cb) => { + const result = action(staging, pkg, log, cb) + if (result && result.then) { + result.then(() => cb(), cb) + } + }) +} function markAsFailed (pkg) { pkg.failed = true - pkg.requires.forEach(function (req) { - req.requiredBy = req.requiredBy.filter(function (reqReqBy) { return reqReqBy !== pkg }) - if (req.requiredBy.length === 0 && !req.userRequired && !req.existing) { + pkg.requires.forEach((req) => { + req.requiredBy = req.requiredBy.filter((reqReqBy) => { + return reqReqBy !== pkg + }) + if (req.requiredBy.length === 0 && !req.userRequired) { markAsFailed(req) } }) } -function andHandleOptionalDepErrors (pkg, next) { - return function (er) { - if (!er) return next.apply(null, arguments) - markAsFailed(pkg) - var anyFatal = pkg.userRequired || pkg.isTop - for (var ii = 0; ii < pkg.requiredBy.length; ++ii) { - var parent = pkg.requiredBy[ii] - var isFatal = failedDependency(parent, pkg) - if (isFatal) anyFatal = true - } - if (anyFatal) return next.apply(null, arguments) - reportOptionalFailure(pkg, null, er) - next() +function handleOptionalDepErrors (pkg, err) { + markAsFailed(pkg) + var anyFatal = pkg.userRequired || pkg.isTop + for (var ii = 0; ii < pkg.requiredBy.length; ++ii) { + var parent = pkg.requiredBy[ii] + var isFatal = failedDependency(parent, pkg) + if (isFatal) anyFatal = true } -} - -function prepareAction (staging, log) { - validate('SO', arguments) - return function (action) { - validate('SO', action) - var cmd = action[0] - var pkg = action[1] - if (!actions[cmd]) throw new Error('Unknown decomposed command "' + cmd + '" (is it new?)') - return [actions[cmd], staging, pkg, log.newGroup(cmd + ':' + moduleName(pkg))] + if (anyFatal) { + throw err + } else { + reportOptionalFailure(pkg, null, err) } } -exports.actions = actions - -function execAction (todo, done) { - validate('AF', arguments) - var cmd = todo.shift() - todo.push(done) - cmd.apply(null, todo) +exports.doOne = doOne +function doOne (cmd, staging, pkg, log, next) { + validate('SSOOF', arguments) + execAction(prepareAction([cmd, pkg], staging, log)).then(() => next(), next) } -exports.doOne = function (cmd, staging, pkg, log, next) { - validate('SSOOF', arguments) - execAction(prepareAction(staging, log)([cmd, pkg]), next) +exports.doParallel = doParallel +function doParallel (type, staging, actionsToRun, log, next) { + validate('SSAOF', arguments) + const acts = actionsToRun.reduce((acc, todo) => { + if (todo[0] === type) { + acc.push(prepareAction(todo, staging, log)) + } + return acc + }, []) + log.silly('doParallel', type + ' ' + actionsToRun.length) + time(log) + BB.map(acts, execAction, { + concurrency: npm.limit.action + }).nodeify((err) => { + log.finish() + timeEnd(log) + next(err) + }) } -exports.doSerial = function (type, staging, actionsToRun, log, next) { +exports.doSerial = doSerial +function doSerial (type, staging, actionsToRun, log, next) { validate('SSAOF', arguments) - actionsToRun = actionsToRun - .filter(function (value) { return value[0] === type }) log.silly('doSerial', '%s %d', type, actionsToRun.length) - chain(actionsToRun.map(prepareAction(staging, log)), andFinishTracker(log, next)) + runSerial(type, staging, actionsToRun, log, next) } -exports.doReverseSerial = function (type, staging, actionsToRun, log, next) { +exports.doReverseSerial = doReverseSerial +function doReverseSerial (type, staging, actionsToRun, log, next) { validate('SSAOF', arguments) - actionsToRun = actionsToRun - .filter(function (value) { return value[0] === type }) - .reverse() log.silly('doReverseSerial', '%s %d', type, actionsToRun.length) - chain(actionsToRun.map(prepareAction(staging, log)), andFinishTracker(log, next)) + runSerial(type, staging, actionsToRun.reverse(), log, next) } -exports.doParallel = function (type, staging, actionsToRun, log, next) { - validate('SSAOF', arguments) - actionsToRun = actionsToRun.filter(function (value) { return value[0] === type }) - log.silly('doParallel', type + ' ' + actionsToRun.length) +function runSerial (type, staging, actionsToRun, log, next) { + const acts = actionsToRun.reduce((acc, todo) => { + if (todo[0] === type) { + acc.push(prepareAction(todo, staging, log)) + } + return acc + }, []) + time(log) + BB.each(acts, execAction).nodeify((err) => { + log.finish() + timeEnd(log) + next(err) + }) +} + +function time (log) { + process.emit('time', 'action:' + log.name) +} +function timeEnd (log) { + process.emit('timeEnd', 'action:' + log.name) +} + +function prepareAction (action, staging, log) { + validate('ASO', arguments) + validate('SO', action) + var cmd = action[0] + var pkg = action[1] + if (!actions[cmd]) throw new Error('Unknown decomposed command "' + cmd + '" (is it new?)') + return [actions[cmd], staging, pkg, log.newGroup(cmd + ':' + moduleName(pkg))] +} - asyncMap(actionsToRun.map(prepareAction(staging, log)), execAction, andFinishTracker(log, next)) +function execAction (todo) { + return todo[0].apply(null, todo.slice(1)) } diff --git a/deps/npm/lib/install/copy-tree.js b/deps/npm/lib/install/copy-tree.js index 67a9c687a2..a5b558cf59 100644 --- a/deps/npm/lib/install/copy-tree.js +++ b/deps/npm/lib/install/copy-tree.js @@ -1,25 +1,31 @@ 'use strict' - -module.exports = function (tree) { - return copyTree(tree, {}) +var createNode = require('./node.js').create +module.exports = function (tree, filter) { + return copyTree(tree, {}, filter) } -function copyTree (tree, cache) { - if (cache[tree.path]) return cache[tree.path] - var newTree = cache[tree.path] = Object.create(tree) - copyModuleList(newTree, 'children', cache) +function copyTree (tree, cache, filter) { + if (filter && !filter(tree)) { return null } + if (cache[tree.path]) { return cache[tree.path] } + var newTree = cache[tree.path] = createNode(Object.assign({}, tree)) + copyModuleList(newTree, 'children', cache, filter) newTree.children.forEach(function (child) { child.parent = newTree }) - copyModuleList(newTree, 'requires', cache) - copyModuleList(newTree, 'requiredBy', cache) + copyModuleList(newTree, 'requires', cache, filter) + copyModuleList(newTree, 'requiredBy', cache, filter) return newTree } -function copyModuleList (tree, key, cache) { +function copyModuleList (tree, key, cache, filter) { var newList = [] - tree[key].forEach(function (child) { - newList.push(copyTree(child, cache)) - }) + if (tree[key]) { + tree[key].forEach(function (child) { + const copy = copyTree(child, cache, filter) + if (copy) { + newList.push(copy) + } + }) + } tree[key] = newList } diff --git a/deps/npm/lib/install/decompose-actions.js b/deps/npm/lib/install/decompose-actions.js index 70db70d035..1d954f5cab 100644 --- a/deps/npm/lib/install/decompose-actions.js +++ b/deps/npm/lib/install/decompose-actions.js @@ -19,7 +19,6 @@ module.exports = function (differences, decomposed, next) { moveSteps(decomposed, pkg, done) break case 'remove': - case 'update-linked': default: defaultSteps(decomposed, cmd, pkg, done) } @@ -27,10 +26,9 @@ module.exports = function (differences, decomposed, next) { } function addSteps (decomposed, pkg, done) { - if (!pkg.fromBundle) { + if (!pkg.fromBundle && !pkg.isLink) { decomposed.push(['fetch', pkg]) decomposed.push(['extract', pkg]) - decomposed.push(['test', pkg]) } if (!pkg.fromBundle || npm.config.get('rebuild-bundle')) { decomposed.push(['preinstall', pkg]) @@ -38,7 +36,10 @@ function addSteps (decomposed, pkg, done) { decomposed.push(['install', pkg]) decomposed.push(['postinstall', pkg]) } - decomposed.push(['finalize', pkg]) + if (!pkg.fromBundle || !pkg.isLink) { + decomposed.push(['finalize', pkg]) + } + decomposed.push(['refresh-package-json', pkg]) done() } @@ -52,7 +53,7 @@ function moveSteps (decomposed, pkg, done) { decomposed.push(['build', pkg]) decomposed.push(['install', pkg]) decomposed.push(['postinstall', pkg]) - decomposed.push(['test', pkg]) + decomposed.push(['refresh-package-json', pkg]) done() } diff --git a/deps/npm/lib/install/deps.js b/deps/npm/lib/install/deps.js index d1feb6cd4b..3f3433535f 100644 --- a/deps/npm/lib/install/deps.js +++ b/deps/npm/lib/install/deps.js @@ -1,20 +1,19 @@ 'use strict' + +const BB = require('bluebird') + +var fs = require('fs') var assert = require('assert') var path = require('path') var semver = require('semver') var asyncMap = require('slide').asyncMap var chain = require('slide').chain -var union = require('lodash.union') var iferr = require('iferr') var npa = require('npm-package-arg') var validate = require('aproba') -var realizePackageSpecifier = require('realize-package-specifier') -var realizeShrinkwrapSpecifier = require('./realize-shrinkwrap-specifier') -var asap = require('asap') var dezalgo = require('dezalgo') var fetchPackageMetadata = require('../fetch-package-metadata.js') var andAddParentToErrors = require('./and-add-parent-to-errors.js') -var addShrinkwrap = require('../fetch-package-metadata.js').addShrinkwrap var addBundled = require('../fetch-package-metadata.js').addBundled var readShrinkwrap = require('./read-shrinkwrap.js') var inflateShrinkwrap = require('./inflate-shrinkwrap.js') @@ -24,35 +23,17 @@ var npm = require('../npm.js') var flatNameFromTree = require('./flatten-tree.js').flatNameFromTree var createChild = require('./node.js').create var resetMetadata = require('./node.js').reset -var andIgnoreErrors = require('./and-ignore-errors.js') var isInstallable = require('./validate-args.js').isInstallable var packageId = require('../utils/package-id.js') var moduleName = require('../utils/module-name.js') var isDevDep = require('./is-dev-dep.js') var isProdDep = require('./is-prod-dep.js') var reportOptionalFailure = require('./report-optional-failure.js') +var getSaveType = require('./save.js').getSaveType // The export functions in this module mutate a dependency tree, adding // items to them. -function isDep (tree, child, cb) { - var name = moduleName(child) - var prodVer = isProdDep(tree, name) - var devVer = isDevDep(tree, name) - - childDependencySpecifier(tree, name, prodVer, function (er, prodSpec) { - if (er) return cb(child.fromShrinkwrap) - var matches - if (prodSpec) matches = doesChildVersionMatch(child, prodSpec, tree) - if (matches) return cb(true, prodSpec) - if (devVer === prodVer) return cb(child.fromShrinkwrap) - childDependencySpecifier(tree, name, devVer, function (er, devSpec) { - if (er) return cb(child.fromShrinkwrap) - cb(doesChildVersionMatch(child, devSpec, tree) || child.fromShrinkwrap, null, devSpec) - }) - }) -} - var registryTypes = { range: true, version: true } function doesChildVersionMatch (child, requested, requestor) { @@ -61,130 +42,121 @@ function doesChildVersionMatch (child, requested, requestor) { if (child.parent === requestor && child.fromShrinkwrap) return true // ranges of * ALWAYS count as a match, because when downloading we allow // prereleases to match * if there are ONLY prereleases - if (requested.spec === '*') return true + if (requested.type === 'range' && requested.fetchSpec === '*') return true - var childReq = child.package._requested - if (!childReq) childReq = npa(moduleName(child) + '@' + child.package._from) - if (childReq) { - if (childReq.rawSpec === requested.rawSpec) return true - if (childReq.type === requested.type && childReq.spec === requested.spec) return true + if (requested.type === 'directory') { + if (!child.isLink) return false + return path.relative(child.realpath, requested.fetchSpec) === '' } - // If _requested didn't exist OR if it didn't match then we'll try using - // _from. We pass it through npa to normalize the specifier. - // This can happen when installing from an `npm-shrinkwrap.json` where `_requested` will - // be the tarball URL from `resolved` and thus can't match what's in the `package.json`. - // In those cases _from, will be preserved and we can compare that to ensure that they - // really came from the same sources. - // You'll see this scenario happen with at least tags and git dependencies. + if (!registryTypes[requested.type]) { + var childReq = child.package._requested + if (!childReq && child.package._from) { + childReq = npa.resolve(moduleName(child), child.package._from.replace(new RegExp('^' + moduleName(child) + '@'), '')) + } + if (childReq) { + if (childReq.rawSpec === requested.rawSpec) return true + if (childReq.type === requested.type && childReq.saveSpec === requested.saveSpec) return true + if (childReq.type === requested.type && childReq.spec === requested.saveSpec) return true + } + // If _requested didn't exist OR if it didn't match then we'll try using + // _from. We pass it through npa to normalize the specifier. + // This can happen when installing from an `npm-shrinkwrap.json` where `_requested` will + // be the tarball URL from `resolved` and thus can't match what's in the `package.json`. + // In those cases _from, will be preserved and we can compare that to ensure that they + // really came from the same sources. + // You'll see this scenario happen with at least tags and git dependencies. if (child.package._from) { var fromReq = npa(child.package._from) if (fromReq.rawSpec === requested.rawSpec) return true - if (fromReq.type === requested.type && fromReq.spec === requested.spec) return true + if (fromReq.type === requested.type && fromReq.saveSpec && fromReq.saveSpec === requested.saveSpec) return true } return false } - return semver.satisfies(child.package.version, requested.spec) -} - -// TODO: Rename to maybe computeMetadata or computeRelationships -exports.recalculateMetadata = function (tree, log, next) { - recalculateMetadata(tree, log, {}, next) + try { + return semver.satisfies(child.package.version, requested.fetchSpec) + } catch (e) { + return false + } } -exports._childDependencySpecifier = childDependencySpecifier -function childDependencySpecifier (tree, name, spec, cb) { - if (!tree.resolved) tree.resolved = {} - if (!tree.resolved[name]) tree.resolved[name] = {} - if (tree.resolved[name][spec]) { - return asap(function () { - cb(null, tree.resolved[name][spec]) - }) - } - realizePackageSpecifier(name + '@' + spec, packageRelativePath(tree), function (er, req) { - if (er) return cb(er) - tree.resolved[name][spec] = req - cb(null, req) - }) +function childDependencySpecifier (tree, name, spec) { + return npa.resolve(name, spec, packageRelativePath(tree)) } -function recalculateMetadata (tree, log, seen, next) { - validate('OOOF', arguments) - if (seen[tree.path]) return next() +exports.computeMetadata = computeMetadata +function computeMetadata (tree, seen) { + if (!seen) seen = {} + if (!tree || seen[tree.path]) return seen[tree.path] = true if (tree.parent == null) { resetMetadata(tree) tree.isTop = true } + tree.location = flatNameFromTree(tree) - function markDeps (toMark, done) { - var name = toMark.name - var spec = toMark.spec - var kind = toMark.kind - childDependencySpecifier(tree, name, spec, function (er, req) { - if (er || !req.name) return done() - var child = findRequirement(tree, req.name, req) - if (child) { - resolveWithExistingModule(child, tree, log, andIgnoreErrors(done)) - } else if (kind === 'dep') { - tree.missingDeps[req.name] = req.rawSpec - done() - } else if (kind === 'dev') { - tree.missingDevDeps[req.name] = req.rawSpec - done() - } else { - done() - } - }) + function findChild (name, spec, kind) { + try { + var req = childDependencySpecifier(tree, name, spec) + } catch (err) { + return + } + var child = findRequirement(tree, req.name, req) + if (child) { + resolveWithExistingModule(child, tree) + return true + } + return } - function makeMarkable (deps, kind) { - if (!deps) return [] - return Object.keys(deps).map(function (depname) { return { name: depname, spec: deps[depname], kind: kind } }) + const deps = tree.package.dependencies || {} + for (let name of Object.keys(deps)) { + if (findChild(name, deps[name])) continue + tree.missingDeps[name] = deps[name] + } + if (tree.isTop) { + const devDeps = tree.package.devDependencies || {} + for (let name of Object.keys(devDeps)) { + if (findChild(name, devDeps[name])) continue + tree.missingDevDeps[name] = devDeps[name] + } } - // Ensure dependencies and dev dependencies are marked as required - var tomark = makeMarkable(tree.package.dependencies, 'dep') - if (tree.isTop) tomark = union(tomark, makeMarkable(tree.package.devDependencies, 'dev')) + tree.children.filter((child) => !child.removed && !child.failed).forEach((child) => computeMetadata(child, seen)) - // Ensure any children ONLY from a shrinkwrap are also included - var childrenOnlyInShrinkwrap = tree.children.filter(function (child) { - return child.fromShrinkwrap && - !tree.package.dependencies[child.package.name] && - !tree.package.devDependencies[child.package.name] - }) - var tomarkOnlyInShrinkwrap = childrenOnlyInShrinkwrap.map(function (child) { - var name = child.package.name - var matched = child.package._spec.match(/^@?[^@]+@(.*)$/) - var spec = matched ? matched[1] : child.package._spec - var kind = tree.package.dependencies[name] ? 'dep' - : tree.package.devDependencies[name] ? 'dev' - : 'dep' - return { name: name, spec: spec, kind: kind } - }) - tomark = union(tomark, tomarkOnlyInShrinkwrap) + return tree +} - // Don't bother trying to recalc children of failed deps - tree.children = tree.children.filter(function (child) { return !child.failed }) +function isDep (tree, child) { + var name = moduleName(child) + var prodVer = isProdDep(tree, name) + var devVer = isDevDep(tree, name) - chain([ - [asyncMap, tomark, markDeps], - [asyncMap, tree.children, function (child, done) { recalculateMetadata(child, log, seen, done) }] - ], function () { - tree.location = flatNameFromTree(tree) - next(null, tree) - }) + try { + var prodSpec = childDependencySpecifier(tree, name, prodVer) + } catch (err) { + return {isDep: true, isProdDep: false, isDevDep: false} + } + var matches + if (prodSpec) matches = doesChildVersionMatch(child, prodSpec, tree) + if (matches) return {isDep: true, isProdDep: prodSpec, isDevDep: false} + if (devVer === prodVer) return {isDep: child.fromShrinkwrap, isProdDep: false, isDevDep: false} + try { + var devSpec = childDependencySpecifier(tree, name, devVer) + return {isDep: doesChildVersionMatch(child, devSpec, tree) || child.fromShrinkwrap, isProdDep: false, isDevDep: devSpec} + } catch (err) { + return {isDep: child.fromShrinkwrap, isProdDep: false, isDevDep: false} + } } -function addRequiredDep (tree, child, cb) { - isDep(tree, child, function (childIsDep, childIsProdDep, childIsDevDep) { - if (!childIsDep) return cb(false) - replaceModuleByPath(child, 'requiredBy', tree) - replaceModuleByName(tree, 'requires', child) - if (childIsProdDep && tree.missingDeps) delete tree.missingDeps[moduleName(child)] - if (childIsDevDep && tree.missingDevDeps) delete tree.missingDevDeps[moduleName(child)] - cb(true) - }) +function addRequiredDep (tree, child) { + var dep = isDep(tree, child) + if (!dep.isDep) return false + replaceModuleByPath(child, 'requiredBy', tree) + replaceModuleByName(tree, 'requires', child) + if (dep.isProdDep && tree.missingDeps) delete tree.missingDeps[moduleName(child)] + if (dep.isDevDep && tree.missingDevDeps) delete tree.missingDevDeps[moduleName(child)] + return true } exports.removeObsoleteDep = removeObsoleteDep @@ -207,45 +179,38 @@ function removeObsoleteDep (child, log) { }) } -function matchingDep (tree, name) { - if (tree.package.dependencies && tree.package.dependencies[name]) return tree.package.dependencies[name] - if (tree.package.devDependencies && tree.package.devDependencies[name]) return tree.package.devDependencies[name] - return -} - function packageRelativePath (tree) { if (!tree) return '' var requested = tree.package._requested || {} - var isLocal = requested.type === 'directory' || requested.type === 'local' - return isLocal ? requested.spec : tree.path + var isLocal = requested.type === 'directory' || requested.type === 'file' + return isLocal ? requested.fetchSpec : tree.path } -function getShrinkwrap (tree, name) { - return tree.package._shrinkwrap && tree.package._shrinkwrap.dependencies && tree.package._shrinkwrap.dependencies[name] +function matchingDep (tree, name) { + if (!tree || !tree.package) return + if (tree.package.dependencies && tree.package.dependencies[name]) return tree.package.dependencies[name] + if (tree.package.devDependencies && tree.package.devDependencies[name]) return tree.package.devDependencies[name] + return } exports.getAllMetadata = function (args, tree, where, next) { asyncMap(args, function (arg, done) { - function fetchMetadataWithVersion () { - var version = matchingDep(tree, arg) - var spec = version == null ? arg : arg + '@' + version - return fetchPackageMetadata(spec, where, done) - } - if (tree && arg.lastIndexOf('@') <= 0) { - var sw = getShrinkwrap(tree, arg) - if (sw) { - return realizeShrinkwrapSpecifier(arg, sw, where, function (err, spec) { - if (err) { - return fetchMetadataWithVersion() + var spec = npa(arg) + if (spec.type !== 'file' && spec.type !== 'directory' && (spec.name == null || spec.rawSpec === '')) { + return fs.stat(path.join(arg, 'package.json'), (err) => { + if (err) { + var version = matchingDep(tree, spec.name) + if (version) { + return fetchPackageMetadata(npa.resolve(spec.name, version), where, done) } else { return fetchPackageMetadata(spec, where, done) } - }) - } else { - return fetchMetadataWithVersion() - } + } else { + return fetchPackageMetadata(npa('file:' + arg), where, done) + } + }) } else { - return fetchPackageMetadata(arg, where, done) + return fetchPackageMetadata(spec, where, done) } }, next) } @@ -261,13 +226,12 @@ exports.loadRequestedDeps = function (args, tree, saveToDependencies, log, next) child.isGlobal = true } var childName = moduleName(child) + child.saveSpec = computeVersionSpec(tree, child) if (saveToDependencies) { - tree.package[saveToDependencies][childName] = - child.package._requested.rawSpec || child.package._requested.spec + tree.package[getSaveType(tree, child)][childName] = child.saveSpec } - if (saveToDependencies && saveToDependencies !== 'devDependencies') { - tree.package.dependencies[childName] = - child.package._requested.rawSpec || child.package._requested.spec + if (getSaveType(tree, child) === 'optionalDependencies') { + tree.package.dependencies[childName] = child.saveSpec } child.userRequired = true child.save = saveToDependencies @@ -275,14 +239,32 @@ exports.loadRequestedDeps = function (args, tree, saveToDependencies, log, next) // For things the user asked to install, that aren't a dependency (or // won't be when we're done), flag it as "depending" on the user // themselves, so we don't remove it as a dep that no longer exists - addRequiredDep(tree, child, function (childIsDep) { - if (!childIsDep) child.userRequired = true - depLoaded(null, child, tracker) - }) + var childIsDep = addRequiredDep(tree, child) + if (!childIsDep) child.userRequired = true + depLoaded(null, child, tracker) })) }, andForEachChild(loadDeps, andFinishTracker(log, next))) } +function computeVersionSpec (tree, child) { + validate('OO', arguments) + var requested = child.package._requested + if (requested.registry) { + var version = child.package.version + var rangeDescriptor = '' + if (semver.valid(version, true) && + semver.gte(version, '0.1.0', true) && + !npm.config.get('save-exact')) { + rangeDescriptor = npm.config.get('save-prefix') + } + return rangeDescriptor + version + } else if (requested.type === 'directory' || requested.type === 'file') { + return 'file:' + path.relative(tree.path, requested.fetchSpec) + } else { + return requested.saveSpec + } +} + function moduleNameMatches (name) { return function (child) { return moduleName(child) === name } } @@ -299,11 +281,18 @@ exports.removeDeps = function (args, tree, saveToDependencies, log, next) { var pkgName = moduleName(pkg) var toRemove = tree.children.filter(moduleNameMatches(pkgName)) var pkgToRemove = toRemove[0] || createChild({package: {name: pkgName}}) - if (saveToDependencies) { - replaceModuleByPath(tree, 'removed', pkgToRemove) - pkgToRemove.save = saveToDependencies + if (tree.isTop) { + if (saveToDependencies) { + pkgToRemove.save = getSaveType(tree, pkg) + delete tree.package[pkgToRemove.save][pkgName] + if (pkgToRemove.save === 'optionalDependencies') { + delete tree.package.dependencies[pkgName] + } + replaceModuleByPath(tree, 'removed', pkgToRemove) + } + pkgToRemove.requiredBy = pkgToRemove.requiredBy.filter((parent) => parent !== tree) } - removeObsoleteDep(pkgToRemove) + if (pkgToRemove.requiredBy.length === 0) removeObsoleteDep(pkgToRemove) }) log.finish() next() @@ -387,6 +376,43 @@ function andHandleOptionalErrors (log, tree, name, done) { } } +exports.prefetchDeps = prefetchDeps +function prefetchDeps (tree, deps, log, next) { + validate('OOOF', arguments) + var skipOptional = !npm.config.get('optional') + var seen = {} + const finished = andFinishTracker(log, next) + const fpm = BB.promisify(fetchPackageMetadata) + resolveBranchDeps(tree.package, deps).then( + () => finished(), finished + ) + + function resolveBranchDeps (pkg, deps) { + return BB.resolve(null).then(() => { + var allDependencies = Object.keys(deps).map((dep) => { + return npa.resolve(dep, deps[dep]) + }).filter((dep) => { + return dep.registry && + !seen[dep.toString()] && + !findRequirement(tree, dep.name, dep) + }) + if (skipOptional) { + var optDeps = pkg.optionalDependencies || {} + allDependencies = allDependencies.filter((dep) => !optDeps[dep.name]) + } + return BB.map(allDependencies, (dep) => { + seen[dep.toString()] = true + return fpm(dep, '', {tracker: log.newItem('fetchMetadata')}).then( + (pkg) => { + return pkg && pkg.dependencies && resolveBranchDeps(pkg, pkg.dependencies) + }, + () => null + ) + }) + }) + } +} + // Load any missing dependencies in the given tree exports.loadDeps = loadDeps function loadDeps (tree, log, next) { @@ -427,15 +453,19 @@ exports.loadDevDeps = function (tree, log, next) { var loadExtraneous = exports.loadExtraneous = function (tree, log, next) { var seen = {} - function loadExtraneous (tree, log, next) { - validate('OOF', arguments) - if (seen[tree.path]) return next() + + function loadExtraneous (tree) { + if (seen[tree.path]) return seen[tree.path] = true - asyncMap(tree.children.filter(function (child) { return !child.loaded }), function (child, done) { - resolveWithExistingModule(child, tree, log, done) - }, andForEachChild(loadExtraneous, andFinishTracker(log, next))) + for (var child of tree.children) { + if (child.loaded) continue + resolveWithExistingModule(child, tree) + loadExtraneous(child) + } } - loadExtraneous(tree, log, next) + loadExtraneous(tree) + log.finish() + next() } exports.loadExtraneous.andResolveDeps = function (tree, log, next) { @@ -444,37 +474,38 @@ exports.loadExtraneous.andResolveDeps = function (tree, log, next) { // resolving the dependencies of extraneous deps. if (tree.loaded) return loadExtraneous(tree, log, next) asyncMap(tree.children.filter(function (child) { return !child.loaded }), function (child, done) { - resolveWithExistingModule(child, tree, log, done) + resolveWithExistingModule(child, tree) + done(null, child, log) }, andForEachChild(loadDeps, andFinishTracker(log, next))) } function addDependency (name, versionSpec, tree, log, done) { validate('SSOOF', arguments) var next = andAddParentToErrors(tree, done) - childDependencySpecifier(tree, name, versionSpec, iferr(done, function (req) { - var child = findRequirement(tree, name, req) - if (child) { - resolveWithExistingModule(child, tree, log, iferr(next, function (child, log) { - if (child.package._shrinkwrap === undefined) { - readShrinkwrap.andInflate(child, function (er) { next(er, child, log) }) - } else { - next(null, child, log) - } - })) + try { + var req = childDependencySpecifier(tree, name, versionSpec) + } catch (err) { + return done(err) + } + var child = findRequirement(tree, name, req) + if (child) { + resolveWithExistingModule(child, tree) + if (child.package._shrinkwrap === undefined) { + readShrinkwrap.andInflate(child, function (er) { next(er, child, log) }) } else { - fetchPackageMetadata(req, packageRelativePath(tree), {tracker: log.newItem('fetchMetadata')}, iferr(next, function (pkg) { - resolveWithNewModule(pkg, tree, log, next) - })) + next(null, child, log) } - })) + } else { + fetchPackageMetadata(req, packageRelativePath(tree), {tracker: log.newItem('fetchMetadata')}, iferr(next, function (pkg) { + resolveWithNewModule(pkg, tree, log, next) + })) + } } -function resolveWithExistingModule (child, tree, log, next) { - validate('OOOF', arguments) - addRequiredDep(tree, child, function () { - if (tree.parent && child.parent !== tree) updatePhantomChildren(tree.parent, child) - next(null, child, log) - }) +function resolveWithExistingModule (child, tree) { + validate('OO', arguments) + addRequiredDep(tree, child) + if (tree.parent && child.parent !== tree) updatePhantomChildren(tree.parent, child) } var updatePhantomChildren = exports.updatePhantomChildren = function (current, child) { @@ -521,44 +552,39 @@ function resolveWithNewModule (pkg, tree, log, next) { log.silly('resolveWithNewModule', packageId(pkg), 'checking installable status') return isInstallable(pkg, iferr(next, function () { - if (!pkg._from) { - pkg._from = pkg._requested.name + '@' + pkg._requested.spec - } - addShrinkwrap(pkg, iferr(next, function () { - addBundled(pkg, iferr(next, function () { - var parent = earliestInstallable(tree, tree, pkg) || tree - var child = createChild({ - package: pkg, - parent: parent, - path: path.join(parent.path, 'node_modules', pkg.name), - realpath: path.resolve(parent.realpath, 'node_modules', pkg.name), - children: pkg._bundled || [], - isLink: tree.isLink, - knownInstallable: true - }) - delete pkg._bundled - var hasBundled = child.children.length - - var replaced = replaceModuleByName(parent, 'children', child) - if (replaced) removeObsoleteDep(replaced) - addRequiredDep(tree, child, function () { - child.location = flatNameFromTree(child) + addBundled(pkg, iferr(next, function () { + var parent = earliestInstallable(tree, tree, pkg) || tree + var isLink = pkg._requested.type === 'directory' + var child = createChild({ + package: pkg, + parent: parent, + path: path.join(parent.isLink ? parent.realpath : parent.path, 'node_modules', pkg.name), + realpath: isLink ? pkg._requested.fetchSpec : path.join(parent.realpath, 'node_modules', pkg.name), + children: pkg._bundled || [], + isLink: isLink, + isInLink: parent.isLink, + knownInstallable: true + }) + delete pkg._bundled + var hasBundled = child.children.length - if (tree.parent && parent !== tree) updatePhantomChildren(tree.parent, child) + var replaced = replaceModuleByName(parent, 'children', child) + if (replaced) removeObsoleteDep(replaced) + addRequiredDep(tree, child) + child.location = flatNameFromTree(child) - if (hasBundled) { - inflateBundled(child, child, child.children) - } + if (tree.parent && parent !== tree) updatePhantomChildren(tree.parent, child) - if (pkg._shrinkwrap && pkg._shrinkwrap.dependencies) { - return inflateShrinkwrap(child, pkg._shrinkwrap.dependencies, function (er) { - next(er, child, log) - }) - } + if (hasBundled) { + inflateBundled(child, child, child.children) + } - next(null, child, log) + if (pkg._shrinkwrap && pkg._shrinkwrap.dependencies) { + return inflateShrinkwrap(child, pkg._shrinkwrap.dependencies, function (er) { + next(er, child, log) }) - })) + } + next(null, child, log) })) })) } @@ -567,7 +593,7 @@ var validatePeerDeps = exports.validatePeerDeps = function (tree, onInvalid) { if (!tree.package.peerDependencies) return Object.keys(tree.package.peerDependencies).forEach(function (pkgname) { var version = tree.package.peerDependencies[pkgname] - var match = findRequirement(tree.parent || tree, pkgname, npa(pkgname + '@' + version)) + var match = findRequirement(tree.parent || tree, pkgname, npa.resolve(pkgname, version)) if (!match) onInvalid(tree, pkgname, version) }) } @@ -590,7 +616,7 @@ var findRequirement = exports.findRequirement = function (tree, name, requested, validate('OSO', [tree, name, requested]) if (!requestor) requestor = tree var nameMatch = function (child) { - return moduleName(child) === name && child.parent && !child.removed + return moduleName(child) === name && child.parent && !child.removed && !child.failed } var versionMatch = function (child) { return doesChildVersionMatch(child, requested, requestor) @@ -618,7 +644,6 @@ var findRequirement = exports.findRequirement = function (tree, name, requested, // If it is, then it's the level below where its installed. var earliestInstallable = exports.earliestInstallable = function (requiredBy, tree, pkg) { validate('OOO', arguments) - function undeletedModuleMatches (child) { return !child.removed && moduleName(child) === pkg.name } @@ -645,7 +670,7 @@ var earliestInstallable = exports.earliestInstallable = function (requiredBy, tr var devDeps = tree.package.devDependencies || {} if (tree.isTop && devDeps[pkg.name]) { - var requested = npa(pkg.name + '@' + devDeps[pkg.name]) + var requested = npa.resolve(pkg.name, devDeps[pkg.name], tree.path) if (!doesChildVersionMatch({package: pkg}, requested, tree)) { return null } @@ -659,5 +684,7 @@ var earliestInstallable = exports.earliestInstallable = function (requiredBy, tr if (npm.config.get('global-style') && tree.parent.isTop) return tree if (npm.config.get('legacy-bundling')) return tree + if (!process.env.NODE_PRESERVE_SYMLINKS && /^[.][.][\\/]/.test(path.relative(tree.parent.realpath, tree.realpath))) return tree + return (earliestInstallable(requiredBy, tree.parent, pkg) || tree) } diff --git a/deps/npm/lib/install/diff-trees.js b/deps/npm/lib/install/diff-trees.js index 1429c71dcb..67fe72d044 100644 --- a/deps/npm/lib/install/diff-trees.js +++ b/deps/npm/lib/install/diff-trees.js @@ -9,13 +9,13 @@ function nonRegistrySource (pkg) { if (!requested) return false if (requested.type === 'hosted') return true - if (requested.type === 'local') return true + if (requested.type === 'file' || requested.type === 'directory') return true return false } function pkgAreEquiv (aa, bb) { - var aaSha = (aa.dist && aa.dist.shasum) || aa._shasum - var bbSha = (bb.dist && bb.dist.shasum) || bb._shasum + var aaSha = (aa.dist && aa.dist.integrity) || aa._integrity + var bbSha = (bb.dist && bb.dist.integrity) || bb._integrity if (aaSha === bbSha) return true if (aaSha || bbSha) return false if (nonRegistrySource(aa) || nonRegistrySource(bb)) return false @@ -24,13 +24,13 @@ function pkgAreEquiv (aa, bb) { } function getUniqueId (pkg) { - var versionspec = pkg._shasum + var versionspec = pkg._integrity if (!versionspec && nonRegistrySource(pkg)) { if (pkg._requested) { - versionspec = pkg._requested.spec + versionspec = pkg._requested.fetchSpec } else if (pkg._from) { - versionspec = npa(pkg._from).spec + versionspec = npa(pkg._from).fetchSpec } } if (!versionspec) { @@ -50,15 +50,6 @@ module.exports = function (oldTree, newTree, differences, log, next) { next() } -function isLink (node) { - return node && node.isLink -} - -function requiredByAllLinked (node) { - if (!node.requiredBy.length) return false - return node.requiredBy.filter(isLink).length === node.requiredBy.length -} - function isNotTopOrExtraneous (node) { return !node.isTop && !node.userRequired && !node.existing } @@ -136,16 +127,9 @@ var diffTrees = module.exports._diffTrees = function (oldTree, newTree) { Object.keys(flatNewTree).forEach(function (path) { var pkg = flatNewTree[path] pkg.oldPkg = flatOldTree[path] - pkg.isInLink = (pkg.oldPkg && isLink(pkg.oldPkg.parent)) || - (pkg.parent && isLink(pkg.parent)) || - requiredByAllLinked(pkg) if (pkg.oldPkg) { if (!pkg.userRequired && pkgAreEquiv(pkg.oldPkg.package, pkg.package)) return - if (!pkg.isInLink && (isLink(pkg.oldPkg) || isLink(pkg))) { - setAction(differences, 'update-linked', pkg) - } else { - setAction(differences, 'update', pkg) - } + setAction(differences, 'update', pkg) } else { var vername = getUniqueId(pkg.package) var removing = toRemoveByUniqueId[vername] && toRemoveByUniqueId[vername].length @@ -155,7 +139,7 @@ var diffTrees = module.exports._diffTrees = function (oldTree, newTree) { pkg.fromPath = toRemove[flatname].path setAction(differences, 'move', pkg) delete toRemove[flatname] - } else { + } else if (!(pkg.isInLink && pkg.fromBundle)) { setAction(differences, 'add', pkg) } } diff --git a/deps/npm/lib/install/filter-invalid-actions.js b/deps/npm/lib/install/filter-invalid-actions.js deleted file mode 100644 index beac30b7b0..0000000000 --- a/deps/npm/lib/install/filter-invalid-actions.js +++ /dev/null @@ -1,36 +0,0 @@ -'use strict' -var path = require('path') -var validate = require('aproba') -var log = require('npmlog') -var packageId = require('../utils/package-id.js') - -module.exports = function (top, differences, next) { - validate('SAF', arguments) - var action - var keep = [] - - differences.forEach(function (action) { - var cmd = action[0] - var pkg = action[1] - if (cmd === 'remove') { - pkg.removing = true - } - }) - - /*eslint no-cond-assign:0*/ - while (action = differences.shift()) { - var cmd = action[0] - var pkg = action[1] - if (pkg.isInLink || (pkg.parent && (pkg.parent.target || pkg.parent.isLink))) { - // we want to skip warning if this is a child of another module that we're removing - if (!pkg.parent.removing) { - log.verbose('skippingAction', 'Module is inside a symlinked module: not running ' + - cmd + ' ' + packageId(pkg) + ' ' + path.relative(top, pkg.path)) - } - } else { - keep.push(action) - } - } - differences.push.apply(differences, keep) - next() -} diff --git a/deps/npm/lib/install/get-requested.js b/deps/npm/lib/install/get-requested.js new file mode 100644 index 0000000000..f6c44d1463 --- /dev/null +++ b/deps/npm/lib/install/get-requested.js @@ -0,0 +1,12 @@ +'use strict' +const npa = require('npm-package-arg') +const moduleName = require('../utils/module-name.js') + +module.exports = function (child) { + if (!child.requiredBy.length) return + const reqBy = child.requiredBy[0] + const deps = reqBy.package.dependencies || {} + const devDeps = reqBy.package.devDependencies || {} + const name = moduleName(child) + return npa.resolve(name, deps[name] || devDeps[name], reqBy.realpath) +} diff --git a/deps/npm/lib/install/inflate-bundled.js b/deps/npm/lib/install/inflate-bundled.js index 5694841290..70da583df4 100644 --- a/deps/npm/lib/install/inflate-bundled.js +++ b/deps/npm/lib/install/inflate-bundled.js @@ -8,9 +8,10 @@ module.exports = function inflateBundled (bundler, parent, children) { children.forEach(function (child) { reset(child) child.fromBundle = bundler + child.isInLink = bundler.isLink child.parent = parent child.path = childPath(parent.path, child) - child.realpath = childPath(parent.path, child) + child.realpath = bundler.isLink ? child.realpath : childPath(parent.realpath, child) child.isLink = child.isLink || parent.isLink || parent.target inflateBundled(bundler, child, child.children) }) diff --git a/deps/npm/lib/install/inflate-shrinkwrap.js b/deps/npm/lib/install/inflate-shrinkwrap.js index aca4204930..9878b0f19a 100644 --- a/deps/npm/lib/install/inflate-shrinkwrap.js +++ b/deps/npm/lib/install/inflate-shrinkwrap.js @@ -1,105 +1,193 @@ 'use strict' -var asyncMap = require('slide').asyncMap -var validate = require('aproba') -var iferr = require('iferr') -var realizeShrinkwrapSpecifier = require('./realize-shrinkwrap-specifier.js') -var isRegistrySpecifier = require('./is-registry-specifier.js') -var fetchPackageMetadata = require('../fetch-package-metadata.js') -var annotateMetadata = require('../fetch-package-metadata.js').annotateMetadata -var addShrinkwrap = require('../fetch-package-metadata.js').addShrinkwrap -var addBundled = require('../fetch-package-metadata.js').addBundled -var inflateBundled = require('./inflate-bundled.js') -var npm = require('../npm.js') -var createChild = require('./node.js').create -var moduleName = require('../utils/module-name.js') -var childPath = require('../utils/child-path.js') + +const BB = require('bluebird') + +const addBundled = BB.promisify(require('../fetch-package-metadata.js').addBundled) +const childPath = require('../utils/child-path.js') +const createChild = require('./node.js').create +const fetchPackageMetadata = BB.promisify(require('../fetch-package-metadata.js')) +const inflateBundled = require('./inflate-bundled.js') +const moduleName = require('../utils/module-name.js') +const normalizePackageData = require('normalize-package-data') +const npm = require('../npm.js') +const realizeShrinkwrapSpecifier = require('./realize-shrinkwrap-specifier.js') +const validate = require('aproba') +const path = require('path') module.exports = function (tree, swdeps, finishInflating) { if (!npm.config.get('shrinkwrap')) return finishInflating() tree.loaded = true - return inflateShrinkwrap(tree.path, tree, swdeps, finishInflating) + return inflateShrinkwrap(tree.path, tree, swdeps).then( + () => finishInflating(), + finishInflating + ) } -function inflateShrinkwrap (topPath, tree, swdeps, finishInflating) { - validate('SOOF', arguments) - var onDisk = {} - tree.children.forEach(function (child) { onDisk[moduleName(child)] = child }) - var dev = npm.config.get('dev') || (!/^prod(uction)?$/.test(npm.config.get('only')) && !npm.config.get('production')) || /^dev(elopment)?$/.test(npm.config.get('only')) - var prod = !/^dev(elopment)?$/.test(npm.config.get('only')) - - // If the shrinkwrap has no dev dependencies in it then we'll leave the one's - // already on disk. If it DOES have dev dependencies then ONLY those in the - // shrinkwrap will be included. - var swHasDev = Object.keys(swdeps).some(function (name) { return swdeps[name].dev }) - tree.children = swHasDev ? [] : tree.children.filter(function (child) { - return tree.package.devDependencies[moduleName(child)] +function inflateShrinkwrap (topPath, tree, swdeps) { + validate('SOO', arguments) + const onDisk = {} + tree.children.forEach((child) => { + onDisk[moduleName(child)] = child }) + const dev = npm.config.get('dev') || (!/^prod(uction)?$/.test(npm.config.get('only')) && !npm.config.get('production')) || /^dev(elopment)?$/.test(npm.config.get('only')) + const prod = !/^dev(elopment)?$/.test(npm.config.get('only')) - return asyncMap(Object.keys(swdeps), doRealizeAndInflate, finishInflating) + tree.children = [] - function doRealizeAndInflate (name, next) { - return realizeShrinkwrapSpecifier(name, swdeps[name], topPath, iferr(next, andInflate(name, next))) - } + return BB.each(Object.keys(swdeps), (name) => { + const sw = swdeps[name] + if ( + (!prod && !sw.dev) || + (!dev && sw.dev) + ) { return null } + const dependencies = sw.dependencies || {} + const requested = realizeShrinkwrapSpecifier(name, sw, topPath) + return inflatableChild( + onDisk[name], name, topPath, tree, sw, requested + ).then((child) => { + return inflateShrinkwrap(topPath, child, dependencies) + }) + }) +} - function andInflate (name, next) { - return function (requested) { - var sw = swdeps[name] - var dependencies = sw.dependencies || {} - if ((!prod && !sw.dev) || (!dev && sw.dev)) return next() - var child = onDisk[name] - if (childIsEquivalent(sw, requested, child)) { - if (!child.fromShrinkwrap) child.fromShrinkwrap = requested.raw - if (sw.dev) child.shrinkwrapDev = true - tree.children.push(child) - annotateMetadata(child.package, requested, requested.raw, topPath) - return inflateShrinkwrap(topPath, child, dependencies || {}, next) - } else { - var from = sw.from || requested.raw - var optional = sw.optional - return fetchPackageMetadata(requested, topPath, iferr(next, andAddShrinkwrap(from, optional, dependencies, next))) - } - } +function normalizePackageDataNoErrors (pkg) { + try { + normalizePackageData(pkg) + } catch (ex) { + // don't care } +} - function andAddShrinkwrap (from, optional, dependencies, next) { - return function (pkg) { - pkg._from = from - pkg._optional = optional - addShrinkwrap(pkg, iferr(next, andAddBundled(pkg, dependencies, next))) - } +function inflatableChild (onDiskChild, name, topPath, tree, sw, requested) { + validate('OSSOOO|ZSSOOO', arguments) + if (onDiskChild && childIsEquivalent(sw, requested, onDiskChild)) { + // The version on disk matches the shrinkwrap entry. + if (!onDiskChild.fromShrinkwrap) onDiskChild.fromShrinkwrap = true + if (sw.dev) onDiskChild.shrinkwrapDev = true + onDiskChild.package._requested = requested + onDiskChild.package._spec = requested.rawSpec + onDiskChild.package._where = topPath + onDiskChild.fromBundle = sw.bundled ? tree.fromBundle || tree : null + if (!onDiskChild.package._args) onDiskChild.package._args = [] + onDiskChild.package._args.push([String(requested), topPath]) + // non-npm registries can and will return unnormalized data, plus + // even the npm registry may have package data normalized with older + // normalization rules. This ensures we get package data in a consistent, + // stable format. + normalizePackageDataNoErrors(onDiskChild.package) + tree.children.push(onDiskChild) + return BB.resolve(onDiskChild) + } else if (sw.version && sw.integrity) { + // The shrinkwrap entry has an integrity field. We can fake a pkg to get + // the installer to do a content-address fetch from the cache, if possible. + return BB.resolve(makeFakeChild(name, topPath, tree, sw, requested)) + } else { + // It's not on disk, and we can't just look it up by address -- do a full + // fpm/inflate bundle pass. For registry deps, this will go straight to the + // tarball URL, as if it were a remote tarball dep. + return fetchChild(topPath, tree, sw, requested) } +} - function andAddBundled (pkg, dependencies, next) { - return function () { - return addBundled(pkg, iferr(next, andAddChild(pkg, dependencies, next))) +function makeFakeChild (name, topPath, tree, sw, requested) { + const from = sw.from || requested.raw + const pkg = { + name: name, + version: sw.version, + _resolved: adaptResolved(requested, sw.resolved), + _requested: requested, + _optional: sw.optional, + _integrity: sw.integrity, + _from: from, + _spec: requested.rawSpec, + _where: topPath, + _args: [[requested.toString(), topPath]], + _injectedFromShrinkwrap: sw + } + let bundleAdded = BB.resolve() + if (Object.keys(sw.dependencies || {}).some((d) => { + return sw.dependencies[d].bundled + })) { + pkg.bundleDependencies = [] + bundleAdded = addBundled(pkg) + } + return bundleAdded.then(() => { + const child = createChild({ + package: pkg, + loaded: true, + parent: tree, + children: pkg._bundled || [], + fromShrinkwrap: true, + fromBundle: sw.bundled ? tree.fromBundle || tree : null, + path: childPath(tree.path, pkg), + realpath: childPath(tree.realpath, pkg), + location: tree.location + '/' + pkg.name, + isInLink: tree.isLink + }) + tree.children.push(child) + if (pkg._bundled) { + delete pkg._bundled + inflateBundled(child, child, child.children) } + return child + }) +} + +function adaptResolved (requested, resolved) { + const registry = requested.scope + ? npm.config.get(`${requested.scope}:registry`) || npm.config.get('registry') + : npm.config.get('registry') + if (!requested.registry || (resolved && resolved.indexOf(registry) === 0)) { + // Nothing to worry about here. Pass it through. + return resolved + } else { + // We could fast-path for registry.npmjs.org here, but if we do, it + // would end up getting written back to the `resolved` field. By always + // returning `null` for other registries, `pacote.extract()` will take + // care of any required metadata fetches internally, without altering + // the tree we're going to write out to shrinkwrap/lockfile. + return null } +} - function andAddChild (pkg, dependencies, next) { - return function () { - var child = createChild({ - package: pkg, - loaded: true, - parent: tree, - fromShrinkwrap: pkg._from, - path: childPath(tree.path, pkg), - realpath: childPath(tree.realpath, pkg), - children: pkg._bundled || [] - }) - tree.children.push(child) - if (pkg._bundled) { - delete pkg._bundled - inflateBundled(child, child, child.children) - } - inflateShrinkwrap(topPath, child, dependencies || {}, next) +function fetchChild (topPath, tree, sw, requested) { + const from = sw.from || requested.raw + const optional = sw.optional + return fetchPackageMetadata(requested, topPath).then((pkg) => { + pkg._from = from + pkg._optional = optional + return addBundled(pkg).then(() => pkg) + }).then((pkg) => { + var isLink = pkg._requested.type === 'directory' + const child = createChild({ + package: pkg, + loaded: true, + parent: tree, + fromShrinkwrap: requested, + path: childPath(tree.path, pkg), + realpath: isLink ? requested.fetchSpec : childPath(tree.realpath, pkg), + children: pkg._bundled || [], + location: tree.location + '/' + pkg.name, + isLink: isLink, + isInLink: tree.isLink + }) + tree.children.push(child) + if (pkg._bundled) { + delete pkg._bundled + inflateBundled(child, child, child.children) } - } + return child + }) } function childIsEquivalent (sw, requested, child) { if (!child) return false if (child.fromShrinkwrap) return true + if (sw.integrity && child.package._integrity === sw.integrity) return true + if (child.isLink && requested.type === 'directory') return path.relative(child.realpath, requested.fetchSpec) === '' + if (sw.resolved) return child.package._resolved === sw.resolved - if (!isRegistrySpecifier(requested) && sw.from) return child.package._from === sw.from + if (!requested.registry && sw.from) return child.package._from === sw.from + if (!requested.registry && child.package._resolved) return sw.version === child.package._resolved return child.package.version === sw.version } diff --git a/deps/npm/lib/install/is-registry-specifier.js b/deps/npm/lib/install/is-registry-specifier.js deleted file mode 100644 index 606be2bd13..0000000000 --- a/deps/npm/lib/install/is-registry-specifier.js +++ /dev/null @@ -1,6 +0,0 @@ -'use strict' -module.exports = isRegistrySpecifier - -function isRegistrySpecifier (spec) { - return spec.type === 'range' || spec.type === 'version' || spec.type === 'tag' -} diff --git a/deps/npm/lib/install/node.js b/deps/npm/lib/install/node.js index a5b766b054..b1b01fa8b9 100644 --- a/deps/npm/lib/install/node.js +++ b/deps/npm/lib/install/node.js @@ -18,26 +18,41 @@ var defaultTemplate = { realpath: null, location: null, userRequired: false, - existing: false, - isTop: false + save: false, + saveSpec: null, + isTop: false, + fromBundle: false } function isLink (node) { return node && node.isLink } -var create = exports.create = function (node, template) { +var create = exports.create = function (node, template, isNotTop) { if (!template) template = defaultTemplate Object.keys(template).forEach(function (key) { if (template[key] != null && typeof template[key] === 'object' && !(template[key] instanceof Array)) { if (!node[key]) node[key] = {} - return create(node[key], template[key]) + return create(node[key], template[key], true) } if (node[key] != null) return node[key] = template[key] }) - if (isLink(node.parent)) { - node.isLink = true + if (!isNotTop) { + // isLink is true for the symlink and everything inside it. + // by contrast, isInLink is true for only the things inside a link + if (node.isLink == null && isLink(node.parent)) { + node.isLink = true + node.isInLink = true + } else if (node.isLink == null) { + node.isLink = false + node.isInLink = false + } + if (node.fromBundle == null && node.package) { + node.fromBundle = node.package._inBundle + } else if (node.fromBundle == null) { + node.fromBundle = false + } } return node } diff --git a/deps/npm/lib/install/read-shrinkwrap.js b/deps/npm/lib/install/read-shrinkwrap.js index 3453e3192f..913c303482 100644 --- a/deps/npm/lib/install/read-shrinkwrap.js +++ b/deps/npm/lib/install/read-shrinkwrap.js @@ -1,25 +1,59 @@ 'use strict' -var path = require('path') -var fs = require('graceful-fs') -var iferr = require('iferr') -var inflateShrinkwrap = require('./inflate-shrinkwrap.js') -var parseJSON = require('../utils/parse-json.js') -var readShrinkwrap = module.exports = function (child, next) { +const BB = require('bluebird') + +const fs = require('graceful-fs') +const iferr = require('iferr') +const inflateShrinkwrap = require('./inflate-shrinkwrap.js') +const log = require('npmlog') +const parseJSON = require('../utils/parse-json.js') +const path = require('path') +const PKGLOCK_VERSION = require('../npm.js').lockfileVersion +const pkgSri = require('../utils/package-integrity.js') + +const readFileAsync = BB.promisify(fs.readFile) + +module.exports = readShrinkwrap +function readShrinkwrap (child, next) { if (child.package._shrinkwrap) return process.nextTick(next) - fs.readFile(path.join(child.path, 'npm-shrinkwrap.json'), function (er, data) { - if (er) { - child.package._shrinkwrap = null - return next() + BB.join( + maybeReadFile('npm-shrinkwrap.json', child), + // Don't read non-root lockfiles + child.isTop && maybeReadFile('package-lock.json', child), + child.isTop && maybeReadFile('package.json', child), + (shrinkwrap, lockfile, pkgJson) => { + if (shrinkwrap && lockfile) { + log.warn('read-shrinkwrap', 'Ignoring package-lock.json because there is already an npm-shrinkwrap.json. Please use only one of the two.') + } + const name = shrinkwrap ? 'npm-shrinkwrap.json' : 'package-lock.json' + let parsed = null + if (shrinkwrap || lockfile) { + try { + parsed = parseJSON(shrinkwrap || lockfile) + } catch (ex) { + throw ex + } + } + if ( + pkgJson && + parsed && + parsed.packageIntegrity && + !pkgSri.check(JSON.parse(pkgJson), parsed.packageIntegrity) + ) { + log.info('read-shrinkwrap', `${name} will be updated because package.json does not match what it was generated against.`) + } + if (parsed && parsed.lockfileVersion !== PKGLOCK_VERSION) { + log.warn('read-shrinkwrap', `This version of npm is compatible with lockfileVersion@${PKGLOCK_VERSION}, but ${name} was generated for lockfileVersion@${parsed.lockfileVersion || 0}. I'll try to do my best with it!`) + } + child.package._shrinkwrap = parsed } - try { - child.package._shrinkwrap = parseJSON(data) - } catch (ex) { - child.package._shrinkwrap = null - return next(ex) - } - return next() - }) + ).then(() => next(), next) +} + +function maybeReadFile (name, child) { + return readFileAsync( + path.join(child.path, name) + ).catch({code: 'ENOENT'}, () => null) } module.exports.andInflate = function (child, next) { diff --git a/deps/npm/lib/install/realize-shrinkwrap-specifier.js b/deps/npm/lib/install/realize-shrinkwrap-specifier.js index 0c491a6028..91030bfa82 100644 --- a/deps/npm/lib/install/realize-shrinkwrap-specifier.js +++ b/deps/npm/lib/install/realize-shrinkwrap-specifier.js @@ -1,25 +1,18 @@ 'use strict' -var realizePackageSpecifier = require('realize-package-specifier') -var isRegistrySpecifier = require('./is-registry-specifier.js') +var npa = require('npm-package-arg') -module.exports = function (name, sw, where, cb) { - function lookup (ver, cb) { - realizePackageSpecifier(name + '@' + ver, where, cb) - } - if (sw.resolved) { - return lookup(sw.resolved, cb) - } else if (sw.from) { - return lookup(sw.from, function (err, spec) { - if (err || isRegistrySpecifier(spec)) { - return thenUseVersion() - } else { - return cb(null, spec) - } - }) - } else { - return thenUseVersion() - } - function thenUseVersion () { - lookup(sw.version, cb) - } +module.exports = function (name, sw, where) { + try { + if (sw.version && sw.integrity) { + return npa.resolve(name, sw.version, where) + } + if (sw.resolved) { + return npa.resolve(name, sw.resolved, where) + } + if (sw.from) { + var spec = npa(sw.from, where) + if (!spec.registry) return spec + } + } catch (_) { } + return npa.resolve(name, sw.version, where) } diff --git a/deps/npm/lib/install/save.js b/deps/npm/lib/install/save.js index 18028a3c26..5d5f4e7f7a 100644 --- a/deps/npm/lib/install/save.js +++ b/deps/npm/lib/install/save.js @@ -1,19 +1,20 @@ 'use strict' -var fs = require('graceful-fs') -var path = require('path') -var url = require('url') -var writeFileAtomic = require('write-file-atomic') -var log = require('npmlog') -var semver = require('semver') -var iferr = require('iferr') -var validate = require('aproba') -var without = require('lodash.without') -var npm = require('../npm.js') -var deepSortObject = require('../utils/deep-sort-object.js') -var parseJSON = require('../utils/parse-json.js') -var moduleName = require('../utils/module-name.js') -var isDevDep = require('./is-dev-dep.js') -var createShrinkwrap = require('../shrinkwrap.js').createShrinkwrap + +const BB = require('bluebird') + +const createShrinkwrap = require('../shrinkwrap.js').createShrinkwrap +const deepSortObject = require('../utils/deep-sort-object.js') +const detectIndent = require('detect-indent') +const fs = BB.promisifyAll(require('graceful-fs')) +const iferr = require('iferr') +const log = require('npmlog') +const moduleName = require('../utils/module-name.js') +const npm = require('../npm.js') +const parseJSON = require('../utils/parse-json.js') +const path = require('path') +const validate = require('aproba') +const without = require('lodash.without') +const writeFileAtomic = require('write-file-atomic') // if the -S|--save option is specified, then write installed packages // as dependencies to a package.json file. @@ -42,30 +43,13 @@ function andWarnErrors (cb) { function saveShrinkwrap (tree, next) { validate('OF', arguments) - var saveTarget = path.resolve(tree.path, 'npm-shrinkwrap.json') - fs.stat(saveTarget, function (er, stat) { - if (er) return next() - var save = npm.config.get('save') - var saveDev = npm.config.get('save-dev') - var saveOptional = npm.config.get('save-optional') - - var shrinkwrap = tree.package._shrinkwrap || {dependencies: {}} - var shrinkwrapHasAnyDevOnlyDeps = tree.requires.some(function (dep) { - var name = moduleName(dep) - return isDevDep(tree, name) && - shrinkwrap.dependencies[name] != null - }) - - if (!saveOptional && saveDev && !shrinkwrapHasAnyDevOnlyDeps) return next() - if (saveOptional || !(save || saveDev)) return next() - - var silent = false - createShrinkwrap(tree.path, tree.package, shrinkwrapHasAnyDevOnlyDeps, silent, next) - }) + createShrinkwrap(tree, {silent: false}, next) } function savePackageJson (args, tree, next) { validate('AOF', arguments) + if (!args || !args.length) { return next() } + var saveBundle = npm.config.get('save-bundle') // each item in the tree is a top-level thing that should be saved @@ -74,33 +58,34 @@ function savePackageJson (args, tree, next) { var saveTarget = path.resolve(tree.path, 'package.json') // don't use readJson, because we don't want to do all the other // tricky npm-specific stuff that's in there. - fs.readFile(saveTarget, iferr(next, function (packagejson) { + fs.readFile(saveTarget, 'utf8', iferr(next, function (packagejson) { + const indent = detectIndent(packagejson).indent || ' ' try { - packagejson = parseJSON(packagejson) + tree.package = parseJSON(packagejson) } catch (ex) { return next(ex) } // If we're saving bundled deps, normalize the key before we start if (saveBundle) { - var bundle = packagejson.bundleDependencies || packagejson.bundledDependencies - delete packagejson.bundledDependencies + var bundle = tree.package.bundleDependencies || tree.package.bundledDependencies + delete tree.package.bundledDependencies if (!Array.isArray(bundle)) bundle = [] } var toSave = getThingsToSave(tree) - var toRemove = getThingsToRemove(args, tree) + var toRemove = getThingsToRemove(tree) var savingTo = {} toSave.forEach(function (pkg) { savingTo[pkg.save] = true }) toRemove.forEach(function (pkg) { savingTo[pkg.save] = true }) Object.keys(savingTo).forEach(function (save) { - if (!packagejson[save]) packagejson[save] = {} + if (!tree.package[save]) tree.package[save] = {} }) log.verbose('saving', toSave) toSave.forEach(function (pkg) { - packagejson[pkg.save][pkg.name] = pkg.spec + tree.package[pkg.save][pkg.name] = pkg.spec if (saveBundle) { var ii = bundle.indexOf(pkg.name) if (ii === -1) bundle.push(pkg.name) @@ -108,71 +93,46 @@ function savePackageJson (args, tree, next) { }) toRemove.forEach(function (pkg) { - delete packagejson[pkg.save][pkg.name] + delete tree.package[pkg.save][pkg.name] if (saveBundle) { bundle = without(bundle, pkg.name) } }) Object.keys(savingTo).forEach(function (key) { - packagejson[key] = deepSortObject(packagejson[key]) + tree.package[key] = deepSortObject(tree.package[key]) }) if (saveBundle) { - packagejson.bundledDependencies = deepSortObject(bundle) + tree.package.bundleDependencies = deepSortObject(bundle) } - var json = JSON.stringify(packagejson, null, 2) + '\n' + var json = JSON.stringify(tree.package, null, indent) + '\n' writeFileAtomic(saveTarget, json, next) })) } -var getSaveType = exports.getSaveType = function (args) { - validate('A', arguments) - var nothingToSave = !args.length +exports.getSaveType = function (tree, arg) { + if (arguments.length) validate('OO', arguments) var globalInstall = npm.config.get('global') var noSaveFlags = !npm.config.get('save') && !npm.config.get('save-dev') && !npm.config.get('save-optional') - if (nothingToSave || globalInstall || noSaveFlags) return null - - if (npm.config.get('save-optional')) return 'optionalDependencies' - else if (npm.config.get('save-dev')) return 'devDependencies' - else return 'dependencies' -} + if (globalInstall || noSaveFlags) return null -function computeVersionSpec (child) { - validate('O', arguments) - var requested = child.package._requested - if (!requested || requested.type === 'tag') { - requested = { - type: 'version', - spec: child.package.version - } - } - if (requested.type === 'version' || requested.type === 'range') { - var version = child.package.version - var rangeDescriptor = '' - if (semver.valid(version, true) && - semver.gte(version, '0.1.0', true) && - !npm.config.get('save-exact')) { - rangeDescriptor = npm.config.get('save-prefix') - } - return rangeDescriptor + version - } else if (requested.type === 'directory' || requested.type === 'local') { - var relativePath = path.relative(child.parent.path, requested.spec) - if (/^[.][.]/.test(relativePath)) { - return url.format({ - protocol: 'file', - slashes: true, - pathname: requested.spec - }) - } else { - return 'file:' + relativePath - } - } else if (requested.type === 'hosted') { - return requested.spec + if (npm.config.get('save-optional')) { + return 'optionalDependencies' + } else if (npm.config.get('save-dev')) { + return 'devDependencies' } else { - return requested.rawSpec + if (arg) { + var name = moduleName(arg) + if (tree.package.optionalDependencies[name]) { + return 'optionalDependencies' + } else if (tree.package.devDependencies[name]) { + return 'devDependencies' + } + } + return 'dependencies' } } @@ -183,15 +143,15 @@ function getThingsToSave (tree) { }).map(function (child) { return { name: moduleName(child), - spec: computeVersionSpec(child), + spec: child.saveSpec, save: child.save } }) return toSave } -function getThingsToRemove (args, tree) { - validate('AO', arguments) +function getThingsToRemove (tree) { + validate('O', arguments) if (!tree.removed) return [] var toRemove = tree.removed.map(function (child) { return { @@ -199,12 +159,5 @@ function getThingsToRemove (args, tree) { save: child.save } }) - var saveType = getSaveType(args) - args.forEach(function (arg) { - toRemove.push({ - name: arg, - save: saveType - }) - }) return toRemove } diff --git a/deps/npm/lib/install/update-package-json.js b/deps/npm/lib/install/update-package-json.js index eee530c3cd..14339d0012 100644 --- a/deps/npm/lib/install/update-package-json.js +++ b/deps/npm/lib/install/update-package-json.js @@ -20,20 +20,24 @@ module.exports = function (mod, buildpath, next) { pkg._requiredBy = mod.requiredBy .map(function (req) { - if (req.package.devDependencies[name] && !req.package.dependencies[name]) { + if ( + req.package.devDependencies && + req.package.devDependencies[name] && + !req.package.dependencies[name] + ) { return '#DEV:' + req.location } else { return req.location } }) .concat(mod.userRequired ? ['#USER'] : []) - .concat(mod.existing ? ['#EXISTING'] : []) .sort() pkg._location = mod.location pkg._phantomChildren = {} Object.keys(mod.phantomChildren).sort().forEach(function (name) { pkg._phantomChildren[name] = mod.phantomChildren[name].package.version }) + pkg._inBundle = !!mod.fromBundle // sort keys that are known safe to sort to produce more consistent output sortKeys.forEach(function (key) { @@ -42,5 +46,8 @@ module.exports = function (mod, buildpath, next) { var data = JSON.stringify(sortedObject(pkg), null, 2) + '\n' - writeFileAtomic(path.resolve(buildpath, 'package.json'), data, next) + writeFileAtomic(path.resolve(buildpath, 'package.json'), data, { + // We really don't need this guarantee, and fsyncing here is super slow. + fsync: false + }, next) } |