'use strict' var assert = require('assert') var path = require('path') var url = require('url') var semver = require('semver') var asyncMap = require('slide').asyncMap var chain = require('slide').chain var union = require('lodash.union') var iferr = require('iferr') var npa = require('npm-package-arg') var validate = require('aproba') var realizePackageSpecifier = require('realize-package-specifier') var dezalgo = require('dezalgo') var fetchPackageMetadata = require('../fetch-package-metadata.js') var andAddParentToErrors = require('./and-add-parent-to-errors.js') var addShrinkwrap = require('../fetch-package-metadata.js').addShrinkwrap var addBundled = require('../fetch-package-metadata.js').addBundled var readShrinkwrap = require('./read-shrinkwrap.js') var inflateShrinkwrap = require('./inflate-shrinkwrap.js') var inflateBundled = require('./inflate-bundled.js') var andFinishTracker = require('./and-finish-tracker.js') var npm = require('../npm.js') var flatName = require('./flatten-tree.js').flatName var createChild = require('./node.js').create var resetMetadata = require('./node.js').reset var andIgnoreErrors = require('./and-ignore-errors.js') var isInstallable = require('./validate-args.js').isInstallable var packageId = require('../utils/package-id.js') var moduleName = require('../utils/module-name.js') // The export functions in this module mutate a dependency tree, adding // items to them. function isDep (tree, child, cb) { var name = moduleName(child) var prodVer = isProdDep(tree, name) var devVer = isDevDep(tree, name) childDependencySpecifier(tree, name, prodVer, function (er, prodSpec) { if (er) return cb(child.fromShrinkwrap) var matches if (prodSpec) matches = doesChildVersionMatch(child, prodSpec, tree) if (matches) return cb(true, prodSpec) if (devVer === prodVer) return cb(child.fromShrinkwrap) childDependencySpecifier(tree, name, devVer, function (er, devSpec) { if (er) return cb(child.fromShrinkwrap) cb(doesChildVersionMatch(child, devSpec, tree) || child.fromShrinkwrap, null, devSpec) }) }) } function isDevDep (tree, name, cb) { var devDeps = tree.package.devDependencies || {} return devDeps[name] } function isProdDep (tree, name, cb) { var deps = tree.package.dependencies || {} return deps[name] } var registryTypes = { range: true, version: true } function doesChildVersionMatch (child, requested, requestor) { // we always consider deps provided by a shrinkwrap as "correct" or else // we'll subvert them if they're intentionally "invalid" if (child.parent === requestor && child.fromShrinkwrap) return true // ranges of * ALWAYS count as a match, because when downloading we allow // prereleases to match * if there are ONLY prereleases if (requested.spec === '*') return true var childReq = child.package._requested if (childReq) { if (childReq.rawSpec === requested.rawSpec) return true if (childReq.type === requested.type && childReq.spec === requested.spec) return true } if (!registryTypes[requested.type]) return requested.rawSpec === child.package._from return semver.satisfies(child.package.version, requested.spec) } exports.recalculateMetadata = function (tree, log, next) { recalculateMetadata(tree, log, {}, next) } function childDependencySpecifier (tree, name, spec, cb) { if (!tree.resolved) tree.resolved = {} if (!tree.resolved[name]) tree.resolved[name] = {} if (tree.resolved[name][spec]) { return process.nextTick(function () { cb(null, tree.resolved[name][spec]) }) } realizePackageSpecifier(name + '@' + spec, packageRelativePath(tree), function (er, req) { if (er) return cb(er) tree.resolved[name][spec] = req cb(null, req) }) } function recalculateMetadata (tree, log, seen, next) { validate('OOOF', arguments) if (seen[tree.path]) return next() seen[tree.path] = true if (tree.parent == null) resetMetadata(tree) function markDeps (spec, done) { validate('SF', arguments) var matched = spec.match(/^(@?[^@]+)@(.*)$/) childDependencySpecifier(tree, matched[1], matched[2], function (er, req) { if (er || !req.name) return done() var child = findRequirement(tree, req.name, req) if (child) { resolveWithExistingModule(child, tree, log, andIgnoreErrors(done)) } else if (tree.package.dependencies[req.name] != null) { tree.missingDeps[req.name] = req.rawSpec done() } else if (tree.package.devDependencies[req.name] != null) { tree.missingDevDeps[req.name] = req.rawSpec done() } else { done() } }) } function specs (deps) { return Object.keys(deps).map(function (depname) { return depname + '@' + deps[depname] }) } // Ensure dependencies and dev dependencies are marked as required var tomark = specs(tree.package.dependencies) if (!tree.parent && (npm.config.get('dev') || !npm.config.get('production'))) { tomark = union(tomark, specs(tree.package.devDependencies)) } // Ensure any children ONLY from a shrinkwrap are also included var childrenOnlyInShrinkwrap = tree.children.filter(function (child) { return child.fromShrinkwrap && !tree.package.dependencies[child.package.name] && !tree.package.devDependencies[child.package.name] }) var tomarkOnlyInShrinkwrap = childrenOnlyInShrinkwrap.map(function (child) { return child.package._spec }) tomark = union(tomark, tomarkOnlyInShrinkwrap) // Don't bother trying to recalc children of failed deps tree.children = tree.children.filter(function (child) { return !child.failed }) chain([ [asyncMap, tomark, markDeps], [asyncMap, tree.children, function (child, done) { recalculateMetadata(child, log, seen, done) }] ], function () { tree.userRequired = tree.package._requiredBy.some(function (req) { return req === '#USER' }) tree.existing = tree.package._requiredBy.some(function (req) { return req === '#EXISTING' }) tree.package._location = flatNameFromTree(tree) next(null, tree) }) } function addRequiredDep (tree, child, cb) { isDep(tree, child, function (childIsDep, childIsProdDep, childIsDevDep) { if (!childIsDep) return cb(false) var name = childIsProdDep ? flatNameFromTree(tree) : '#DEV:' + flatNameFromTree(tree) replaceModuleName(child.package, '_requiredBy', name) replaceModule(child, 'requiredBy', tree) replaceModule(tree, 'requires', child) cb(true) }) } exports._removeObsoleteDep = removeObsoleteDep function removeObsoleteDep (child) { if (child.removed) return child.removed = true var requires = child.requires || [] requires.forEach(function (requirement) { requirement.requiredBy = requirement.requiredBy.filter(function (reqBy) { return reqBy !== child }) if (requirement.requiredBy.length === 0) removeObsoleteDep(requirement) }) } function matchingDep (tree, name) { if (tree.package.dependencies && tree.package.dependencies[name]) return tree.package.dependencies[name] if (tree.package.devDependencies && tree.package.devDependencies[name]) return tree.package.devDependencies[name] return } function packageRelativePath (tree) { if (!tree) return '' var requested = tree.package._requested || {} var isLocal = requested.type === 'directory' || requested.type === 'local' return isLocal ? requested.spec : tree.path } function getShrinkwrap (tree, name) { return tree.package._shrinkwrap && tree.package._shrinkwrap.dependencies && tree.package._shrinkwrap.dependencies[name] } exports.getAllMetadata = function (args, tree, next) { asyncMap(args, function (spec, done) { if (tree && spec.lastIndexOf('@') <= 0) { var sw = getShrinkwrap(tree, spec) if (sw) { // FIXME: This is duplicated in inflate-shrinkwrap and should be factoed // into a shared function spec = sw.resolved ? spec + '@' + sw.resolved : (sw.from && url.parse(sw.from).protocol) ? spec + '@' + sw.from : spec + '@' + sw.version } else { var version = matchingDep(tree, spec) if (version != null) { spec += '@' + version } } } fetchPackageMetadata(spec, packageRelativePath(tree), done) }, next) } // Add a list of args to tree's top level dependencies exports.loadRequestedDeps = function (args, tree, saveToDependencies, log, next) { validate('AOOF', [args, tree, log, next]) asyncMap(args, function (pkg, done) { var depLoaded = andAddParentToErrors(tree, done) resolveWithNewModule(pkg, tree, log.newGroup('loadRequestedDeps'), iferr(depLoaded, function (child, tracker) { validate('OO', arguments) if (npm.config.get('global')) { child.isGlobal = true } var childName = moduleName(child) if (saveToDependencies) { tree.package[saveToDependencies][childName] = child.package._requested.rawSpec || child.package._requested.spec } if (saveToDependencies && saveToDependencies !== 'devDependencies') { tree.package.dependencies[childName] = child.package._requested.rawSpec || child.package._requested.spec } child.userRequired = true child.save = saveToDependencies // For things the user asked to install, that aren't a dependency (or // won't be when we're done), flag it as "depending" on the user // themselves, so we don't remove it as a dep that no longer exists addRequiredDep(tree, child, function (childIsDep) { if (!childIsDep) { replaceModuleName(child.package, '_requiredBy', '#USER') } depLoaded(null, child, tracker) }) })) }, andForEachChild(loadDeps, andFinishTracker(log, next))) } function moduleNameMatches (name) { return function (child) { return moduleName(child) === name } } function noModuleNameMatches (name) { return function (child) { return moduleName(child) !== name } } // while this implementation does not require async calling, doing so // gives this a consistent interface with loadDeps et al exports.removeDeps = function (args, tree, saveToDependencies, log, next) { validate('AOOF', [args, tree, log, next]) args.forEach(function (pkg) { var pkgName = moduleName(pkg) if (saveToDependencies) { var toRemove = tree.children.filter(moduleNameMatches(pkgName)) var pkgToRemove = toRemove[0] || createChild({package: {name: pkgName}}) replaceModule(tree, 'removed', pkgToRemove) pkgToRemove.save = saveToDependencies } tree.children = tree.children.filter(noModuleNameMatches(pkgName)) }) log.finish() next() } function andForEachChild (load, next) { validate('F', [next]) next = dezalgo(next) return function (er, children, logs) { // when children is empty, logs won't be passed in at all (asyncMap is weird) // so shortcircuit before arg validation if (!er && (!children || children.length === 0)) return next() validate('EAA', arguments) if (er) return next(er) assert(children.length === logs.length) var cmds = [] for (var ii = 0; ii < children.length; ++ii) { cmds.push([load, children[ii], logs[ii]]) } var sortedCmds = cmds.sort(function installOrder (aa, bb) { return moduleName(aa[1]).localeCompare(moduleName(bb[1])) }) chain(sortedCmds, next) } } function isDepOptional (tree, name) { if (!tree.package.optionalDependencies) return false if (tree.package.optionalDependencies[name] != null) return true return false } var failedDependency = exports.failedDependency = function (tree, name_pkg) { var name, pkg if (typeof name_pkg === 'string') { name = name_pkg } else { pkg = name_pkg name = moduleName(pkg) } tree.children = tree.children.filter(noModuleNameMatches(name)) if (isDepOptional(tree, name)) { return false } tree.failed = true if (!tree.parent) return true if (tree.userRequired) return true for (var ii = 0; ii < tree.requiredBy.length; ++ii) { var requireParent = tree.requiredBy[ii] if (failedDependency(requireParent, tree.package)) { return true } } return false } function top (tree) { if (tree.parent) return top(tree.parent) return tree } function treeWarn (tree, what, error) { var topTree = top(tree) if (!topTree.warnings) topTree.warnings = [] error.optional = flatNameFromTree(tree) + '/' + what topTree.warnings.push(error) } function andHandleOptionalErrors (log, tree, name, done) { validate('OOSF', arguments) return function (er, child, childLog) { if (!er) validate('OO', [child, childLog]) if (!er) return done(er, child, childLog) var isFatal = failedDependency(tree, name) if (er && !isFatal) { tree.children = tree.children.filter(noModuleNameMatches(name)) treeWarn(tree, name, er) return done() } else { return done(er, child, childLog) } } } // Load any missing dependencies in the given tree exports.loadDeps = loadDeps function loadDeps (tree, log, next) { validate('OOF', arguments) if (tree.loaded || (tree.parent && tree.parent.failed)) return andFinishTracker.now(log, next) if (tree.parent) tree.loaded = true if (!tree.package.dependencies) tree.package.dependencies = {} asyncMap(Object.keys(tree.package.dependencies), function (dep, done) { var version = tree.package.dependencies[dep] if (tree.package.optionalDependencies && tree.package.optionalDependencies[dep] && !npm.config.get('optional')) { return done() } addDependency(dep, version, tree, log.newGroup('loadDep:' + dep), andHandleOptionalErrors(log, tree, dep, done)) }, andForEachChild(loadDeps, andFinishTracker(log, next))) } // Load development dependencies into the given tree exports.loadDevDeps = function (tree, log, next) { validate('OOF', arguments) if (!tree.package.devDependencies) return andFinishTracker.now(log, next) asyncMap(Object.keys(tree.package.devDependencies), function (dep, done) { // things defined as both dev dependencies and regular dependencies are treated // as the former if (tree.package.dependencies[dep]) return done() var logGroup = log.newGroup('loadDevDep:' + dep) addDependency(dep, tree.package.devDependencies[dep], tree, logGroup, done) }, andForEachChild(loadDeps, andFinishTracker(log, next))) } exports.loadExtraneous = function loadExtraneous (tree, log, next) { var seen = {} function loadExtraneous (tree, log, next) { validate('OOF', arguments) if (seen[tree.path]) return next() seen[tree.path] = true asyncMap(tree.children.filter(function (child) { return !child.loaded }), function (child, done) { resolveWithExistingModule(child, tree, log, done) }, andForEachChild(loadExtraneous, andFinishTracker(log, next))) } loadExtraneous(tree, log, next) } exports.loadExtraneous.andResolveDeps = function (tree, log, next) { validate('OOF', arguments) asyncMap(tree.children.filter(function (child) { return !child.loaded }), function (child, done) { resolveWithExistingModule(child, tree, log, done) }, andForEachChild(loadDeps, andFinishTracker(log, next))) } function addDependency (name, versionSpec, tree, log, done) { validate('SSOOF', arguments) var next = andAddParentToErrors(tree, done) childDependencySpecifier(tree, name, versionSpec, iferr(done, function (req) { var child = findRequirement(tree, name, req) if (child) { resolveWithExistingModule(child, tree, log, iferr(next, function (child, log) { if (child.package._shrinkwrap === undefined) { readShrinkwrap.andInflate(child, function (er) { next(er, child, log) }) } else { next(null, child, log) } })) } else { resolveWithNewModule(req, tree, log, next) } })) } function resolveWithExistingModule (child, tree, log, next) { validate('OOOF', arguments) addRequiredDep(tree, child, function () { if (tree.parent && child.parent !== tree) updatePhantomChildren(tree.parent, child) next(null, child, log) }) } var updatePhantomChildren = exports.updatePhantomChildren = function (current, child) { validate('OO', arguments) while (current && current !== child.parent) { // FIXME: phantomChildren doesn't actually belong in the package.json if (!current.package._phantomChildren) current.package._phantomChildren = {} current.package._phantomChildren[moduleName(child)] = child.package.version current = current.parent } } function flatNameFromTree (tree) { validate('O', arguments) if (!tree.parent) return '/' var path = flatNameFromTree(tree.parent) if (path !== '/') path += '/' return flatName(path, tree) } exports._replaceModuleName = replaceModuleName function replaceModuleName (obj, key, name) { validate('OSS', arguments) obj[key] = union(obj[key] || [], [name]) } exports._replaceModule = replaceModule function replaceModule (obj, key, child) { validate('OSO', arguments) if (!obj[key]) obj[key] = [] // we replace children with a new array object instead of mutating it // because mutating it results in weird failure states. // I would very much like to know _why_ this is. =/ var children = [].concat(obj[key]) var childName = moduleName(child) for (var replaceAt = 0; replaceAt < children.length; ++replaceAt) { if (moduleName(children[replaceAt]) === childName) break } var replacing = children.splice(replaceAt, 1, child) obj[key] = children return replacing[0] } function resolveWithNewModule (pkg, tree, log, next) { validate('OOOF', arguments) if (pkg.type) { return fetchPackageMetadata(pkg, packageRelativePath(tree), log.newItem('fetchMetadata'), iferr(next, function (pkg) { resolveWithNewModule(pkg, tree, log, next) })) } if (!pkg._installable) { log.silly('resolveWithNewModule', packageId(pkg), 'checking installable status') return isInstallable(pkg, iferr(next, function () { pkg._installable = true resolveWithNewModule(pkg, tree, log, next) })) } if (!pkg._from) { pkg._from = pkg._requested.name + '@' + pkg._requested.spec } addShrinkwrap(pkg, iferr(next, function () { addBundled(pkg, iferr(next, function () { var parent = earliestInstallable(tree, tree, pkg) || tree var child = createChild({ package: pkg, parent: parent, path: path.join(parent.path, 'node_modules', pkg.name), realpath: path.resolve(parent.realpath, 'node_modules', pkg.name), children: pkg._bundled || [], isLink: tree.isLink }) var replaced = replaceModule(parent, 'children', child) if (replaced) removeObsoleteDep(replaced) addRequiredDep(tree, child, function () { pkg._location = flatNameFromTree(child) if (tree.parent && parent !== tree) updatePhantomChildren(tree.parent, child) if (pkg._bundled) { inflateBundled(child, child.children) } if (pkg._shrinkwrap && pkg._shrinkwrap.dependencies) { return inflateShrinkwrap(child, pkg._shrinkwrap.dependencies, function (er) { next(er, child, log) }) } next(null, child, log) }) })) })) } var validatePeerDeps = exports.validatePeerDeps = function (tree, onInvalid) { if (!tree.package.peerDependencies) return Object.keys(tree.package.peerDependencies).forEach(function (pkgname) { var version = tree.package.peerDependencies[pkgname] var match = findRequirement(tree.parent || tree, pkgname, npa(pkgname + '@' + version)) if (!match) onInvalid(tree, pkgname, version) }) } exports.validateAllPeerDeps = function (tree, onInvalid) { validateAllPeerDeps(tree, onInvalid, {}) } function validateAllPeerDeps (tree, onInvalid, seen) { validate('OFO', arguments) if (seen[tree.path]) return seen[tree.path] = true validatePeerDeps(tree, onInvalid) tree.children.forEach(function (child) { validateAllPeerDeps(child, onInvalid, seen) }) } // Determine if a module requirement is already met by the tree at or above // our current location in the tree. var findRequirement = exports.findRequirement = function (tree, name, requested, requestor) { validate('OSO', [tree, name, requested]) if (!requestor) requestor = tree var nameMatch = function (child) { return moduleName(child) === name && child.parent && !child.removed } var versionMatch = function (child) { return doesChildVersionMatch(child, requested, requestor) } if (nameMatch(tree)) { // this *is* the module, but it doesn't match the version, so a // new copy will have to be installed return versionMatch(tree) ? tree : null } var matches = tree.children.filter(nameMatch) if (matches.length) { matches = matches.filter(versionMatch) // the module exists as a dependent, but the version doesn't match, so // a new copy will have to be installed above here if (matches.length) return matches[0] return null } if (!tree.parent) return null return findRequirement(tree.parent, name, requested, requestor) } // Find the highest level in the tree that we can install this module in. // If the module isn't installed above us yet, that'd be the very top. // If it is, then it's the level below where its installed. var earliestInstallable = exports.earliestInstallable = function (requiredBy, tree, pkg) { validate('OOO', arguments) function undeletedModuleMatches (child) { return !child.removed && moduleName(child) === pkg.name } if (tree.children.some(undeletedModuleMatches)) return null // If any of the children of this tree have conflicting // binaries then we need to decline to install this package here. var binaryMatches = typeof pkg.bin === 'object' && tree.children.some(function (child) { if (child.removed) return false if (typeof child.package.bin !== 'object') return false return Object.keys(child.package.bin).some(function (bin) { return pkg.bin[bin] }) }) if (binaryMatches) return null // if this tree location requested the same module then we KNOW it // isn't compatible because if it were findRequirement would have // found that version. var deps = tree.package.dependencies || {} if (!tree.removed && requiredBy !== tree && deps[pkg.name]) { return null } // FIXME: phantomChildren doesn't actually belong in the package.json if (tree.package._phantomChildren && tree.package._phantomChildren[pkg.name]) return null if (!tree.parent) return tree if (tree.isGlobal) return tree if (npm.config.get('global-style') && !tree.parent.parent) return tree if (npm.config.get('legacy-bundling')) return tree return (earliestInstallable(requiredBy, tree.parent, pkg) || tree) }