diff options
Diffstat (limited to 'deps/npm/node_modules/pacote/lib/fetchers/registry')
8 files changed, 128 insertions, 280 deletions
diff --git a/deps/npm/node_modules/pacote/lib/fetchers/registry/check-warning-header.js b/deps/npm/node_modules/pacote/lib/fetchers/registry/check-warning-header.js deleted file mode 100644 index b17a233d43..0000000000 --- a/deps/npm/node_modules/pacote/lib/fetchers/registry/check-warning-header.js +++ /dev/null @@ -1,39 +0,0 @@ -'use strict' - -const LRU = require('lru-cache') - -const WARNING_REGEXP = /^\s*(\d{3})\s+(\S+)\s+"(.*)"\s+"([^"]+)"/ -const BAD_HOSTS = new LRU({ max: 50 }) - -module.exports = checkWarnings -function checkWarnings (res, registry, opts) { - if (res.headers.has('warning') && !BAD_HOSTS.has(registry)) { - const warnings = {} - res.headers.raw()['warning'].forEach(w => { - const match = w.match(WARNING_REGEXP) - if (match) { - warnings[match[1]] = { - code: match[1], - host: match[2], - message: match[3], - date: new Date(match[4]) - } - } - }) - BAD_HOSTS.set(registry, true) - if (warnings['199']) { - if (warnings['199'].message.match(/ENOTFOUND/)) { - opts.log.warn('registry', `Using stale data from ${registry} because the host is inaccessible -- are you offline?`) - } else { - opts.log.warn('registry', `Unexpected warning for ${registry}: ${warnings['199'].message}`) - } - } - if (warnings['111']) { - // 111 Revalidation failed -- we're using stale data - opts.log.warn( - 'registry', - `Using stale package data from ${registry} due to a request error during revalidation.` - ) - } - } -} diff --git a/deps/npm/node_modules/pacote/lib/fetchers/registry/fetch.js b/deps/npm/node_modules/pacote/lib/fetchers/registry/fetch.js deleted file mode 100644 index 3a2a4a5a77..0000000000 --- a/deps/npm/node_modules/pacote/lib/fetchers/registry/fetch.js +++ /dev/null @@ -1,109 +0,0 @@ -'use strict' - -const BB = require('bluebird') -const Buffer = require('safe-buffer').Buffer - -const checkWarnings = require('./check-warning-header') -const fetch = require('make-fetch-happen') -const registryKey = require('./registry-key') -const url = require('url') - -module.exports = regFetch -function regFetch (uri, registry, opts) { - const startTime = Date.now() - return fetch(uri, { - agent: opts.agent, - algorithms: opts.algorithms, - cache: getCacheMode(opts), - cacheManager: opts.cache, - ca: opts.ca, - cert: opts.cert, - headers: getHeaders(uri, registry, opts), - integrity: opts.integrity, - key: opts.key, - localAddress: opts.localAddress, - maxSockets: opts.maxSockets, - memoize: opts.memoize, - noProxy: opts.noProxy, - Promise: BB, - proxy: opts.proxy, - referer: opts.refer, - retry: opts.retry, - strictSSL: !!opts.strictSSL, - timeout: opts.timeout, - uid: opts.uid, - gid: opts.gid - }).then(res => { - if (res.headers.has('npm-notice') && !res.headers.has('x-local-cache')) { - opts.log.warn('notice', res.headers.get('npm-notice')) - } - checkWarnings(res, registry, opts) - if (res.status >= 400) { - const err = new Error(`${res.status} ${res.statusText}: ${ - opts.spec ? opts.spec : uri - }`) - err.code = `E${res.status}` - err.uri = uri - err.response = res - err.spec = opts.spec - logRequest(uri, res, startTime, opts) - throw err - } else { - res.body.on('end', () => logRequest(uri, res, startTime, opts)) - return res - } - }) -} - -function logRequest (uri, res, startTime, opts) { - const elapsedTime = Date.now() - startTime - const attempt = res.headers.get('x-fetch-attempts') - const attemptStr = attempt && attempt > 1 ? ` attempt #${attempt}` : '' - const cacheStr = res.headers.get('x-local-cache') ? ' (from cache)' : '' - opts.log.http( - 'fetch', - `GET ${res.status} ${uri} ${elapsedTime}ms${attemptStr}${cacheStr}` - ) -} - -function getCacheMode (opts) { - return opts.offline - ? 'only-if-cached' - : opts.preferOffline - ? 'force-cache' - : opts.preferOnline - ? 'no-cache' - : 'default' -} - -function getHeaders (uri, registry, opts) { - const headers = Object.assign({ - 'npm-in-ci': opts.isFromCI, - 'npm-scope': opts.projectScope, - 'npm-session': opts.npmSession, - 'user-agent': opts.userAgent, - 'referer': opts.refer - }, opts.headers) - // check for auth settings specific to this registry - let auth = ( - opts.auth && - opts.auth[registryKey(registry)] - ) || opts.auth - // If a tarball is hosted on a different place than the manifest, only send - // credentials on `alwaysAuth` - const shouldAuth = auth && ( - auth.alwaysAuth || - url.parse(uri).host === url.parse(registry).host - ) - if (shouldAuth && auth.token) { - headers.authorization = `Bearer ${auth.token}` - } else if (shouldAuth && auth.username && auth.password) { - const encoded = Buffer.from( - `${auth.username}:${auth.password}`, 'utf8' - ).toString('base64') - headers.authorization = `Basic ${encoded}` - } else if (shouldAuth && auth._auth) { - headers.authorization = `Basic ${auth._auth}` - } - return headers -} diff --git a/deps/npm/node_modules/pacote/lib/fetchers/registry/index.js b/deps/npm/node_modules/pacote/lib/fetchers/registry/index.js index 78bdc23426..2cca7040bd 100644 --- a/deps/npm/node_modules/pacote/lib/fetchers/registry/index.js +++ b/deps/npm/node_modules/pacote/lib/fetchers/registry/index.js @@ -3,11 +3,16 @@ const cacache = require('cacache') const Fetcher = require('../../fetch') const regManifest = require('./manifest') +const regPackument = require('./packument') const regTarball = require('./tarball') const fetchRegistry = module.exports = Object.create(null) Fetcher.impl(fetchRegistry, { + packument (spec, opts) { + return regPackument(spec, opts) + }, + manifest (spec, opts) { return regManifest(spec, opts) }, @@ -22,6 +27,6 @@ Fetcher.impl(fetchRegistry, { clearMemoized () { cacache.clearMemoized() - regManifest.clearMemoized() + regPackument.clearMemoized() } }) diff --git a/deps/npm/node_modules/pacote/lib/fetchers/registry/manifest.js b/deps/npm/node_modules/pacote/lib/fetchers/registry/manifest.js index 4e5a8010e2..d29ec71c33 100644 --- a/deps/npm/node_modules/pacote/lib/fetchers/registry/manifest.js +++ b/deps/npm/node_modules/pacote/lib/fetchers/registry/manifest.js @@ -1,56 +1,48 @@ 'use strict' -const BB = require('bluebird') - -const fetch = require('./fetch') -const LRU = require('lru-cache') +const fetch = require('npm-registry-fetch') +const fetchPackument = require('./packument') const optCheck = require('../../util/opt-check') const pickManifest = require('npm-pick-manifest') -const pickRegistry = require('./pick-registry') const ssri = require('ssri') -const url = require('url') - -// Corgis are cute. 🐕🐶 -const CORGI_DOC = 'application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*' -const JSON_DOC = 'application/json' module.exports = manifest function manifest (spec, opts) { opts = optCheck(opts) - const registry = pickRegistry(spec, opts) - const uri = metadataUrl(registry, spec.escapedName) - - return getManifest(uri, registry, spec, opts).then(manifest => { - return annotateManifest(uri, registry, manifest) + return getManifest(spec, opts).then(manifest => { + return annotateManifest(spec, manifest, opts) }) } -function metadataUrl (registry, name) { - const normalized = registry.slice(-1) !== '/' - ? registry + '/' - : registry - return url.resolve(normalized, name) -} - -function getManifest (uri, registry, spec, opts) { - return fetchPackument(uri, spec, registry, opts).then(packument => { +function getManifest (spec, opts) { + opts = opts.concat({ + fullMetadata: opts.enjoyBy ? true : opts.fullMetadata + }) + return fetchPackument(spec, opts).then(packument => { try { return pickManifest(packument, spec.fetchSpec, { defaultTag: opts.defaultTag, + enjoyBy: opts.enjoyBy, includeDeprecated: opts.includeDeprecated }) } catch (err) { if (err.code === 'ETARGET' && packument._cached && !opts.offline) { opts.log.silly( 'registry:manifest', - `no matching version for ${spec.name}@${spec.fetchSpec} in the cache. Forcing revalidation` + `no matching version for ${spec.name}@${spec.fetchSpec} in the cache. Forcing revalidation.` ) - opts.preferOffline = false - opts.preferOnline = true - return fetchPackument(uri, spec, registry, opts).then(packument => { + opts = opts.concat({ + preferOffline: false, + preferOnline: true + }) + return fetchPackument(spec, opts.concat({ + // Fetch full metadata in case ETARGET was due to corgi delay + fullMetadata: true + })).then(packument => { return pickManifest(packument, spec.fetchSpec, { - defaultTag: opts.defaultTag + defaultTag: opts.defaultTag, + enjoyBy: opts.enjoyBy }) }) } else { @@ -60,69 +52,7 @@ function getManifest (uri, registry, spec, opts) { }) } -// TODO - make this an opt -const MEMO = new LRU({ - length: m => m._contentLength, - max: 200 * 1024 * 1024, // 200MB - maxAge: 30 * 1000 // 30s -}) - -module.exports.clearMemoized = clearMemoized -function clearMemoized () { - MEMO.reset() -} - -function fetchPackument (uri, spec, registry, opts) { - const mem = pickMem(opts) - if (mem && !opts.preferOnline && mem.has(uri)) { - return BB.resolve(mem.get(uri)) - } - - return fetch(uri, registry, Object.assign({ - headers: { - 'pacote-req-type': 'packument', - 'pacote-pkg-id': `registry:${manifest.name}`, - accept: opts.fullMetadata ? JSON_DOC : CORGI_DOC - }, - spec - }, opts, { - // Force integrity to null: we never check integrity hashes for manifests - integrity: null - })).then(res => res.json().then(packument => { - packument._cached = decodeURIComponent(res.headers.has('x-local-cache')) - packument._contentLength = +res.headers.get('content-length') - // NOTE - we need to call pickMem again because proxy - // objects get reused! - const mem = pickMem(opts) - if (mem) { - mem.set(uri, packument) - } - return packument - })) -} - -class ObjProxy { - get (key) { return this.obj[key] } - set (key, val) { this.obj[key] = val } -} - -// This object is used synchronously and immediately, so -// we can safely reuse it instead of consing up new ones -const PROX = new ObjProxy() -function pickMem (opts) { - if (!opts || !opts.memoize) { - return MEMO - } else if (opts.memoize.get && opts.memoize.set) { - return opts.memoize - } else if (typeof opts.memoize === 'object') { - PROX.obj = opts.memoize - return PROX - } else { - return null - } -} - -function annotateManifest (uri, registry, manifest) { +function annotateManifest (spec, manifest, opts) { const shasum = manifest.dist && manifest.dist.shasum manifest._integrity = manifest.dist && manifest.dist.integrity manifest._shasum = shasum @@ -134,6 +64,9 @@ function annotateManifest (uri, registry, manifest) { manifest.dist && manifest.dist.tarball ) if (!manifest._resolved) { + const registry = fetch.pickRegistry(spec, opts) + const uri = registry.replace(/\/?$/, '/') + spec.escapedName + const err = new Error( `Manifest for ${manifest.name}@${manifest.version} from ${uri} is missing a tarball url (pkg.dist.tarball). Guessing a default.` ) diff --git a/deps/npm/node_modules/pacote/lib/fetchers/registry/packument.js b/deps/npm/node_modules/pacote/lib/fetchers/registry/packument.js new file mode 100644 index 0000000000..f5286c8037 --- /dev/null +++ b/deps/npm/node_modules/pacote/lib/fetchers/registry/packument.js @@ -0,0 +1,92 @@ +'use strict' + +const BB = require('bluebird') + +const fetch = require('npm-registry-fetch') +const LRU = require('lru-cache') +const optCheck = require('../../util/opt-check') + +// Corgis are cute. 🐕🐶 +const CORGI_DOC = 'application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*' +const JSON_DOC = 'application/json' + +module.exports = packument +function packument (spec, opts) { + opts = optCheck(opts) + + const registry = fetch.pickRegistry(spec, opts) + const uri = registry.replace(/\/?$/, '/') + spec.escapedName + + return fetchPackument(uri, registry, spec, opts) +} + +const MEMO = new LRU({ + length: m => m._contentLength, + max: 200 * 1024 * 1024, // 200MB + maxAge: 30 * 1000 // 30s +}) + +module.exports.clearMemoized = clearMemoized +function clearMemoized () { + MEMO.reset() +} + +function fetchPackument (uri, registry, spec, opts) { + const mem = pickMem(opts) + const accept = opts.fullMetadata ? JSON_DOC : CORGI_DOC + const memoKey = `${uri}~(${accept})` + if (mem && !opts.preferOnline && mem.has(memoKey)) { + return BB.resolve(mem.get(memoKey)) + } + + return fetch(uri, opts.concat({ + headers: { + 'pacote-req-type': 'packument', + 'pacote-pkg-id': `registry:${spec.name}`, + accept + }, + spec + }, opts, { + // Force integrity to null: we never check integrity hashes for manifests + integrity: null + })).then(res => res.json().then(packument => { + packument._cached = res.headers.has('x-local-cache') + packument._contentLength = +res.headers.get('content-length') + // NOTE - we need to call pickMem again because proxy + // objects get reused! + const mem = pickMem(opts) + if (mem) { + mem.set(memoKey, packument) + } + return packument + })).catch(err => { + if (err.code === 'E404' && !opts.fullMetadata) { + return fetchPackument(uri, registry, spec, opts.concat({ + fullMetadata: true + })) + } else { + throw err + } + }) +} + +class ObjProxy { + get (key) { return this.obj[key] } + set (key, val) { this.obj[key] = val } +} + +// This object is used synchronously and immediately, so +// we can safely reuse it instead of consing up new ones +const PROX = new ObjProxy() +function pickMem (opts) { + if (!opts || !opts.memoize) { + return MEMO + } else if (opts.memoize.get && opts.memoize.set) { + return opts.memoize + } else if (typeof opts.memoize === 'object') { + PROX.obj = opts.memoize + return PROX + } else { + return null + } +} diff --git a/deps/npm/node_modules/pacote/lib/fetchers/registry/pick-registry.js b/deps/npm/node_modules/pacote/lib/fetchers/registry/pick-registry.js deleted file mode 100644 index f326950da4..0000000000 --- a/deps/npm/node_modules/pacote/lib/fetchers/registry/pick-registry.js +++ /dev/null @@ -1,17 +0,0 @@ -'use strict' - -module.exports = pickRegistry -function pickRegistry (spec, opts) { - let registry = spec.scope && opts.scopeTargets[spec.scope] - - if (!registry && opts.scope) { - const prefix = opts.scope[0] === '@' ? '' : '@' - registry = opts.scopeTargets[prefix + opts.scope] - } - - if (!registry) { - registry = opts.registry - } - - return registry -} diff --git a/deps/npm/node_modules/pacote/lib/fetchers/registry/registry-key.js b/deps/npm/node_modules/pacote/lib/fetchers/registry/registry-key.js deleted file mode 100644 index f53e9a9b48..0000000000 --- a/deps/npm/node_modules/pacote/lib/fetchers/registry/registry-key.js +++ /dev/null @@ -1,16 +0,0 @@ -'use strict' - -const url = require('url') - -// Called a nerf dart in the main codebase. Used as a "safe" -// key when fetching registry info from config. -module.exports = registryKey -function registryKey (registry) { - const parsed = url.parse(registry) - const formatted = url.format({ - host: parsed.host, - pathname: parsed.pathname, - slashes: parsed.slashes - }) - return url.resolve(formatted, '.') -} diff --git a/deps/npm/node_modules/pacote/lib/fetchers/registry/tarball.js b/deps/npm/node_modules/pacote/lib/fetchers/registry/tarball.js index 7239981279..134153280e 100644 --- a/deps/npm/node_modules/pacote/lib/fetchers/registry/tarball.js +++ b/deps/npm/node_modules/pacote/lib/fetchers/registry/tarball.js @@ -2,18 +2,17 @@ const BB = require('bluebird') -const fetch = require('./fetch') +const fetch = require('npm-registry-fetch') const manifest = require('./manifest') const optCheck = require('../../util/opt-check') const PassThrough = require('stream').PassThrough -const pickRegistry = require('./pick-registry') const ssri = require('ssri') const url = require('url') module.exports = tarball function tarball (spec, opts) { opts = optCheck(opts) - const registry = pickRegistry(spec, opts) + const registry = fetch.pickRegistry(spec, opts) const stream = new PassThrough() let mani if ( @@ -49,11 +48,11 @@ function tarball (spec, opts) { module.exports.fromManifest = fromManifest function fromManifest (manifest, spec, opts) { opts = optCheck(opts) - opts.scope = spec.scope || opts.scope + if (spec.scope) { opts = opts.concat({ scope: spec.scope }) } const stream = new PassThrough() - const registry = pickRegistry(spec, opts) + const registry = fetch.pickRegistry(spec, opts) const uri = getTarballUrl(spec, registry, manifest, opts) - fetch(uri, registry, Object.assign({ + fetch(uri, opts.concat({ headers: { 'pacote-req-type': 'tarball', 'pacote-pkg-id': `registry:${manifest.name}@${uri}` |