index.js 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626
  1. var fs = require('fs');
  2. var path = require('path');
  3. var relativePath = require('cached-path-relative')
  4. var browserResolve = require('browser-resolve');
  5. var nodeResolve = require('resolve');
  6. var detective = require('detective');
  7. var through = require('through2');
  8. var concat = require('concat-stream');
  9. var combine = require('stream-combiner2');
  10. var duplexer = require('duplexer2');
  11. var xtend = require('xtend');
  12. var defined = require('defined');
  13. var inherits = require('inherits');
  14. var Transform = require('readable-stream').Transform;
  15. module.exports = Deps;
  16. inherits(Deps, Transform);
  17. function leftPad(str, width) {
  18. str = str.toString();
  19. while (str.length < width) str = '0' + str;
  20. return str;
  21. }
  22. function dirname(file) {
  23. if (file !== undefined) {
  24. return path.dirname(file);
  25. }
  26. }
  27. function parents(cwd, opts) {
  28. if (cwd === undefined) cwd = process.cwd();
  29. if (!opts) opts = {};
  30. var platform = opts.platform || process.platform;
  31. var isWindows = /^win/.test(platform);
  32. var p = isWindows ? path.win32 : path;
  33. var normalize = !isWindows ? p.normalize :
  34. p.normalize('c:') === 'c:.' ? fixNormalize(p.normalize) :
  35. p.normalize;
  36. var sep = isWindows ? /[\\\/]/ : '/';
  37. var init = isWindows ? '' : '/';
  38. var join = function (x, y) {
  39. var ps = [ x, y ].filter(function (p) {
  40. return p && typeof p === 'string'
  41. });
  42. return normalize(ps.join(isWindows ? '\\' : '/'));
  43. };
  44. var res = normalize(cwd)
  45. .split(sep)
  46. .reduce(function (acc,dir,ix) {
  47. return acc.concat(join(acc[ix], dir))
  48. }, [init])
  49. .slice(1)
  50. .reverse()
  51. ;
  52. if (res[0] === res[1]) return [ res[0] ];
  53. if (isWindows && /^\\/.test(cwd)) {
  54. return res.slice(0,-1).map(function (d) {
  55. var ch = d.charAt(0)
  56. return ch === '\\' ? d :
  57. ch === '.' ? '\\' + d.slice(1) :
  58. '\\' + d
  59. });
  60. }
  61. return res;
  62. }
  63. function fixNormalize(fn) {
  64. return function(p) {
  65. return fn(p).replace(/:\.$/, ':')
  66. }
  67. }
  68. function Deps (opts) {
  69. var self = this;
  70. if (!(this instanceof Deps)) return new Deps(opts);
  71. Transform.call(this, { objectMode: true });
  72. if (!opts) opts = {};
  73. this.basedir = opts.basedir || process.cwd();
  74. this.cache = opts.cache;
  75. this.fileCache = opts.fileCache;
  76. this.pkgCache = opts.packageCache || {};
  77. this.pkgFileCache = {};
  78. this.pkgFileCachePending = {};
  79. this._emittedPkg = {};
  80. this.visited = {};
  81. this.walking = {};
  82. this.entries = [];
  83. this._input = [];
  84. this.paths = opts.paths || process.env.NODE_PATH || '';
  85. if (typeof this.paths === 'string') {
  86. var delimiter = path.delimiter || (process.platform === 'win32' ? ';' : ':');
  87. this.paths = this.paths.split(delimiter);
  88. }
  89. this.paths = this.paths
  90. .filter(Boolean)
  91. .map(function (p) {
  92. return path.resolve(self.basedir, p);
  93. });
  94. this.transforms = [].concat(opts.transform).filter(Boolean);
  95. this.globalTransforms = [].concat(opts.globalTransform).filter(Boolean);
  96. this.resolver = opts.resolve || browserResolve;
  97. this.options = xtend(opts);
  98. if (!this.options.modules) this.options.modules = {};
  99. // If the caller passes options.expose, store resolved pathnames for exposed
  100. // modules in it. If not, set it anyway so it's defined later.
  101. if (!this.options.expose) this.options.expose = {};
  102. this.pending = 0;
  103. this.inputPending = 0;
  104. var topfile = path.join(this.basedir, '__fake.js');
  105. this.top = {
  106. id: topfile,
  107. filename: topfile,
  108. paths: this.paths,
  109. basedir: this.basedir
  110. };
  111. }
  112. Deps.prototype._isTopLevel = function (file) {
  113. var isTopLevel = this.entries.some(function (main) {
  114. var m = path.relative(dirname(main), file);
  115. return m.split(/[\\\/]/).indexOf('node_modules') < 0;
  116. });
  117. if (!isTopLevel) {
  118. var m = relativePath(this.basedir, file);
  119. isTopLevel = m.split(/[\\\/]/).indexOf('node_modules') < 0;
  120. }
  121. return isTopLevel;
  122. };
  123. Deps.prototype._transform = function (row, enc, next) {
  124. var self = this;
  125. if (typeof row === 'string') {
  126. row = { file: row };
  127. }
  128. if (row.transform && row.global) {
  129. this.globalTransforms.push([ row.transform, row.options ]);
  130. return next();
  131. }
  132. else if (row.transform) {
  133. this.transforms.push([ row.transform, row.options ]);
  134. return next();
  135. }
  136. self.pending ++;
  137. var basedir = defined(row.basedir, self.basedir);
  138. if (row.entry !== false) {
  139. self.entries.push(path.resolve(basedir, row.file || row.id));
  140. }
  141. self.lookupPackage(row.file, function (err, pkg) {
  142. if (err && self.options.ignoreMissing) {
  143. self.emit('missing', row.file, self.top);
  144. self.pending --;
  145. return next();
  146. }
  147. if (err) return self.emit('error', err)
  148. self.pending --;
  149. self._input.push({ row: row, pkg: pkg });
  150. next();
  151. });
  152. };
  153. Deps.prototype._flush = function () {
  154. var self = this;
  155. var files = {};
  156. self._input.forEach(function (r) {
  157. var w = r.row, f = files[w.file || w.id];
  158. if (f) {
  159. f.row.entry = f.row.entry || w.entry;
  160. var ex = f.row.expose || w.expose;
  161. f.row.expose = ex;
  162. if (ex && f.row.file === f.row.id && w.file !== w.id) {
  163. f.row.id = w.id;
  164. }
  165. }
  166. else files[w.file || w.id] = r;
  167. });
  168. Object.keys(files).forEach(function (key) {
  169. var r = files[key];
  170. var pkg = r.pkg || {};
  171. var dir = r.row.file ? path.dirname(r.row.file) : self.basedir;
  172. if (!pkg.__dirname) pkg.__dirname = dir;
  173. self.walk(r.row, xtend(self.top, {
  174. filename: path.join(dir || '', '_fake.js')
  175. }));
  176. });
  177. if (this.pending === 0) this.push(null);
  178. this._ended = true;
  179. };
  180. Deps.prototype.resolve = function (id, parent, cb) {
  181. var self = this;
  182. var opts = self.options;
  183. if (xhas(self.cache, parent.id, 'deps', id)
  184. && self.cache[parent.id].deps[id]) {
  185. var file = self.cache[parent.id].deps[id];
  186. var pkg = self.pkgCache[file];
  187. if (pkg) return cb(null, file, pkg);
  188. return self.lookupPackage(file, function (err, pkg) {
  189. cb(null, file, pkg);
  190. });
  191. }
  192. parent.packageFilter = function (p, x) {
  193. var pkgdir = dirname(x);
  194. if (opts.packageFilter) p = opts.packageFilter(p, x);
  195. p.__dirname = pkgdir;
  196. return p;
  197. };
  198. if (opts.extensions) parent.extensions = opts.extensions;
  199. if (opts.modules) parent.modules = opts.modules;
  200. self.resolver(id, parent, function onresolve (err, file, pkg, fakePath) {
  201. if (err) return cb(err);
  202. if (!file) return cb(new Error(
  203. 'module not found: "' + id + '" from file '
  204. + parent.filename
  205. ));
  206. if (!pkg || !pkg.__dirname) {
  207. self.lookupPackage(file, function (err, p) {
  208. if (err) return cb(err);
  209. if (!p) p = {};
  210. if (!p.__dirname) p.__dirname = dirname(file);
  211. self.pkgCache[file] = p;
  212. onresolve(err, file, opts.packageFilter
  213. ? opts.packageFilter(p, p.__dirname) : p,
  214. fakePath
  215. );
  216. });
  217. }
  218. else cb(err, file, pkg, fakePath);
  219. });
  220. };
  221. Deps.prototype.readFile = function (file, id, pkg) {
  222. var self = this;
  223. if (xhas(this.fileCache, file)) {
  224. var tr = through();
  225. tr.push(this.fileCache[file]);
  226. tr.push(null);
  227. return tr;
  228. }
  229. var rs = fs.createReadStream(file, {
  230. encoding: 'utf8'
  231. });
  232. rs.on('error', function (err) { self.emit('error', err) });
  233. this.emit('file', file, id);
  234. return rs;
  235. };
  236. Deps.prototype.getTransforms = function (file, pkg, opts) {
  237. if (!opts) opts = {};
  238. var self = this;
  239. var isTopLevel;
  240. if (opts.builtin || opts.inNodeModules) isTopLevel = false;
  241. else isTopLevel = this._isTopLevel(file);
  242. var transforms = [].concat(isTopLevel ? this.transforms : [])
  243. .concat(getTransforms(pkg, {
  244. globalTransform: this.globalTransforms,
  245. transformKey: this.options.transformKey
  246. }))
  247. ;
  248. if (transforms.length === 0) return through();
  249. var pending = transforms.length;
  250. var streams = [];
  251. var input = through();
  252. var output = through();
  253. var dup = duplexer(input, output);
  254. for (var i = 0; i < transforms.length; i++) (function (i) {
  255. makeTransform(transforms[i], function (err, trs) {
  256. if (err) return self.emit('error', err)
  257. streams[i] = trs;
  258. if (-- pending === 0) done();
  259. });
  260. })(i);
  261. return dup;
  262. function done () {
  263. var middle = combine.apply(null, streams);
  264. middle.on('error', function (err) {
  265. err.message += ' while parsing file: ' + file;
  266. if (!err.filename) err.filename = file;
  267. self.emit('error', err);
  268. });
  269. input.pipe(middle).pipe(output);
  270. }
  271. function makeTransform (tr, cb) {
  272. var trOpts = {};
  273. if (Array.isArray(tr)) {
  274. trOpts = tr[1] || {};
  275. tr = tr[0];
  276. }
  277. if (typeof tr === 'function') {
  278. var t = tr(file, trOpts);
  279. self.emit('transform', t, file);
  280. nextTick(cb, null, wrapTransform(t));
  281. }
  282. else {
  283. loadTransform(tr, trOpts, function (err, trs) {
  284. if (err) return cb(err);
  285. cb(null, wrapTransform(trs));
  286. });
  287. }
  288. }
  289. function loadTransform (id, trOpts, cb) {
  290. var params = { basedir: dirname(file) };
  291. nodeResolve(id, params, function nr (err, res, again) {
  292. if (err && again) return cb && cb(err);
  293. if (err) {
  294. params.basedir = pkg.__dirname;
  295. return nodeResolve(id, params, function (e, r) {
  296. nr(e, r, true)
  297. });
  298. }
  299. if (!res) return cb(new Error(
  300. 'cannot find transform module ' + tr
  301. + ' while transforming ' + file
  302. ));
  303. var r = require(res);
  304. if (typeof r !== 'function') {
  305. return cb(new Error(
  306. 'Unexpected ' + typeof r + ' exported by the '
  307. + JSON.stringify(res) + ' package. '
  308. + 'Expected a transform function.'
  309. ));
  310. }
  311. var trs = r(file, trOpts);
  312. self.emit('transform', trs, file);
  313. cb(null, trs);
  314. });
  315. }
  316. };
  317. Deps.prototype.walk = function (id, parent, cb) {
  318. var self = this;
  319. var opts = self.options;
  320. var sortKey = parent.sortKey || '';
  321. this.pending ++;
  322. var rec = {};
  323. var input;
  324. if (typeof id === 'object') {
  325. rec = xtend(id);
  326. if (rec.entry === false) delete rec.entry;
  327. id = rec.file || rec.id;
  328. input = true;
  329. this.inputPending ++;
  330. }
  331. self.resolve(id, parent, function (err, file, pkg, fakePath) {
  332. // this is checked early because parent.modules is also modified
  333. // by this function.
  334. var builtin = has(parent.modules, id);
  335. if (rec.expose) {
  336. // Set options.expose to make the resolved pathname available to the
  337. // caller. They may or may not have requested it, but it's harmless
  338. // to set this if they didn't.
  339. self.options.expose[rec.expose] =
  340. self.options.modules[rec.expose] = file;
  341. }
  342. if (pkg && !self._emittedPkg[pkg.__dirname]) {
  343. self._emittedPkg[pkg.__dirname] = true;
  344. self.emit('package', pkg);
  345. }
  346. if (opts.postFilter && !opts.postFilter(id, file, pkg)) {
  347. if (--self.pending === 0) self.push(null);
  348. if (input) --self.inputPending;
  349. return cb && cb(null, undefined);
  350. }
  351. if (err && rec.source) {
  352. file = rec.file;
  353. var ts = self.getTransforms(file, pkg);
  354. ts.pipe(concat(function (body) {
  355. rec.source = body.toString('utf8');
  356. fromSource(file, rec.source, pkg, undefined, sortKey);
  357. }));
  358. return ts.end(rec.source);
  359. }
  360. if (err && self.options.ignoreMissing) {
  361. if (--self.pending === 0) self.push(null);
  362. if (input) --self.inputPending;
  363. self.emit('missing', id, parent);
  364. return cb && cb(null, undefined);
  365. }
  366. if (err) return self.emit('error', err);
  367. if (self.visited[file]) {
  368. if (-- self.pending === 0) self.push(null);
  369. if (input) --self.inputPending;
  370. return cb && cb(null, file);
  371. }
  372. self.visited[file] = true;
  373. if (rec.source) {
  374. var ts = self.getTransforms(file, pkg);
  375. ts.pipe(concat(function (body) {
  376. rec.source = body.toString('utf8');
  377. fromSource(file, rec.source, pkg, undefined, sortKey);
  378. }));
  379. return ts.end(rec.source);
  380. }
  381. var c = self.cache && self.cache[file];
  382. if (c) return fromDeps(file, c.source, c.package, fakePath, Object.keys(c.deps), sortKey);
  383. self.readFile(file, id, pkg)
  384. .pipe(self.getTransforms(fakePath || file, pkg, {
  385. builtin: builtin,
  386. inNodeModules: parent.inNodeModules
  387. }))
  388. .pipe(concat(function (body) {
  389. fromSource(file, body.toString('utf8'), pkg, fakePath, sortKey);
  390. }))
  391. ;
  392. });
  393. function fromSource (file, src, pkg, fakePath, sortKey) {
  394. var deps = rec.noparse ? [] : self.parseDeps(file, src);
  395. if (deps) fromDeps(file, src, pkg, fakePath, deps, sortKey);
  396. }
  397. function fromDeps (file, src, pkg, fakePath, deps, sortKey) {
  398. var p = deps.length;
  399. var resolved = {};
  400. if (input) --self.inputPending;
  401. (function resolve () {
  402. if (self.inputPending > 0) return setTimeout(resolve);
  403. deps.forEach(function (id, i) {
  404. if (opts.filter && !opts.filter(id)) {
  405. resolved[id] = false;
  406. if (--p === 0) done();
  407. return;
  408. }
  409. var isTopLevel = self._isTopLevel(fakePath || file);
  410. var current = {
  411. id: file,
  412. filename: file,
  413. paths: self.paths,
  414. package: pkg,
  415. inNodeModules: parent.inNodeModules || !isTopLevel,
  416. sortKey: sortKey + '!' + file + ':' + leftPad(i, 8)
  417. };
  418. self.walk(id, current, function (err, r) {
  419. resolved[id] = r;
  420. if (--p === 0) done();
  421. });
  422. });
  423. if (deps.length === 0) done();
  424. })();
  425. function done () {
  426. if (!rec.id) rec.id = file;
  427. if (!rec.source) rec.source = src;
  428. if (!rec.deps) rec.deps = resolved;
  429. if (!rec.file) rec.file = file;
  430. rec.sortKey = sortKey + '!' + file;
  431. if (self.entries.indexOf(file) >= 0) {
  432. rec.entry = true;
  433. }
  434. self.push(rec);
  435. if (cb) cb(null, file);
  436. if (-- self.pending === 0) self.push(null);
  437. }
  438. }
  439. };
  440. Deps.prototype.parseDeps = function (file, src, cb) {
  441. if (this.options.noParse === true) return [];
  442. if (/\.json$/.test(file)) return [];
  443. if (Array.isArray(this.options.noParse)
  444. && this.options.noParse.indexOf(file) >= 0) {
  445. return [];
  446. }
  447. try { var deps = detective(src) }
  448. catch (ex) {
  449. var message = ex && ex.message ? ex.message : ex;
  450. this.emit('error', new Error(
  451. 'Parsing file ' + file + ': ' + message
  452. ));
  453. return;
  454. }
  455. return deps;
  456. };
  457. Deps.prototype.lookupPackage = function (file, cb) {
  458. var self = this;
  459. var cached = this.pkgCache[file];
  460. if (cached) return nextTick(cb, null, cached);
  461. if (cached === false) return nextTick(cb, null, undefined);
  462. var dirs = parents(path.dirname(file || ''));
  463. (function next () {
  464. if (dirs.length === 0) {
  465. self.pkgCache[file] = false;
  466. return cb(null, undefined);
  467. }
  468. var dir = dirs.shift();
  469. if (dir.split(/[\\\/]/).slice(-1)[0] === 'node_modules') {
  470. return cb(null, undefined);
  471. }
  472. var pkgfile = path.join(dir, 'package.json');
  473. var cached = self.pkgCache[pkgfile];
  474. if (cached) return nextTick(cb, null, cached);
  475. else if (cached === false) return next();
  476. var pcached = self.pkgFileCachePending[pkgfile];
  477. if (pcached) return pcached.push(onpkg);
  478. pcached = self.pkgFileCachePending[pkgfile] = [];
  479. fs.readFile(pkgfile, function (err, src) {
  480. if (err) return onpkg();
  481. try { var pkg = JSON.parse(src) }
  482. catch (err) {
  483. return onpkg(new Error([
  484. err + ' while parsing json file ' + pkgfile
  485. ].join('')))
  486. }
  487. pkg.__dirname = dir;
  488. self.pkgCache[pkgfile] = pkg;
  489. self.pkgCache[file] = pkg;
  490. onpkg(null, pkg);
  491. });
  492. function onpkg (err, pkg) {
  493. if (self.pkgFileCachePending[pkgfile]) {
  494. var fns = self.pkgFileCachePending[pkgfile];
  495. delete self.pkgFileCachePending[pkgfile];
  496. fns.forEach(function (f) { f(err, pkg) });
  497. }
  498. if (err) cb(err)
  499. else if (pkg) cb(null, pkg)
  500. else {
  501. self.pkgCache[pkgfile] = false;
  502. next();
  503. }
  504. }
  505. })();
  506. };
  507. function getTransforms (pkg, opts) {
  508. var trx = [];
  509. if (opts.transformKey) {
  510. var n = pkg;
  511. var keys = opts.transformKey;
  512. for (var i = 0; i < keys.length; i++) {
  513. if (n && typeof n === 'object') n = n[keys[i]];
  514. else break;
  515. }
  516. if (i === keys.length) {
  517. trx = [].concat(n).filter(Boolean);
  518. }
  519. }
  520. return trx.concat(opts.globalTransform || []);
  521. }
  522. function nextTick (cb) {
  523. var args = [].slice.call(arguments, 1);
  524. process.nextTick(function () { cb.apply(null, args) });
  525. }
  526. function xhas (obj) {
  527. if (!obj) return false;
  528. for (var i = 1; i < arguments.length; i++) {
  529. var key = arguments[i];
  530. if (!has(obj, key)) return false;
  531. obj = obj[key];
  532. }
  533. return true;
  534. }
  535. function has (obj, key) {
  536. return obj && Object.prototype.hasOwnProperty.call(obj, key);
  537. }
  538. function wrapTransform (tr) {
  539. if (typeof tr.read === 'function') return tr;
  540. var input = through(), output = through();
  541. input.pipe(tr).pipe(output);
  542. var wrapper = duplexer(input, output);
  543. tr.on('error', function (err) { wrapper.emit('error', err) });
  544. return wrapper;
  545. }