glob.js
1 // Approach: 2 // 3 // 1. Get the minimatch set 4 // 2. For each pattern in the set, PROCESS(pattern, false) 5 // 3. Store matches per-set, then uniq them 6 // 7 // PROCESS(pattern, inGlobStar) 8 // Get the first [n] items from pattern that are all strings 9 // Join these together. This is PREFIX. 10 // If there is no more remaining, then stat(PREFIX) and 11 // add to matches if it succeeds. END. 12 // 13 // If inGlobStar and PREFIX is symlink and points to dir 14 // set ENTRIES = [] 15 // else readdir(PREFIX) as ENTRIES 16 // If fail, END 17 // 18 // with ENTRIES 19 // If pattern[n] is GLOBSTAR 20 // // handle the case where the globstar match is empty 21 // // by pruning it out, and testing the resulting pattern 22 // PROCESS(pattern[0..n] + pattern[n+1 .. $], false) 23 // // handle other cases. 24 // for ENTRY in ENTRIES (not dotfiles) 25 // // attach globstar + tail onto the entry 26 // // Mark that this entry is a globstar match 27 // PROCESS(pattern[0..n] + ENTRY + pattern[n .. $], true) 28 // 29 // else // not globstar 30 // for ENTRY in ENTRIES (not dotfiles, unless pattern[n] is dot) 31 // Test ENTRY against pattern[n] 32 // If fails, continue 33 // If passes, PROCESS(pattern[0..n] + item + pattern[n+1 .. $]) 34 // 35 // Caveat: 36 // Cache all stats and readdirs results to minimize syscall. Since all 37 // we ever care about is existence and directory-ness, we can just keep 38 // `true` for files, and [children,...] for directories, or `false` for 39 // things that don't exist. 40 41 module.exports = glob 42 43 var rp = require('fs.realpath') 44 var minimatch = require('minimatch') 45 var Minimatch = minimatch.Minimatch 46 var inherits = require('inherits') 47 var EE = require('events').EventEmitter 48 var path = require('path') 49 var assert = require('assert') 50 var isAbsolute = require('path-is-absolute') 51 var globSync = require('./sync.js') 52 var common = require('./common.js') 53 var setopts = common.setopts 54 var ownProp = common.ownProp 55 var inflight = require('inflight') 56 var util = require('util') 57 var childrenIgnored = common.childrenIgnored 58 var isIgnored = common.isIgnored 59 60 var once = require('once') 61 62 function glob (pattern, options, cb) { 63 if (typeof options === 'function') cb = options, options = {} 64 if (!options) options = {} 65 66 if (options.sync) { 67 if (cb) 68 throw new TypeError('callback provided to sync glob') 69 return globSync(pattern, options) 70 } 71 72 return new Glob(pattern, options, cb) 73 } 74 75 glob.sync = globSync 76 var GlobSync = glob.GlobSync = globSync.GlobSync 77 78 // old api surface 79 glob.glob = glob 80 81 function extend (origin, add) { 82 if (add === null || typeof add !== 'object') { 83 return origin 84 } 85 86 var keys = Object.keys(add) 87 var i = keys.length 88 while (i--) { 89 origin[keys[i]] = add[keys[i]] 90 } 91 return origin 92 } 93 94 glob.hasMagic = function (pattern, options_) { 95 var options = extend({}, options_) 96 options.noprocess = true 97 98 var g = new Glob(pattern, options) 99 var set = g.minimatch.set 100 101 if (!pattern) 102 return false 103 104 if (set.length > 1) 105 return true 106 107 for (var j = 0; j < set[0].length; j++) { 108 if (typeof set[0][j] !== 'string') 109 return true 110 } 111 112 return false 113 } 114 115 glob.Glob = Glob 116 inherits(Glob, EE) 117 function Glob (pattern, options, cb) { 118 if (typeof options === 'function') { 119 cb = options 120 options = null 121 } 122 123 if (options && options.sync) { 124 if (cb) 125 throw new TypeError('callback provided to sync glob') 126 return new GlobSync(pattern, options) 127 } 128 129 if (!(this instanceof Glob)) 130 return new Glob(pattern, options, cb) 131 132 setopts(this, pattern, options) 133 this._didRealPath = false 134 135 // process each pattern in the minimatch set 136 var n = this.minimatch.set.length 137 138 // The matches are stored as {<filename>: true,...} so that 139 // duplicates are automagically pruned. 140 // Later, we do an Object.keys() on these. 141 // Keep them as a list so we can fill in when nonull is set. 142 this.matches = new Array(n) 143 144 if (typeof cb === 'function') { 145 cb = once(cb) 146 this.on('error', cb) 147 this.on('end', function (matches) { 148 cb(null, matches) 149 }) 150 } 151 152 var self = this 153 this._processing = 0 154 155 this._emitQueue = [] 156 this._processQueue = [] 157 this.paused = false 158 159 if (this.noprocess) 160 return this 161 162 if (n === 0) 163 return done() 164 165 var sync = true 166 for (var i = 0; i < n; i ++) { 167 this._process(this.minimatch.set[i], i, false, done) 168 } 169 sync = false 170 171 function done () { 172 --self._processing 173 if (self._processing <= 0) { 174 if (sync) { 175 process.nextTick(function () { 176 self._finish() 177 }) 178 } else { 179 self._finish() 180 } 181 } 182 } 183 } 184 185 Glob.prototype._finish = function () { 186 assert(this instanceof Glob) 187 if (this.aborted) 188 return 189 190 if (this.realpath && !this._didRealpath) 191 return this._realpath() 192 193 common.finish(this) 194 this.emit('end', this.found) 195 } 196 197 Glob.prototype._realpath = function () { 198 if (this._didRealpath) 199 return 200 201 this._didRealpath = true 202 203 var n = this.matches.length 204 if (n === 0) 205 return this._finish() 206 207 var self = this 208 for (var i = 0; i < this.matches.length; i++) 209 this._realpathSet(i, next) 210 211 function next () { 212 if (--n === 0) 213 self._finish() 214 } 215 } 216 217 Glob.prototype._realpathSet = function (index, cb) { 218 var matchset = this.matches[index] 219 if (!matchset) 220 return cb() 221 222 var found = Object.keys(matchset) 223 var self = this 224 var n = found.length 225 226 if (n === 0) 227 return cb() 228 229 var set = this.matches[index] = Object.create(null) 230 found.forEach(function (p, i) { 231 // If there's a problem with the stat, then it means that 232 // one or more of the links in the realpath couldn't be 233 // resolved. just return the abs value in that case. 234 p = self._makeAbs(p) 235 rp.realpath(p, self.realpathCache, function (er, real) { 236 if (!er) 237 set[real] = true 238 else if (er.syscall === 'stat') 239 set[p] = true 240 else 241 self.emit('error', er) // srsly wtf right here 242 243 if (--n === 0) { 244 self.matches[index] = set 245 cb() 246 } 247 }) 248 }) 249 } 250 251 Glob.prototype._mark = function (p) { 252 return common.mark(this, p) 253 } 254 255 Glob.prototype._makeAbs = function (f) { 256 return common.makeAbs(this, f) 257 } 258 259 Glob.prototype.abort = function () { 260 this.aborted = true 261 this.emit('abort') 262 } 263 264 Glob.prototype.pause = function () { 265 if (!this.paused) { 266 this.paused = true 267 this.emit('pause') 268 } 269 } 270 271 Glob.prototype.resume = function () { 272 if (this.paused) { 273 this.emit('resume') 274 this.paused = false 275 if (this._emitQueue.length) { 276 var eq = this._emitQueue.slice(0) 277 this._emitQueue.length = 0 278 for (var i = 0; i < eq.length; i ++) { 279 var e = eq[i] 280 this._emitMatch(e[0], e[1]) 281 } 282 } 283 if (this._processQueue.length) { 284 var pq = this._processQueue.slice(0) 285 this._processQueue.length = 0 286 for (var i = 0; i < pq.length; i ++) { 287 var p = pq[i] 288 this._processing-- 289 this._process(p[0], p[1], p[2], p[3]) 290 } 291 } 292 } 293 } 294 295 Glob.prototype._process = function (pattern, index, inGlobStar, cb) { 296 assert(this instanceof Glob) 297 assert(typeof cb === 'function') 298 299 if (this.aborted) 300 return 301 302 this._processing++ 303 if (this.paused) { 304 this._processQueue.push([pattern, index, inGlobStar, cb]) 305 return 306 } 307 308 //console.error('PROCESS %d', this._processing, pattern) 309 310 // Get the first [n] parts of pattern that are all strings. 311 var n = 0 312 while (typeof pattern[n] === 'string') { 313 n ++ 314 } 315 // now n is the index of the first one that is *not* a string. 316 317 // see if there's anything else 318 var prefix 319 switch (n) { 320 // if not, then this is rather simple 321 case pattern.length: 322 this._processSimple(pattern.join('/'), index, cb) 323 return 324 325 case 0: 326 // pattern *starts* with some non-trivial item. 327 // going to readdir(cwd), but not include the prefix in matches. 328 prefix = null 329 break 330 331 default: 332 // pattern has some string bits in the front. 333 // whatever it starts with, whether that's 'absolute' like /foo/bar, 334 // or 'relative' like '../baz' 335 prefix = pattern.slice(0, n).join('/') 336 break 337 } 338 339 var remain = pattern.slice(n) 340 341 // get the list of entries. 342 var read 343 if (prefix === null) 344 read = '.' 345 else if (isAbsolute(prefix) || isAbsolute(pattern.join('/'))) { 346 if (!prefix || !isAbsolute(prefix)) 347 prefix = '/' + prefix 348 read = prefix 349 } else 350 read = prefix 351 352 var abs = this._makeAbs(read) 353 354 //if ignored, skip _processing 355 if (childrenIgnored(this, read)) 356 return cb() 357 358 var isGlobStar = remain[0] === minimatch.GLOBSTAR 359 if (isGlobStar) 360 this._processGlobStar(prefix, read, abs, remain, index, inGlobStar, cb) 361 else 362 this._processReaddir(prefix, read, abs, remain, index, inGlobStar, cb) 363 } 364 365 Glob.prototype._processReaddir = function (prefix, read, abs, remain, index, inGlobStar, cb) { 366 var self = this 367 this._readdir(abs, inGlobStar, function (er, entries) { 368 return self._processReaddir2(prefix, read, abs, remain, index, inGlobStar, entries, cb) 369 }) 370 } 371 372 Glob.prototype._processReaddir2 = function (prefix, read, abs, remain, index, inGlobStar, entries, cb) { 373 374 // if the abs isn't a dir, then nothing can match! 375 if (!entries) 376 return cb() 377 378 // It will only match dot entries if it starts with a dot, or if 379 // dot is set. Stuff like @(.foo|.bar) isn't allowed. 380 var pn = remain[0] 381 var negate = !!this.minimatch.negate 382 var rawGlob = pn._glob 383 var dotOk = this.dot || rawGlob.charAt(0) === '.' 384 385 var matchedEntries = [] 386 for (var i = 0; i < entries.length; i++) { 387 var e = entries[i] 388 if (e.charAt(0) !== '.' || dotOk) { 389 var m 390 if (negate && !prefix) { 391 m = !e.match(pn) 392 } else { 393 m = e.match(pn) 394 } 395 if (m) 396 matchedEntries.push(e) 397 } 398 } 399 400 //console.error('prd2', prefix, entries, remain[0]._glob, matchedEntries) 401 402 var len = matchedEntries.length 403 // If there are no matched entries, then nothing matches. 404 if (len === 0) 405 return cb() 406 407 // if this is the last remaining pattern bit, then no need for 408 // an additional stat *unless* the user has specified mark or 409 // stat explicitly. We know they exist, since readdir returned 410 // them. 411 412 if (remain.length === 1 && !this.mark && !this.stat) { 413 if (!this.matches[index]) 414 this.matches[index] = Object.create(null) 415 416 for (var i = 0; i < len; i ++) { 417 var e = matchedEntries[i] 418 if (prefix) { 419 if (prefix !== '/') 420 e = prefix + '/' + e 421 else 422 e = prefix + e 423 } 424 425 if (e.charAt(0) === '/' && !this.nomount) { 426 e = path.join(this.root, e) 427 } 428 this._emitMatch(index, e) 429 } 430 // This was the last one, and no stats were needed 431 return cb() 432 } 433 434 // now test all matched entries as stand-ins for that part 435 // of the pattern. 436 remain.shift() 437 for (var i = 0; i < len; i ++) { 438 var e = matchedEntries[i] 439 var newPattern 440 if (prefix) { 441 if (prefix !== '/') 442 e = prefix + '/' + e 443 else 444 e = prefix + e 445 } 446 this._process([e].concat(remain), index, inGlobStar, cb) 447 } 448 cb() 449 } 450 451 Glob.prototype._emitMatch = function (index, e) { 452 if (this.aborted) 453 return 454 455 if (isIgnored(this, e)) 456 return 457 458 if (this.paused) { 459 this._emitQueue.push([index, e]) 460 return 461 } 462 463 var abs = isAbsolute(e) ? e : this._makeAbs(e) 464 465 if (this.mark) 466 e = this._mark(e) 467 468 if (this.absolute) 469 e = abs 470 471 if (this.matches[index][e]) 472 return 473 474 if (this.nodir) { 475 var c = this.cache[abs] 476 if (c === 'DIR' || Array.isArray(c)) 477 return 478 } 479 480 this.matches[index][e] = true 481 482 var st = this.statCache[abs] 483 if (st) 484 this.emit('stat', e, st) 485 486 this.emit('match', e) 487 } 488 489 Glob.prototype._readdirInGlobStar = function (abs, cb) { 490 if (this.aborted) 491 return 492 493 // follow all symlinked directories forever 494 // just proceed as if this is a non-globstar situation 495 if (this.follow) 496 return this._readdir(abs, false, cb) 497 498 var lstatkey = 'lstat\0' + abs 499 var self = this 500 var lstatcb = inflight(lstatkey, lstatcb_) 501 502 if (lstatcb) 503 self.fs.lstat(abs, lstatcb) 504 505 function lstatcb_ (er, lstat) { 506 if (er && er.code === 'ENOENT') 507 return cb() 508 509 var isSym = lstat && lstat.isSymbolicLink() 510 self.symlinks[abs] = isSym 511 512 // If it's not a symlink or a dir, then it's definitely a regular file. 513 // don't bother doing a readdir in that case. 514 if (!isSym && lstat && !lstat.isDirectory()) { 515 self.cache[abs] = 'FILE' 516 cb() 517 } else 518 self._readdir(abs, false, cb) 519 } 520 } 521 522 Glob.prototype._readdir = function (abs, inGlobStar, cb) { 523 if (this.aborted) 524 return 525 526 cb = inflight('readdir\0'+abs+'\0'+inGlobStar, cb) 527 if (!cb) 528 return 529 530 //console.error('RD %j %j', +inGlobStar, abs) 531 if (inGlobStar && !ownProp(this.symlinks, abs)) 532 return this._readdirInGlobStar(abs, cb) 533 534 if (ownProp(this.cache, abs)) { 535 var c = this.cache[abs] 536 if (!c || c === 'FILE') 537 return cb() 538 539 if (Array.isArray(c)) 540 return cb(null, c) 541 } 542 543 var self = this 544 self.fs.readdir(abs, readdirCb(this, abs, cb)) 545 } 546 547 function readdirCb (self, abs, cb) { 548 return function (er, entries) { 549 if (er) 550 self._readdirError(abs, er, cb) 551 else 552 self._readdirEntries(abs, entries, cb) 553 } 554 } 555 556 Glob.prototype._readdirEntries = function (abs, entries, cb) { 557 if (this.aborted) 558 return 559 560 // if we haven't asked to stat everything, then just 561 // assume that everything in there exists, so we can avoid 562 // having to stat it a second time. 563 if (!this.mark && !this.stat) { 564 for (var i = 0; i < entries.length; i ++) { 565 var e = entries[i] 566 if (abs === '/') 567 e = abs + e 568 else 569 e = abs + '/' + e 570 this.cache[e] = true 571 } 572 } 573 574 this.cache[abs] = entries 575 return cb(null, entries) 576 } 577 578 Glob.prototype._readdirError = function (f, er, cb) { 579 if (this.aborted) 580 return 581 582 // handle errors, and cache the information 583 switch (er.code) { 584 case 'ENOTSUP': // https://github.com/isaacs/node-glob/issues/205 585 case 'ENOTDIR': // totally normal. means it *does* exist. 586 var abs = this._makeAbs(f) 587 this.cache[abs] = 'FILE' 588 if (abs === this.cwdAbs) { 589 var error = new Error(er.code + ' invalid cwd ' + this.cwd) 590 error.path = this.cwd 591 error.code = er.code 592 this.emit('error', error) 593 this.abort() 594 } 595 break 596 597 case 'ENOENT': // not terribly unusual 598 case 'ELOOP': 599 case 'ENAMETOOLONG': 600 case 'UNKNOWN': 601 this.cache[this._makeAbs(f)] = false 602 break 603 604 default: // some unusual error. Treat as failure. 605 this.cache[this._makeAbs(f)] = false 606 if (this.strict) { 607 this.emit('error', er) 608 // If the error is handled, then we abort 609 // if not, we threw out of here 610 this.abort() 611 } 612 if (!this.silent) 613 console.error('glob error', er) 614 break 615 } 616 617 return cb() 618 } 619 620 Glob.prototype._processGlobStar = function (prefix, read, abs, remain, index, inGlobStar, cb) { 621 var self = this 622 this._readdir(abs, inGlobStar, function (er, entries) { 623 self._processGlobStar2(prefix, read, abs, remain, index, inGlobStar, entries, cb) 624 }) 625 } 626 627 628 Glob.prototype._processGlobStar2 = function (prefix, read, abs, remain, index, inGlobStar, entries, cb) { 629 //console.error('pgs2', prefix, remain[0], entries) 630 631 // no entries means not a dir, so it can never have matches 632 // foo.txt/** doesn't match foo.txt 633 if (!entries) 634 return cb() 635 636 // test without the globstar, and with every child both below 637 // and replacing the globstar. 638 var remainWithoutGlobStar = remain.slice(1) 639 var gspref = prefix ? [ prefix ] : [] 640 var noGlobStar = gspref.concat(remainWithoutGlobStar) 641 642 // the noGlobStar pattern exits the inGlobStar state 643 this._process(noGlobStar, index, false, cb) 644 645 var isSym = this.symlinks[abs] 646 var len = entries.length 647 648 // If it's a symlink, and we're in a globstar, then stop 649 if (isSym && inGlobStar) 650 return cb() 651 652 for (var i = 0; i < len; i++) { 653 var e = entries[i] 654 if (e.charAt(0) === '.' && !this.dot) 655 continue 656 657 // these two cases enter the inGlobStar state 658 var instead = gspref.concat(entries[i], remainWithoutGlobStar) 659 this._process(instead, index, true, cb) 660 661 var below = gspref.concat(entries[i], remain) 662 this._process(below, index, true, cb) 663 } 664 665 cb() 666 } 667 668 Glob.prototype._processSimple = function (prefix, index, cb) { 669 // XXX review this. Shouldn't it be doing the mounting etc 670 // before doing stat? kinda weird? 671 var self = this 672 this._stat(prefix, function (er, exists) { 673 self._processSimple2(prefix, index, er, exists, cb) 674 }) 675 } 676 Glob.prototype._processSimple2 = function (prefix, index, er, exists, cb) { 677 678 //console.error('ps2', prefix, exists) 679 680 if (!this.matches[index]) 681 this.matches[index] = Object.create(null) 682 683 // If it doesn't exist, then just mark the lack of results 684 if (!exists) 685 return cb() 686 687 if (prefix && isAbsolute(prefix) && !this.nomount) { 688 var trail = /[\/\\]$/.test(prefix) 689 if (prefix.charAt(0) === '/') { 690 prefix = path.join(this.root, prefix) 691 } else { 692 prefix = path.resolve(this.root, prefix) 693 if (trail) 694 prefix += '/' 695 } 696 } 697 698 if (process.platform === 'win32') 699 prefix = prefix.replace(/\\/g, '/') 700 701 // Mark this as a match 702 this._emitMatch(index, prefix) 703 cb() 704 } 705 706 // Returns either 'DIR', 'FILE', or false 707 Glob.prototype._stat = function (f, cb) { 708 var abs = this._makeAbs(f) 709 var needDir = f.slice(-1) === '/' 710 711 if (f.length > this.maxLength) 712 return cb() 713 714 if (!this.stat && ownProp(this.cache, abs)) { 715 var c = this.cache[abs] 716 717 if (Array.isArray(c)) 718 c = 'DIR' 719 720 // It exists, but maybe not how we need it 721 if (!needDir || c === 'DIR') 722 return cb(null, c) 723 724 if (needDir && c === 'FILE') 725 return cb() 726 727 // otherwise we have to stat, because maybe c=true 728 // if we know it exists, but not what it is. 729 } 730 731 var exists 732 var stat = this.statCache[abs] 733 if (stat !== undefined) { 734 if (stat === false) 735 return cb(null, stat) 736 else { 737 var type = stat.isDirectory() ? 'DIR' : 'FILE' 738 if (needDir && type === 'FILE') 739 return cb() 740 else 741 return cb(null, type, stat) 742 } 743 } 744 745 var self = this 746 var statcb = inflight('stat\0' + abs, lstatcb_) 747 if (statcb) 748 self.fs.lstat(abs, statcb) 749 750 function lstatcb_ (er, lstat) { 751 if (lstat && lstat.isSymbolicLink()) { 752 // If it's a symlink, then treat it as the target, unless 753 // the target does not exist, then treat it as a file. 754 return self.fs.stat(abs, function (er, stat) { 755 if (er) 756 self._stat2(f, abs, null, lstat, cb) 757 else 758 self._stat2(f, abs, er, stat, cb) 759 }) 760 } else { 761 self._stat2(f, abs, er, lstat, cb) 762 } 763 } 764 } 765 766 Glob.prototype._stat2 = function (f, abs, er, stat, cb) { 767 if (er && (er.code === 'ENOENT' || er.code === 'ENOTDIR')) { 768 this.statCache[abs] = false 769 return cb() 770 } 771 772 var needDir = f.slice(-1) === '/' 773 this.statCache[abs] = stat 774 775 if (abs.slice(-1) === '/' && stat && !stat.isDirectory()) 776 return cb(null, false, stat) 777 778 var c = true 779 if (stat) 780 c = stat.isDirectory() ? 'DIR' : 'FILE' 781 this.cache[abs] = this.cache[abs] || c 782 783 if (needDir && c === 'FILE') 784 return cb() 785 786 return cb(null, c, stat) 787 }