You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

380 lines
12 KiB

4 years ago
  1. 'use strict';
  2. const Readable = require('stream').Readable;
  3. const EventEmitter = require('events').EventEmitter;
  4. const path = require('path');
  5. const normalizeOptions = require('./normalize-options');
  6. const stat = require('./stat');
  7. const call = require('./call');
  8. /**
  9. * Asynchronously reads the contents of a directory and streams the results
  10. * via a {@link stream.Readable}.
  11. */
  12. class DirectoryReader {
  13. /**
  14. * @param {string} dir - The absolute or relative directory path to read
  15. * @param {object} [options] - User-specified options, if any (see {@link normalizeOptions})
  16. * @param {object} internalOptions - Internal options that aren't part of the public API
  17. * @class
  18. */
  19. constructor (dir, options, internalOptions) {
  20. this.options = options = normalizeOptions(options, internalOptions);
  21. // Indicates whether we should keep reading
  22. // This is set false if stream.Readable.push() returns false.
  23. this.shouldRead = true;
  24. // The directories to read
  25. // (initialized with the top-level directory)
  26. this.queue = [{
  27. path: dir,
  28. basePath: options.basePath,
  29. posixBasePath: options.posixBasePath,
  30. depth: 0
  31. }];
  32. // The number of directories that are currently being processed
  33. this.pending = 0;
  34. // The data that has been read, but not yet emitted
  35. this.buffer = [];
  36. this.stream = new Readable({ objectMode: true });
  37. this.stream._read = () => {
  38. // Start (or resume) reading
  39. this.shouldRead = true;
  40. // If we have data in the buffer, then send the next chunk
  41. if (this.buffer.length > 0) {
  42. this.pushFromBuffer();
  43. }
  44. // If we have directories queued, then start processing the next one
  45. if (this.queue.length > 0) {
  46. if (this.options.facade.sync) {
  47. while (this.queue.length > 0) {
  48. this.readNextDirectory();
  49. }
  50. }
  51. else {
  52. this.readNextDirectory();
  53. }
  54. }
  55. this.checkForEOF();
  56. };
  57. }
  58. /**
  59. * Reads the next directory in the queue
  60. */
  61. readNextDirectory () {
  62. let facade = this.options.facade;
  63. let dir = this.queue.shift();
  64. this.pending++;
  65. // Read the directory listing
  66. call.safe(facade.fs.readdir, dir.path, (err, items) => {
  67. if (err) {
  68. // fs.readdir threw an error
  69. this.emit('error', err);
  70. return this.finishedReadingDirectory();
  71. }
  72. try {
  73. // Process each item in the directory (simultaneously, if async)
  74. facade.forEach(
  75. items,
  76. this.processItem.bind(this, dir),
  77. this.finishedReadingDirectory.bind(this, dir)
  78. );
  79. }
  80. catch (err2) {
  81. // facade.forEach threw an error
  82. // (probably because fs.readdir returned an invalid result)
  83. this.emit('error', err2);
  84. this.finishedReadingDirectory();
  85. }
  86. });
  87. }
  88. /**
  89. * This method is called after all items in a directory have been processed.
  90. *
  91. * NOTE: This does not necessarily mean that the reader is finished, since there may still
  92. * be other directories queued or pending.
  93. */
  94. finishedReadingDirectory () {
  95. this.pending--;
  96. if (this.shouldRead) {
  97. // If we have directories queued, then start processing the next one
  98. if (this.queue.length > 0 && this.options.facade.async) {
  99. this.readNextDirectory();
  100. }
  101. this.checkForEOF();
  102. }
  103. }
  104. /**
  105. * Determines whether the reader has finished processing all items in all directories.
  106. * If so, then the "end" event is fired (via {@Readable#push})
  107. */
  108. checkForEOF () {
  109. if (this.buffer.length === 0 && // The stuff we've already read
  110. this.pending === 0 && // The stuff we're currently reading
  111. this.queue.length === 0) { // The stuff we haven't read yet
  112. // There's no more stuff!
  113. this.stream.push(null);
  114. }
  115. }
  116. /**
  117. * Processes a single item in a directory.
  118. *
  119. * If the item is a directory, and `option.deep` is enabled, then the item will be added
  120. * to the directory queue.
  121. *
  122. * If the item meets the filter criteria, then it will be emitted to the reader's stream.
  123. *
  124. * @param {object} dir - A directory object from the queue
  125. * @param {string} item - The name of the item (name only, no path)
  126. * @param {function} done - A callback function that is called after the item has been processed
  127. */
  128. processItem (dir, item, done) {
  129. let stream = this.stream;
  130. let options = this.options;
  131. let itemPath = dir.basePath + item;
  132. let posixPath = dir.posixBasePath + item;
  133. let fullPath = path.join(dir.path, item);
  134. // If `options.deep` is a number, and we've already recursed to the max depth,
  135. // then there's no need to check fs.Stats to know if it's a directory.
  136. // If `options.deep` is a function, then we'll need fs.Stats
  137. let maxDepthReached = dir.depth >= options.recurseDepth;
  138. // Do we need to call `fs.stat`?
  139. let needStats =
  140. !maxDepthReached || // we need the fs.Stats to know if it's a directory
  141. options.stats || // the user wants fs.Stats objects returned
  142. options.recurseFn || // we need fs.Stats for the recurse function
  143. options.filterFn || // we need fs.Stats for the filter function
  144. EventEmitter.listenerCount(stream, 'file') || // we need the fs.Stats to know if it's a file
  145. EventEmitter.listenerCount(stream, 'directory') || // we need the fs.Stats to know if it's a directory
  146. EventEmitter.listenerCount(stream, 'symlink'); // we need the fs.Stats to know if it's a symlink
  147. // If we don't need stats, then exit early
  148. if (!needStats) {
  149. if (this.filter(itemPath, posixPath)) {
  150. this.pushOrBuffer({ data: itemPath });
  151. }
  152. return done();
  153. }
  154. // Get the fs.Stats object for this path
  155. stat(options.facade.fs, fullPath, (err, stats) => {
  156. if (err) {
  157. // fs.stat threw an error
  158. this.emit('error', err);
  159. return done();
  160. }
  161. try {
  162. // Add the item's path to the fs.Stats object
  163. // The base of this path, and its separators are determined by the options
  164. // (i.e. options.basePath and options.sep)
  165. stats.path = itemPath;
  166. // Add depth of the path to the fs.Stats object for use this in the filter function
  167. stats.depth = dir.depth;
  168. if (this.shouldRecurse(stats, posixPath, maxDepthReached)) {
  169. // Add this subdirectory to the queue
  170. this.queue.push({
  171. path: fullPath,
  172. basePath: itemPath + options.sep,
  173. posixBasePath: posixPath + '/',
  174. depth: dir.depth + 1,
  175. });
  176. }
  177. // Determine whether this item matches the filter criteria
  178. if (this.filter(stats, posixPath)) {
  179. this.pushOrBuffer({
  180. data: options.stats ? stats : itemPath,
  181. file: stats.isFile(),
  182. directory: stats.isDirectory(),
  183. symlink: stats.isSymbolicLink(),
  184. });
  185. }
  186. done();
  187. }
  188. catch (err2) {
  189. // An error occurred while processing the item
  190. // (probably during a user-specified function, such as options.deep, options.filter, etc.)
  191. this.emit('error', err2);
  192. done();
  193. }
  194. });
  195. }
  196. /**
  197. * Pushes the given chunk of data to the stream, or adds it to the buffer,
  198. * depending on the state of the stream.
  199. *
  200. * @param {object} chunk
  201. */
  202. pushOrBuffer (chunk) {
  203. // Add the chunk to the buffer
  204. this.buffer.push(chunk);
  205. // If we're still reading, then immediately emit the next chunk in the buffer
  206. // (which may or may not be the chunk that we just added)
  207. if (this.shouldRead) {
  208. this.pushFromBuffer();
  209. }
  210. }
  211. /**
  212. * Immediately pushes the next chunk in the buffer to the reader's stream.
  213. * The "data" event will always be fired (via {@link Readable#push}).
  214. * In addition, the "file", "directory", and/or "symlink" events may be fired,
  215. * depending on the type of properties of the chunk.
  216. */
  217. pushFromBuffer () {
  218. let stream = this.stream;
  219. let chunk = this.buffer.shift();
  220. // Stream the data
  221. try {
  222. this.shouldRead = stream.push(chunk.data);
  223. }
  224. catch (err) {
  225. this.emit('error', err);
  226. }
  227. // Also emit specific events, based on the type of chunk
  228. chunk.file && this.emit('file', chunk.data);
  229. chunk.symlink && this.emit('symlink', chunk.data);
  230. chunk.directory && this.emit('directory', chunk.data);
  231. }
  232. /**
  233. * Determines whether the given directory meets the user-specified recursion criteria.
  234. * If the user didn't specify recursion criteria, then this function will default to true.
  235. *
  236. * @param {fs.Stats} stats - The directory's {@link fs.Stats} object
  237. * @param {string} posixPath - The item's POSIX path (used for glob matching)
  238. * @param {boolean} maxDepthReached - Whether we've already crawled the user-specified depth
  239. * @returns {boolean}
  240. */
  241. shouldRecurse (stats, posixPath, maxDepthReached) {
  242. let options = this.options;
  243. if (maxDepthReached) {
  244. // We've already crawled to the maximum depth. So no more recursion.
  245. return false;
  246. }
  247. else if (!stats.isDirectory()) {
  248. // It's not a directory. So don't try to crawl it.
  249. return false;
  250. }
  251. else if (options.recurseGlob) {
  252. // Glob patterns are always tested against the POSIX path, even on Windows
  253. // https://github.com/isaacs/node-glob#windows
  254. return options.recurseGlob.test(posixPath);
  255. }
  256. else if (options.recurseRegExp) {
  257. // Regular expressions are tested against the normal path
  258. // (based on the OS or options.sep)
  259. return options.recurseRegExp.test(stats.path);
  260. }
  261. else if (options.recurseFn) {
  262. try {
  263. // Run the user-specified recursion criteria
  264. return options.recurseFn.call(null, stats);
  265. }
  266. catch (err) {
  267. // An error occurred in the user's code.
  268. // In Sync and Async modes, this will return an error.
  269. // In Streaming mode, we emit an "error" event, but continue processing
  270. this.emit('error', err);
  271. }
  272. }
  273. else {
  274. // No recursion function was specified, and we're within the maximum depth.
  275. // So crawl this directory.
  276. return true;
  277. }
  278. }
  279. /**
  280. * Determines whether the given item meets the user-specified filter criteria.
  281. * If the user didn't specify a filter, then this function will always return true.
  282. *
  283. * @param {string|fs.Stats} value - Either the item's path, or the item's {@link fs.Stats} object
  284. * @param {string} posixPath - The item's POSIX path (used for glob matching)
  285. * @returns {boolean}
  286. */
  287. filter (value, posixPath) {
  288. let options = this.options;
  289. if (options.filterGlob) {
  290. // Glob patterns are always tested against the POSIX path, even on Windows
  291. // https://github.com/isaacs/node-glob#windows
  292. return options.filterGlob.test(posixPath);
  293. }
  294. else if (options.filterRegExp) {
  295. // Regular expressions are tested against the normal path
  296. // (based on the OS or options.sep)
  297. return options.filterRegExp.test(value.path || value);
  298. }
  299. else if (options.filterFn) {
  300. try {
  301. // Run the user-specified filter function
  302. return options.filterFn.call(null, value);
  303. }
  304. catch (err) {
  305. // An error occurred in the user's code.
  306. // In Sync and Async modes, this will return an error.
  307. // In Streaming mode, we emit an "error" event, but continue processing
  308. this.emit('error', err);
  309. }
  310. }
  311. else {
  312. // No filter was specified, so match everything
  313. return true;
  314. }
  315. }
  316. /**
  317. * Emits an event. If one of the event listeners throws an error,
  318. * then an "error" event is emitted.
  319. *
  320. * @param {string} eventName
  321. * @param {*} data
  322. */
  323. emit (eventName, data) {
  324. let stream = this.stream;
  325. try {
  326. stream.emit(eventName, data);
  327. }
  328. catch (err) {
  329. if (eventName === 'error') {
  330. // Don't recursively emit "error" events.
  331. // If the first one fails, then just throw
  332. throw err;
  333. }
  334. else {
  335. stream.emit('error', err);
  336. }
  337. }
  338. }
  339. }
  340. module.exports = DirectoryReader;