Merge pull request #249 from mrmlnc/ISSUE-156_partial_matcher

ISSUE-156: add partial matcher for deep filter

Author: mrmlnc

package.json

@@ -26,7 +26,7 @@
     "@types/glob": "^7.1.1",
     "@types/glob-parent": "^5.1.0",
     "@types/merge2": "^1.1.4",
-    "@types/micromatch": "^3.1.0",
+    "@types/micromatch": "^4.0.0",
     "@types/minimist": "^1.2.0",
     "@types/mocha": "^5.2.7",
     "@types/node": "^12.7.8",
@@ -65,14 +65,20 @@
     "build": "npm run clean && npm run compile && npm run lint && npm test",
     "watch": "npm run clean && npm run compile -- --sourceMap --watch",
     "bench": "npm run bench-async && npm run bench-stream && npm run bench-sync",
-    "bench-async": "npm run bench-async-flatten && npm run bench-async-deep",
-    "bench-stream": "npm run bench-stream-flatten && npm run bench-stream-deep",
-    "bench-sync": "npm run bench-sync-flatten && npm run bench-sync-deep",
+    "bench-async": "npm run bench-async-flatten && npm run bench-async-deep && npm run bench-async-partial-flatten && npm run bench-async-partial-deep",
+    "bench-stream": "npm run bench-stream-flatten && npm run bench-stream-deep && npm run bench-stream-partial-flatten && npm run bench-stream-partial-deep",
+    "bench-sync": "npm run bench-sync-flatten && npm run bench-sync-deep && npm run bench-sync-partial-flatten && npm run bench-sync-partial-deep",
     "bench-async-flatten": "node ./out/benchmark --mode async --pattern \"*\"",
     "bench-async-deep": "node ./out/benchmark --mode async --pattern \"**\"",
+    "bench-async-partial-flatten": "node ./out/benchmark --mode async --pattern \"{fixtures,out}/{first,second}/*\"",
+    "bench-async-partial-deep": "node ./out/benchmark --mode async --pattern \"{fixtures,out}/**\"",
     "bench-stream-flatten": "node ./out/benchmark --mode stream --pattern \"*\"",
     "bench-stream-deep": "node ./out/benchmark --mode stream --pattern \"**\"",
+    "bench-stream-partial-flatten": "node ./out/benchmark --mode stream --pattern \"{fixtures,out}/{first,second}/*\"",
+    "bench-stream-partial-deep": "node ./out/benchmark --mode stream --pattern \"{fixtures,out}/**\"",
     "bench-sync-flatten": "node ./out/benchmark --mode sync --pattern \"*\"",
-    "bench-sync-deep": "node ./out/benchmark --mode sync --pattern \"**\""
+    "bench-sync-deep": "node ./out/benchmark --mode sync --pattern \"**\"",
+    "bench-sync-partial-flatten": "node ./out/benchmark --mode sync --pattern \"{fixtures,out}/{first,second}/*\"",
+    "bench-sync-partial-deep": "node ./out/benchmark --mode sync --pattern \"{fixtures,out}/**\""
   }
 }

src/providers/filters/deep.spec.ts

@@ -65,39 +65,37 @@ describe('Providers → Filters → Deep', () => {
 			});
 		});
 
-		describe('Max pattern depth', () => {
-			it('should return `false` when the depth of entry is greater that the pattern depth', () => {
-				const filter = getFilter('root', ['root/*'], []);
-				const entry = tests.entry.builder().path('root/directory').directory().build();
+		describe('options.followSymbolicLinks', () => {
+			it('should return `false` when an entry is symbolic link and option is disabled', () => {
+				const filter = getFilter('.', ['**/*'], [], { followSymbolicLinks: false });
+				const entry = tests.entry.builder().path('root/directory').directory().symlink().build();
 
 				const actual = filter(entry);
 
 				assert.ok(!actual);
 			});
 		});
 
-		describe('options.followSymbolicLinks', () => {
-			it('should return `false` when an entry is symbolic link and option is disabled', () => {
-				const filter = getFilter('.', ['**/*'], [], { followSymbolicLinks: false });
-				const entry = tests.entry.builder().path('root/directory').directory().symlink().build();
+		describe('Positive pattern', () => {
+			it('should return `false` when an entry does not match to the positive pattern', () => {
+				const filter = getFilter('.', ['non-root/directory'], []);
+				const entry = tests.entry.builder().path('root').directory().build();
 
 				const actual = filter(entry);
 
 				assert.ok(!actual);
 			});
-		});
 
-		describe('Pattern', () => {
-			it('should return `false` when an entry match to the negative pattern', () => {
-				const filter = getFilter('.', ['**/*'], ['root/**']);
-				const entry = tests.entry.builder().path('root/directory').directory().build();
+			it('should return `false` when an entry starts with "./" and does not match to the positive pattern', () => {
+				const filter = getFilter('.', ['non-root/directory'], []);
+				const entry = tests.entry.builder().path('./root').directory().build();
 
 				const actual = filter(entry);
 
 				assert.ok(!actual);
 			});
 
-			it('should return `true` when the positive pattern has no affect to depth reading, but the `baseNameMatch` is enabled', () => {
+			it('should return `true` when the positive pattern does not match, but the `baseNameMatch` is enabled', () => {
 				const filter = getFilter('.', ['*'], [], { baseNameMatch: true });
 				const entry = tests.entry.builder().path('root/directory').directory().build();
 
@@ -106,17 +104,28 @@ describe('Providers → Filters → Deep', () => {
 				assert.ok(actual);
 			});
 
-			it('should return `true` when the negative pattern has no effect to depth reading', () => {
-				const filter = getFilter('.', ['**/*'], ['**/*']);
+			it('should return `true` when the positive pattern has a globstar', () => {
+				const filter = getFilter('.', ['**/*'], []);
 				const entry = tests.entry.builder().path('root/directory').directory().build();
 
 				const actual = filter(entry);
 
 				assert.ok(actual);
 			});
+		});
 
-			it('should return `true`', () => {
-				const filter = getFilter('.', ['**/*'], []);
+		describe('Negative pattern', () => {
+			it('should return `false` when an entry match to the negative pattern', () => {
+				const filter = getFilter('.', ['**/*'], ['root/**']);
+				const entry = tests.entry.builder().path('root/directory').directory().build();
+
+				const actual = filter(entry);
+
+				assert.ok(!actual);
+			});
+
+			it('should return `true` when the negative pattern has no effect to depth reading', () => {
+				const filter = getFilter('.', ['**/*'], ['**/*']);
 				const entry = tests.entry.builder().path('root/directory').directory().build();
 
 				const actual = filter(entry);

src/providers/filters/deep.ts

@@ -1,21 +1,20 @@
+import { Entry, MicromatchOptions, EntryFilterFunction, Pattern, PatternRe } from '../../types';
 import Settings from '../../settings';
-import { Entry, EntryFilterFunction, MicromatchOptions, Pattern, PatternRe } from '../../types';
 import * as utils from '../../utils';
+import PartialMatcher from '../matchers/partial';
 
 export default class DeepFilter {
 	constructor(private readonly _settings: Settings, private readonly _micromatchOptions: MicromatchOptions) { }
 
 	public getFilter(basePath: string, positive: Pattern[], negative: Pattern[]): EntryFilterFunction {
-		const maxPatternDepth = this._getMaxPatternDepth(positive);
+		const matcher = this._getMatcher(positive);
 		const negativeRe = this._getNegativePatternsRe(negative);
 
-		return (entry) => this._filter(basePath, entry, negativeRe, maxPatternDepth);
+		return (entry) => this._filter(basePath, entry, matcher, negativeRe);
 	}
 
-	private _getMaxPatternDepth(patterns: Pattern[]): number {
-		const globstar = patterns.some(utils.pattern.hasGlobStar);
-
-		return globstar ? Infinity : utils.pattern.getMaxNaivePatternsDepth(patterns);
+	private _getMatcher(patterns: Pattern[]): PartialMatcher {
+		return new PartialMatcher(patterns, this._micromatchOptions);
 	}
 
 	private _getNegativePatternsRe(patterns: Pattern[]): PatternRe[] {
@@ -24,41 +23,47 @@ export default class DeepFilter {
 		return utils.pattern.convertPatternsToRe(affectDepthOfReadingPatterns, this._micromatchOptions);
 	}
 
-	private _filter(basePath: string, entry: Entry, negativeRe: PatternRe[], maxPatternDepth: number): boolean {
-		const depth = this._getEntryDepth(basePath, entry.path);
+	private _filter(basePath: string, entry: Entry, matcher: PartialMatcher, negativeRe: PatternRe[]): boolean {
+		const depth = this._getEntryLevel(basePath, entry.path);
 
 		if (this._isSkippedByDeep(depth)) {
 			return false;
 		}
 
-		if (this._isSkippedByMaxPatternDepth(depth, maxPatternDepth)) {
+		if (this._isSkippedSymbolicLink(entry)) {
 			return false;
 		}
 
-		if (this._isSkippedSymbolicLink(entry)) {
+		if (this._isSkippedByPositivePatterns(entry, matcher)) {
 			return false;
 		}
 
 		return this._isSkippedByNegativePatterns(entry, negativeRe);
 	}
 
-	private _getEntryDepth(basePath: string, entryPath: string): number {
+	private _isSkippedByDeep(entryDepth: number): boolean {
+		return entryDepth >= this._settings.deep;
+	}
+
+	private _isSkippedSymbolicLink(entry: Entry): boolean {
+		return !this._settings.followSymbolicLinks && entry.dirent.isSymbolicLink();
+	}
+
+	private _getEntryLevel(basePath: string, entryPath: string): number {
 		const basePathDepth = basePath.split('/').length;
 		const entryPathDepth = entryPath.split('/').length;
 
 		return entryPathDepth - (basePath === '' ? 0 : basePathDepth);
 	}
 
-	private _isSkippedByDeep(entryDepth: number): boolean {
-		return entryDepth >= this._settings.deep;
-	}
+	private _isSkippedByPositivePatterns(entry: Entry, matcher: PartialMatcher): boolean {
+		const filepath = entry.path.replace(/^\.[/\\]/, '');
 
-	private _isSkippedByMaxPatternDepth(entryDepth: number, maxPatternDepth: number): boolean {
-		return !this._settings.baseNameMatch && maxPatternDepth !== Infinity && entryDepth > maxPatternDepth;
-	}
+		const parts = filepath.split('/');
+		const level = parts.length - 1;
+		const part = parts[level];
 
-	private _isSkippedSymbolicLink(entry: Entry): boolean {
-		return !this._settings.followSymbolicLinks && entry.dirent.isSymbolicLink();
+		return !this._settings.baseNameMatch && !matcher.match(level, part);
 	}
 
 	private _isSkippedByNegativePatterns(entry: Entry, negativeRe: PatternRe[]): boolean {

src/providers/matchers/matcher.spec.ts

@@ -0,0 +1,37 @@
+import * as assert from 'assert';
+
+import * as tests from '../../tests';
+import { Pattern, MicromatchOptions } from '../../types';
+import Matcher, { PatternInfo } from './matcher';
+
+class TestMatcher extends Matcher {
+	public get storage(): PatternInfo[] {
+		return this._storage;
+	}
+}
+
+function getMatcher(patterns: Pattern[], options: MicromatchOptions = {}): TestMatcher {
+	return new TestMatcher(patterns, options);
+}
+
+describe('Providers → Matchers → Matcher', () => {
+	describe('.storage', () => {
+		it('should return created storage', () => {
+			const matcher = getMatcher(['a*', 'a/**/b']);
+
+			const expected: PatternInfo[] = [
+				tests.pattern.info()
+					.section(tests.pattern.segment().dynamic().pattern('a*').build())
+					.build(),
+				tests.pattern.info()
+					.section(tests.pattern.segment().pattern('a').build())
+					.section(tests.pattern.segment().pattern('b').build())
+					.build()
+			];
+
+			const actual = matcher.storage;
+
+			assert.deepStrictEqual(actual, expected);
+		});
+	});
+});

src/providers/matchers/matcher.ts

@@ -0,0 +1,80 @@
+import { Pattern, MicromatchOptions, PatternRe } from '../../types';
+import * as utils from '../../utils';
+
+export type PatternSegment = StaticPatternSegment | DynamicPatternSegment;
+
+type StaticPatternSegment = {
+	dynamic: false;
+	pattern: Pattern;
+};
+
+type DynamicPatternSegment = {
+	dynamic: true;
+	pattern: Pattern;
+	patternRe: PatternRe;
+};
+
+export type PatternSection = PatternSegment[];
+
+export type PatternInfo = {
+	/**
+	 * Indicates that the pattern has a globstar (more than a single section).
+	 */
+	complete: boolean;
+	pattern: Pattern;
+	segments: PatternSegment[];
+	sections: PatternSection[];
+};
+
+export default abstract class Matcher {
+	protected readonly _storage: PatternInfo[] = [];
+
+	constructor(private readonly _patterns: Pattern[], private readonly _options: MicromatchOptions) {
+		this._fillStorage();
+	}
+
+	private _fillStorage(): void {
+		/**
+		 * The original pattern may include `{,*,**,a/*}`, which will lead to problems with matching (unresolved level).
+		 * So, before expand patterns with brace expansion into separated patterns.
+		 */
+		const patterns = utils.pattern.expandPatternsWithBraceExpansion(this._patterns);
+
+		for (const pattern of patterns) {
+			const segments = this._getPatternSegments(pattern);
+			const sections = this._splitSegmentsIntoSections(segments);
+
+			this._storage.push({
+				complete: sections.length <= 1,
+				pattern,
+				segments,
+				sections
+			});
+		}
+	}
+
+	private _getPatternSegments(pattern: Pattern): PatternSegment[] {
+		const parts = utils.pattern.getPatternParts(pattern, this._options);
+
+		return parts.map((part) => {
+			const dynamic = utils.pattern.isDynamicPattern(part);
+
+			if (!dynamic) {
+				return {
+					dynamic: false,
+					pattern: part
+				};
+			}
+
+			return {
+				dynamic: true,
+				pattern: part,
+				patternRe: utils.pattern.makeRe(part, this._options)
+			};
+		});
+	}
+
+	private _splitSegmentsIntoSections(segments: PatternSegment[]): PatternSection[] {
+		return utils.array.splitWhen(segments, (segment) => segment.dynamic && utils.pattern.hasGlobStar(segment.pattern));
+	}
+}

src/providers/matchers/partial.spec.ts

@@ -0,0 +1,54 @@
+import * as assert from 'assert';
+
+import { Pattern, MicromatchOptions } from '../../types';
+import Matcher from './partial';
+
+function getMatcher(patterns: Pattern[], options: MicromatchOptions = {}): Matcher {
+	return new Matcher(patterns, options);
+}
+
+function assertMatch(patterns: Pattern[], level: number, part: string): void | never {
+	const matcher = getMatcher(patterns);
+
+	assert.ok(matcher.match(level, part));
+}
+
+function assertNotMatch(patterns: Pattern[], level: number, part: string): void | never {
+	const matcher = getMatcher(patterns);
+
+	assert.ok(!matcher.match(level, part));
+}
+
+describe('Providers → Matchers → Partial', () => {
+	describe('.match', () => {
+		it('should handle patterns with globstar', () => {
+			assertMatch(['**'], 0, 'a');
+			assertMatch(['**'], 1, 'b');
+			assertMatch(['**/a'], 0, 'a');
+			assertMatch(['**/a'], 1, 'a');
+			assertNotMatch(['a/**'], 0, 'b');
+			assertMatch(['a/**'], 1, 'b');
+		});
+
+		it('should do not match the latest segment', () => {
+			assertMatch(['b', 'b/*'], 0, 'b');
+			assertNotMatch(['*'], 0, 'a');
+			assertNotMatch(['a/*'], 1, 'b');
+		});
+
+		it('should trying to match all patterns', () => {
+			assertMatch(['a/*', 'b/*'], 0, 'b');
+		});
+
+		it('should match a static segment', () => {
+			assertMatch(['a/b'], 0, 'a');
+			assertNotMatch(['b/b'], 0, 'a');
+		});
+
+		it('should match a dynamic segment', () => {
+			assertMatch(['*/b'], 0, 'a');
+			assertMatch(['{a,b}/*'], 0, 'a');
+			assertNotMatch(['{a,b}/*'], 0, 'c');
+		});
+	});
+});