add manifestignore support to generate docs manifest tool

docs/.manifestignore

@@ -0,0 +1,2 @@
+# Uses the same format as .gitignore
+_media

tools/monorepo-utils/src/md-docs/lib/__tests__/fixtures/example-docs/.manifestignore

@@ -0,0 +1,4 @@
+# Exclude developer-tools.md
+developer-tools
+# Exclude excluded-folder & test wildcard
+excluded-fold*

tools/monorepo-utils/src/md-docs/lib/__tests__/fixtures/example-docs/excluded-folder/README.md

@@ -0,0 +1,3 @@
+---
+category_title: Excluded WooCommerce
+---

tools/monorepo-utils/src/md-docs/lib/__tests__/fixtures/example-docs/excluded-folder/integration-tests.md

@@ -0,0 +1,7 @@
+---
+post_title: Integration Testing
+---
+
+## Integration Tests
+
+Elementary, my dear Watson! Write integration tests.

tools/monorepo-utils/src/md-docs/lib/__tests__/fixtures/example-docs/excluded-folder/unit-tests.md

@@ -0,0 +1,7 @@
+---
+post_title: Unit Testing
+---
+
+## Unit Test
+
+It's simple really, write tests!

tools/monorepo-utils/src/md-docs/lib/__tests__/fixtures/example-docs/get-started/developer-tools.md

@@ -0,0 +1,9 @@
+---
+post_title: Developer tools
+---
+
+## Developer tools
+
+1. [Install](./installation/install-plugin.md).
+2. Configure.
+3. Profit!

tools/monorepo-utils/src/md-docs/lib/__tests__/generate-manifest.ts

@@ -42,6 +42,22 @@ describe( 'generateManifest', () => {
 		);
 	} );
 
+	it( 'should exclude files and folders matching patterns in .manifestignore', async () => {
+		// generate the manifest from fixture directory
+		const manifest = await generateManifestFromDirectory(
+			rootDir,
+			dir,
+			'example-docs',
+			'https://example.com',
+			'https://example.com/edit'
+		);
+
+		const topLevelCategories = manifest.categories;
+
+		expect( topLevelCategories ).toHaveLength( 2 );
+		expect( topLevelCategories[ 0 ].posts ).toHaveLength( 1 );
+	} );
+
 	it( 'should generate a manifest with categories that contain all markdown files in a location as individual posts', async () => {
 		// generate the manifest from fixture directory
 		const manifest = await generateManifestFromDirectory(

tools/monorepo-utils/src/md-docs/lib/generate-manifest.ts

@@ -35,6 +35,7 @@ async function processDirectory(
 	baseUrl: string,
 	baseEditUrl: string,
 	fullPathToDocs: string,
+	exclude: string[],
 	checkReadme = true
 ): Promise< Category > {
 	const category: Category = {};
@@ -63,8 +64,11 @@ async function processDirectory(
 		category.category_title = category.category_title ?? categoryTitle;
 	}
 
-	const markdownFiles = glob.sync( path.join( subDirectory, '*.md' ) );
-
+	const markdownFiles = glob
+		.sync( path.join( subDirectory, '*.md' ) )
+		.filter(
+			( markdownFile ) => ! filenameMatches( markdownFile, exclude )
+		);
 	// If there are markdown files in this directory, add a posts array to the category. Otherwise, assume its a top level category that will contain subcategories.
 	if ( markdownFiles.length > 0 ) {
 		category.posts = [];
@@ -115,6 +119,9 @@ async function processDirectory(
 	const subdirectories = fs
 		.readdirSync( subDirectory, { withFileTypes: true } )
 		.filter( ( dirent ) => dirent.isDirectory() )
+		.filter(
+			( dirent ) => ! filenameMatches( dirent.name, exclude )
+		)
 		.map( ( dirent ) => path.join( subDirectory, dirent.name ) );
 	for ( const subdirectory of subdirectories ) {
 		const subcategory = await processDirectory(
@@ -123,7 +130,8 @@ async function processDirectory(
 			projectName,
 			baseUrl,
 			baseEditUrl,
-			fullPathToDocs
+			fullPathToDocs,
+			exclude
 		);
 
 		category.categories.push( subcategory );
@@ -140,6 +148,17 @@ export async function generateManifestFromDirectory(
 	baseEditUrl: string
 ) {
 	const fullPathToDocs = subDirectory;
+	const manifestIgnore = path.join( subDirectory, '.manifestignore' );
+	let ignoreList;
+
+	if ( fs.existsSync( manifestIgnore ) ) {
+		ignoreList = fs
+			.readFileSync( manifestIgnore, 'utf-8' )
+			.split( "\n" )
+			.map( ( item ) => item.trim() )
+			.filter( ( item ) => item.length > 0 )
+			.filter( ( item ) => item.substring( 0, 1 ) != '#' );
+	}
 
 	const manifest = await processDirectory(
 		rootDirectory,
@@ -148,6 +167,7 @@ export async function generateManifestFromDirectory(
 		baseUrl,
 		baseEditUrl,
 		fullPathToDocs,
+		ignoreList ?? [],
 		false
 	);
 
@@ -159,3 +179,11 @@ export async function generateManifestFromDirectory(
 
 	return { ...manifest, hash };
 }
+
+function filenameMatches(
+	filename: string,
+	hayStack: string[]
+) {
+	const found = hayStack.filter( ( item ) => filename.match( item ) );
+	return found.length > 0;
+}