From 0088cfd3ab77f2f0c2be1654d17c191bed7b376f Mon Sep 17 00:00:00 2001 From: docschina-bot Date: Mon, 29 Nov 2021 23:05:47 +0000 Subject: [PATCH 1/2] docs(en): fetch all --- .../image-minimizer-webpack-plugin.mdx | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/content/plugins/image-minimizer-webpack-plugin.mdx b/src/content/plugins/image-minimizer-webpack-plugin.mdx index 6271f6d1768a..298dd10ba0fb 100644 --- a/src/content/plugins/image-minimizer-webpack-plugin.mdx +++ b/src/content/plugins/image-minimizer-webpack-plugin.mdx @@ -283,7 +283,7 @@ module.exports = { | **[`minify`](#minify)** | `{Function \| Array}` | `ImageMinimizerPlugin.imageminMinify` | Allows to override default minify function | | **[`minimizerOptions`](#minimizeroptions)** | `{Object\|Array}` | `{ plugins: [] }` | Options for `imagemin` | | **[`loader`](#loader)** | `{Boolean}` | `true` | Automatically adding `imagemin-loader` | -| **[`maxConcurrency`](#maxconcurrency)** | `{Number}` | `Math.max(1, os.cpus().length - 1)` | Maximum number of concurrency optimization processes in one time | +| **[`concurrency`](#concurrency)** | `{Number}` | `Math.max(1, os.cpus().length - 1)` | Maximum number of concurrency optimization processes in one time | | **[`filename`](#filename)** | `{string\|Function}` | `'[path][name][ext]'` | Allows to set the filename for the generated asset. Useful for converting to a `webp` | | **[`deleteOriginalAssets`](#deleteoriginalassets)** | `{Boolean}` | `false` | Allows to delete the original asset. Useful for converting to a `webp` and remove original assets | @@ -610,7 +610,7 @@ module.exports = { }; ``` -#### `maxConcurrency` +#### `concurrency` Type: `Number` Default: `Math.max(1, os.cpus().length - 1)` @@ -625,7 +625,7 @@ const ImageMinimizerPlugin = require("image-minimizer-webpack-plugin"); module.exports = { plugins: [ new ImageMinimizerPlugin({ - maxConcurrency: 3, + concurrency: 3, }), ], }; @@ -1230,17 +1230,6 @@ The function normalizes configuration (converts plugins names and options to `Fu ```js const imagemin = require("imagemin"); const { imageminNormalizeConfig } = require("image-minimizer-webpack-plugin"); -const imageminConfig = imageminNormalizeConfig({ - plugins: [ - "jpegtran", - [ - "pngquant", - { - quality: [0.6, 0.8], - }, - ], - ], -}); /* console.log(imageminConfig); @@ -1255,6 +1244,9 @@ const imageminConfig = imageminNormalizeConfig({ */ (async () => { + const imageminConfig = await imageminNormalizeConfig({ + plugins: ["jpegtran", ["pngquant", { quality: [0.6, 0.8] }]], + }); const files = await imagemin(["images/*.{jpg,png}"], { destination: "build/images", plugins: imageminConfig.plugins, From d054d4656cbfb5228fc96f5de26d4f0ce349f742 Mon Sep 17 00:00:00 2001 From: QC-L Date: Sun, 19 Dec 2021 11:58:09 +0800 Subject: [PATCH 2/2] feat: add fetch package --- package.json | 151 +++++++++--------- src/utilities/{constants.js => constants.mjs} | 9 +- src/utilities/fetch-package-readmes.mjs | 97 +++++++++++ src/utilities/fetch-package-repos.mjs | 100 ++++++++++++ src/utilities/process-readme.mjs | 145 +++++++++++++++++ src/utilities/process-readme.test.mjs | 31 ++++ ...yaml-headmatter.js => yaml-headmatter.mjs} | 2 +- 7 files changed, 450 insertions(+), 85 deletions(-) rename src/utilities/{constants.js => constants.mjs} (91%) create mode 100644 src/utilities/fetch-package-readmes.mjs create mode 100644 src/utilities/fetch-package-repos.mjs create mode 100644 src/utilities/process-readme.mjs create mode 100644 src/utilities/process-readme.test.mjs rename src/utilities/{yaml-headmatter.js => yaml-headmatter.mjs} (74%) diff --git a/package.json b/package.json index d4b02ceabd7e..836336219907 100644 --- a/package.json +++ b/package.json @@ -25,25 +25,25 @@ }, "scripts": { "clean-dist": "rimraf ./dist", - "clean-printable": "rimraf src/content/**/printable.md", + "clean-printable": "rimraf src/content/**/printable.mdx", "preclean": "run-s clean-dist clean-printable", - "clean": "rimraf src/content/**/_*.md src/**/_*.json repositories/*.json", - "start": "npm run clean-dist && webpack serve --config webpack.dev.js --env dev --progress --node-env development", - "content": "node src/scripts/build-content-tree.js ./src/content ./src/_content.json", - "bundle-analyze": "run-s clean fetch content && webpack --config webpack.prod.js --node-env production && run-s printable content && webpack --config webpack.ssg.js --node-env production --env ssg --profile --json > stats.json && webpack-bundle-analyzer stats.json", - "fetch-repos": "node src/utilities/fetch-package-repos.js", + "clean": "rimraf src/content/**/_*.mdx src/**/_*.json repositories/*.json", + "start": "npm run clean-dist && webpack serve --config webpack.dev.mjs --env dev --progress --node-env development", + "content": "node src/scripts/build-content-tree.mjs ./src/content ./src/_content.json", + "bundle-analyze": "run-s clean fetch content && webpack --config webpack.prod.mjs --node-env production && run-s printable content && webpack --config webpack.ssg.mjs --node-env production --env ssg --profile --json > stats.json && webpack-bundle-analyzer stats.json", + "fetch-repos": "node src/utilities/fetch-package-repos.mjs", "fetch": "run-p fetch:*", - "fetch:readmes": "node src/utilities/fetch-package-readmes.js", + "fetch:readmes": "node src/utilities/fetch-package-readmes.mjs", "prebuild": "npm run clean", - "build": "run-s content && webpack --config webpack.prod.js --node-env production && run-s printable content && webpack --config webpack.ssg.js --node-env production", + "build": "run-s content && webpack --config webpack.prod.mjs --node-env production && run-s printable content && webpack --config webpack.ssg.mjs --node-env production --env ssg", "postbuild": "npm run sitemap", "build-test": "npm run build && http-server --port 4200 dist/", "test": "npm run lint", "lint": "run-s lint:*", "lint:js": "npm run lint-js .", "lint-js": "eslint --cache --cache-location .cache/.eslintcache", - "lint:markdown": "npm run lint-markdown *.md ./src/content/**/*.md", - "lint-markdown": "markdownlint --config ./.markdownlint.json --ignore './src/content/**/_*.md' --ignore '.vale/**/*.md' --ignore '.github/**/*.md'", + "lint:markdown": "npm run lint-markdown *.md", + "lint-markdown": "markdownlint --config ./.markdownlint.json --ignore '.vale/**/*.md' --ignore '.github/**/*.md'", "lint:prose": "vale --config='.vale.ini' src/content", "lint:links": "hyperlink -c 8 --root dist -r dist/index.html --canonicalroot https://webpack.js.org/ --internal --skip /plugins/extract-text-webpack-plugin/ --skip /printable --skip https:// --skip http:// --skip sw.js > internal-links.tap; cat internal-links.tap | tap-spot", "lint:heading": "textlint --fix src/content/*", @@ -52,8 +52,8 @@ "sitemap": "cd dist && sitemap-static --ignore-file=../sitemap-ignore.json --pretty --prefix=https://webpack.js.org/ > sitemap.xml", "serve": "npm run build && sirv start ./dist --port 4000", "preprintable": "npm run clean-printable", - "printable": "node ./src/scripts/concatenate-docs.js", - "jest": "jest", + "printable": "node ./src/scripts/concatenate-docs.mjs", + "jest": "NODE_OPTIONS=--experimental-vm-modules jest --config=jest.config.mjs", "cypress:open": "cypress open", "cypress:run": "cypress run", "cypress:ci": "start-server-and-test build-test http://localhost:4200 cypress:run", @@ -61,7 +61,7 @@ "prepare": "husky install" }, "lint-staged": { - "*.{js,jsx,md}": [ + "*.{js,jsx,md,mdx}": [ "npm run lint-js" ], "*.md": [ @@ -72,102 +72,99 @@ ] }, "devDependencies": { - "@babel/core": "^7.14.6", - "@babel/eslint-parser": "^7.14.5", + "@babel/core": "^7.16.0", + "@babel/eslint-parser": "^7.16.3", "@babel/plugin-proposal-class-properties": "^7.12.1", - "@babel/preset-env": "^7.14.5", - "@babel/preset-react": "^7.14.5", - "@mdx-js/loader": "^1.6.21", + "@babel/preset-env": "^7.16.4", + "@babel/preset-react": "^7.16.0", + "@mdx-js/loader": "^2.0.0-next.9", "@octokit/auth-action": "^1.3.3", - "@octokit/rest": "^18.6.0", + "@octokit/rest": "^18.12.0", "@pmmmwh/react-refresh-webpack-plugin": "next", - "@svgr/webpack": "^5.5.0", - "autoprefixer": "^10.2.6", - "babel-loader": "^8.2.2", - "copy-webpack-plugin": "^9.0.0", - "css-loader": "^5.2.6", - "css-minimizer-webpack-plugin": "^3.0.1", - "cypress": "^7.5.0", - "directory-tree": "^2.2.9", - "directory-tree-webpack-plugin": "^1.0.2", - "docschina-remark-slugger": "^0.1.4", + "@svgr/webpack": "^6.1.1", + "autoprefixer": "^10.4.0", + "babel-loader": "^8.2.3", + "copy-webpack-plugin": "^10.0.0", + "css-loader": "^6.5.1", + "css-minimizer-webpack-plugin": "^3.2.0", + "cypress": "^9.1.1", + "directory-tree": "^3.0.1", + "directory-tree-webpack-plugin": "^1.0.3", "duplexer": "^0.1.1", - "eslint": "^7.28.0", + "eslint": "^8.4.1", "eslint-config-prettier": "^8.3.0", - "eslint-plugin-cypress": "^2.11.3", - "eslint-plugin-markdown": "^2.2.0", - "eslint-plugin-mdx": "^1.13.0", - "eslint-plugin-react": "^7.24.0", - "eslint-plugin-react-hooks": "^4.2.0", + "eslint-plugin-cypress": "^2.12.1", + "eslint-plugin-mdx": "^1.16.0", + "eslint-plugin-react": "^7.27.1", + "eslint-plugin-react-hooks": "^4.3.0", "front-matter": "^4.0.2", + "github-slugger": "^1.3.0", "html-loader": "^2.1.2", - "html-webpack-plugin": "^5.3.0", - "http-server": "^0.12.3", - "husky": "^6.0.0", - "hyperlink": "^4.6.1", - "jest": "^27.0.4", - "lint-staged": "^11.0.0", + "html-webpack-plugin": "^5.5.0", + "http-server": "^14.0.0", + "husky": "^7.0.4", + "hyperlink": "^5.0.3", + "jest": "^27.4.3", + "lint-staged": "^12.1.2", "lodash": "^4.17.21", - "markdownlint": "^0.23.1", - "markdownlint-cli": "^0.27.1", - "mini-css-extract-plugin": "^1.6.0", + "markdownlint": "^0.24.0", + "markdownlint-cli": "^0.30.0", + "mdast-util-to-string": "^1.1.0", + "mini-css-extract-plugin": "^2.4.5", "mkdirp": "^1.0.4", "modularscale-sass": "^3.0.3", - "node-fetch": "^2.6.1", + "node-fetch": "^3.1.0", "npm-run-all": "^4.1.1", - "postcss": "^8.3.4", - "postcss-loader": "^6.1.0", - "prettier": "^2.3.1", - "react-refresh": "^0.10.0", + "postcss": "^8.4.4", + "postcss-loader": "^6.2.1", + "prettier": "^2.5.1", + "react-refresh": "^0.11.0", "redirect-webpack-plugin": "^1.0.0", - "remark": "^13.0.0", - "remark-autolink-headings": "^6.0.1", - "remark-emoji": "^2.2.0", + "remark": "^14.0.2", + "remark-autolink-headings": "7.0.1", + "remark-emoji": "^3.0.2", "remark-extract-anchors": "1.1.1", - "remark-frontmatter": "^3.0.0", + "remark-frontmatter": "^4.0.1", "remark-gfm": "^1.0.0", - "remark-html": "^13.0.1", - "remark-loader": "^4.0.0", - "remark-refractor": "git://github.com/montogeek/remark-refractor.git", - "remark-slug": "^6.0.0", + "remark-html": "^15.0.0", + "remark-refractor": "montogeek/remark-refractor", + "remark-slug": "^7.0.1", "rimraf": "^3.0.2", - "sass": "^1.35.0", - "sass-loader": "^12.1.0", - "sirv-cli": "^1.0.12", + "sass": "^1.44.0", + "sass-loader": "^12.4.0", + "sirv-cli": "^1.0.14", "sitemap-static": "^0.4.2", - "start-server-and-test": "^1.12.5", + "start-server-and-test": "^1.14.0", "static-site-generator-webpack-plugin": "^3.4.1", - "style-loader": "^2.0.0", - "tailwindcss": "^2.1.4", + "style-loader": "^3.3.1", + "tailwindcss": "^3.0.0", "tap-spot": "^1.1.1", "textlint": "^11.8.2", "textlint-rule-heading": "^1.0.10", - "unist-util-visit": "^2.0.3", - "webpack": "^5.39.0", - "webpack-bundle-analyzer": "^4.4.2", - "webpack-cli": "^4.7.2", - "webpack-dev-server": "^4.0.0-beta.3", + "unist-util-visit": "^4.1.0", + "webpack": "^5.65.0", + "webpack-bundle-analyzer": "^4.5.0", + "webpack-cli": "^4.9.1", + "webpack-dev-server": "^4.6.0", "webpack-merge": "^5.8.0", - "workbox-webpack-plugin": "^6.1.5" + "workbox-webpack-plugin": "^6.4.2" }, "dependencies": { - "docsearch.js": "^2.5.2", + "@docsearch/react": "^3.0.0-alpha.42", "path-browserify": "^1.0.1", "prop-types": "^15.7.2", "react": "^17.0.2", - "react-banner": "^1.0.0-rc.0", "react-dom": "^17.0.2", "react-g-analytics": "0.4.2", - "react-helmet-async": "^1.0.9", - "react-router-dom": "^5.2.0", - "react-spring": "next", + "react-helmet-async": "^1.2.2", + "react-router-dom": "^5.3.0", + "react-spring": "^9.3.2", "react-tiny-popover": "5", - "react-use": "^17.2.4", + "react-use": "^17.3.1", "react-visibility-sensor": "^5.0.2", - "unist-util-visit": "^2.0.3", "webpack-pwa-manifest": "^4.3.0", "webpack.vote": "https://github.com/webpack/voting-app.git", - "workbox-window": "^6.1.5" + "workbox-window": "^6.4.2" }, "resolutions": { "sitemap-static/minimist": "1.2.5", diff --git a/src/utilities/constants.js b/src/utilities/constants.mjs similarity index 91% rename from src/utilities/constants.js rename to src/utilities/constants.mjs index e7b03a17e762..91863b7c3756 100644 --- a/src/utilities/constants.js +++ b/src/utilities/constants.mjs @@ -1,4 +1,4 @@ -const excludedLoaders = [ +export const excludedLoaders = [ 'webpack-contrib/config-loader', 'webpack-contrib/eslint-loader', 'webpack-contrib/transform-loader', @@ -25,7 +25,7 @@ const excludedLoaders = [ 'webpack-contrib/istanbul-instrumenter-loader', 'webpack-contrib/worker-loader', ]; -const excludedPlugins = [ +export const excludedPlugins = [ 'webpack-contrib/component-webpack-plugin', 'webpack-contrib/extract-text-webpack-plugin', 'webpack-contrib/i18n-webpack-plugin', @@ -34,8 +34,3 @@ const excludedPlugins = [ 'webpack-contrib/zopfli-webpack-plugin', 'webpack-contrib/closure-webpack-plugin', ]; - -module.exports = { - excludedLoaders, - excludedPlugins, -}; diff --git a/src/utilities/fetch-package-readmes.mjs b/src/utilities/fetch-package-readmes.mjs new file mode 100644 index 000000000000..82335e9a00e1 --- /dev/null +++ b/src/utilities/fetch-package-readmes.mjs @@ -0,0 +1,97 @@ +import _ from 'lodash'; +import fs from 'fs'; +import path from 'path'; +import { promisify } from 'util'; +import mkdirp from 'mkdirp'; +import fetch from 'node-fetch'; +import { fileURLToPath } from 'url'; + +import yamlHeadmatter from './yaml-headmatter.mjs'; +import processReadme from './process-readme.mjs'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const writeFile = promisify(fs.writeFile); +const rename = promisify(fs.rename); +const readFile = promisify(fs.readFile); +const cwd = process.cwd(); + +const types = ['loaders', 'plugins']; + +const pathMap = { + loaders: path.resolve(__dirname, '../content/loaders'), + plugins: path.resolve(__dirname, '../content/plugins'), +}; + +const loaderGroup = { + 'css-loader': 'CSS', + 'less-loader': 'CSS', + 'postcss-loader': 'CSS', + 'sass-loader': 'CSS', + 'style-loader': 'CSS', + 'stylus-loader': 'CSS', +}; + +async function main() { + for (const type of types) { + const outputDir = pathMap[type]; + + await mkdirp(outputDir); + + const repos = JSON.parse( + await readFile(path.resolve(__dirname, `../../repositories/${type}.json`)) + ); + + for (const repo of repos) { + const [, packageName] = repo.split('/'); + const url = `https://raw.githubusercontent.com/${repo}/master/README.md`; + const htmlUrl = `https://github.com/${repo}`; + const editUrl = `${htmlUrl}/edit/master/README.md`; + const fileName = path.resolve(outputDir, `${packageName}.mdx`); + const mdxFileName = path.resolve(outputDir, `${packageName}.mdx`); + + let title = packageName; + + if (type === 'plugins') { + title = _.camelCase(title); + title = _.upperFirst(title); + title = title.replace(/I18N/, 'I18n'); + } + + // generate yaml matter for file + let headmatter; + + if (type === 'plugins') { + headmatter = yamlHeadmatter({ + title: title, + group: 'webpack contrib', + source: url, + edit: editUrl, + repo: htmlUrl, + }); + } else { + let basic = { + title: title, + source: url, + edit: editUrl, + repo: htmlUrl, + }; + + if (loaderGroup[packageName]) { + basic.group = loaderGroup[packageName]; + } + headmatter = yamlHeadmatter(basic); + } + + const response = await fetch(url); + const content = await response.text(); + const body = processReadme(content, { source: url }); + await writeFile(fileName, headmatter + body); + await rename(fileName, mdxFileName); + console.log('Generated:', path.relative(cwd, mdxFileName)); + } + } +} + +main(); diff --git a/src/utilities/fetch-package-repos.mjs b/src/utilities/fetch-package-repos.mjs new file mode 100644 index 000000000000..d3303a3571bb --- /dev/null +++ b/src/utilities/fetch-package-repos.mjs @@ -0,0 +1,100 @@ +import fs from 'fs'; +import path from 'path'; +import mkdirp from 'mkdirp'; +import { promisify } from 'util'; +import _ from 'lodash'; +import { Octokit as GithubAPI } from '@octokit/rest'; +import { createActionAuth } from '@octokit/auth-action'; +import { excludedLoaders, excludedPlugins } from './constants.mjs'; +import { fileURLToPath } from 'url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const writeFile = promisify(fs.writeFile); +const stat = promisify(fs.stat); + +const fetch = { + loaders: [ + { + organization: 'webpack-contrib', + suffixes: ['-loader'], + hides: excludedLoaders, + }, + 'babel/babel-loader', + ], + plugins: [ + { + organization: 'webpack-contrib', + suffixes: ['-webpack-plugin', '-extract-plugin'], + hides: excludedPlugins, + }, + ], +}; + +async function main() { + let api; + if (process.env.CI && process.env.CI === true) { + const auth = createActionAuth(); + const authentication = await auth(); + api = new GithubAPI({ + auth: authentication, + }); + } else { + api = new GithubAPI(); + } + + async function paginate(org) { + const data = await api.paginate('GET /orgs/:org/repos', { + org: org, + type: 'public', + }); + return data; + } + mkdirp.sync(path.resolve(__dirname, '../../repositories/')); + + for (const [type, collection] of Object.entries(fetch)) { + const jsonPath = path.resolve(__dirname, `../../repositories/${type}.json`); + try { + const result = await Promise.all( + collection.map(async (item) => { + if (typeof item === 'string') { + return item; + } + + const { organization, suffixes, hides } = item; + + const repos = await paginate(organization); + + return repos + .map((repo) => repo.full_name) + .filter((name) => suffixes.some((suffix) => name.endsWith(suffix))) + .filter((name) => !hides.includes(name)); + }) + ); + + const json = JSON.stringify(_.flatten(result), undefined, 2); + + await writeFile(jsonPath, json); + console.log(`Fetched file: ${jsonPath}`); + } catch (e) { + try { + const info = await stat(jsonPath); + + // error is acceptable if the data from cache is less than 48 hours old + if (info.mtimeMs < Date.now() - 48 * 60 * 60 * 1000) { + throw e; + } else { + console.warn(e.message); + } + } catch (e2) { + throw e; + } + } + } +} + +main().catch((e) => { + console.error(e.message); + process.exitCode = 1; +}); diff --git a/src/utilities/process-readme.mjs b/src/utilities/process-readme.mjs new file mode 100644 index 000000000000..13d04d92562b --- /dev/null +++ b/src/utilities/process-readme.mjs @@ -0,0 +1,145 @@ +import url from 'url'; +import { excludedLoaders, excludedPlugins } from './constants.mjs'; + +const beginsWithDocsDomainRegex = /^(?:https?:)\/\/webpack\.js\.org/; +const inlineLinkRegex = /\[[^\]]*\]\(([^)]+)\)/g; + +const fragmentLinkMap = { + '/api/module-variables/#__webpack_public_path__-webpack-specific-': + '/api/module-variables/#__webpack_public_path__-webpack-specific', + '/configuration/module/#rule-exclude': '/configuration/module/#ruleexclude', + '/configuration/module/#rule-include': '/configuration/module/#ruleinclude', + '/configuration/module/#rule-options-rule-query': + '/configuration/module/#ruleoptions--rulequery', + '/configuration/module/#rule-use': '/configuration/module/#ruleuse', + '/configuration/optimization/#optimization-concatenatemodules': + '/configuration/optimization/#optimizationconcatenatemodules', + '/configuration/output/#output-chunkfilename': + '/configuration/output/#outputchunkfilename', + '/configuration/output/#output-publicpath': + '/configuration/output/#outputpublicpath', + '/configuration/resolve/#resolve-modules': + '/configuration/resolve/#resolvemodules', + '/guides/shimming/#exports-loader': '/loaders/exports-loader', + '/guides/shimming/#imports-loader': '/loaders/imports-loader', + '/guides/shimming/#provideplugin': '/plugins/provide-plugin/', +}; + +function linkFixerFactory(sourceUrl) { + return function linkFixer(markdownLink, href) { + const oldHref = href; + + if (href.includes('//npmjs.com')) { + href = href.replace('//www.npmjs.com'); + } + + // Only resolve non-absolute urls from their source if they are not a document fragment link + if (!href.startsWith('#')) { + // Convert Github raw links to rendered links + let rendered_url = sourceUrl + .replace(/raw.githubusercontent.com/, 'github.com') + .replace(/master/, 'blob/master'); + + href = url.resolve(rendered_url, href); + } + + // Modify absolute documentation links to be root relative + if (beginsWithDocsDomainRegex.test(href)) { + href = href.replace(beginsWithDocsDomainRegex, ''); + } + + const fragmentLinkMapMatch = Object.keys(fragmentLinkMap).find((mapFrom) => + href.includes(mapFrom) + ); + if (fragmentLinkMapMatch) { + href = href.replace( + fragmentLinkMapMatch, + fragmentLinkMap[fragmentLinkMapMatch] + ); + console.error(`DEPRECATED EXTERNAL README LINK: + URL: ${sourceUrl} + ACTUAL: ${oldHref} + EXPECTED: ${oldHref.replace( + fragmentLinkMapMatch, + fragmentLinkMap[fragmentLinkMapMatch] + )}`); + } + + // Lowercase all fragment links, since markdown generators do the same + if (href.includes('#')) { + const [urlPath, urlFragment] = href.split('#'); + + href = `${urlPath}#${urlFragment.toLowerCase()}`; + } + + if (oldHref !== href) { + console.log('REWRITE URL:', oldHref, '-->', href); + } + + return markdownLink.replace(oldHref, href); + }; +} + +function getMatches(string, regex) { + const matches = []; + let match; + // eslint-disable-next-line + while ((match = regex.exec(string))) { + matches.push(match); + } + return matches; +} + +export default function processREADME(body, options = {}) { + let processingString = body + // close tags + .replace( + /<(img\s[^>]*?src\s*=\s*['"][^'"]*?['"][^>/]*?)>(?![^<]*<\/img)/g, + '<$1/>' + ) + // Replace lone h1 formats + .replace(/.+?<\/h1>/, '') + .replace(/^# .+/m, '') + .replace(/.*\n=+/, '') + // Replace local github links with absolute links to the github location + // EXAMPLE: [Contributing](./.github/CONTRIBUTING.md) + // EXAMPLE: [Contributing](CONTRIBUTING.md) + // EXAMPLE: [line-identifier]: https://webpack.js.org/loaders/ + .replace(inlineLinkRegex, linkFixerFactory(options.source)) + // Replace any

with `##` + .replace(/]*>/g, '## ') + .replace(/<\/h2>/g, '') + // Drop any comments + .replace(//g, ''); + + // find the laoders links + const loaderMatches = getMatches( + processingString, + /https?:\/\/github.com\/(webpack|webpack-contrib)\/([-A-za-z0-9]+-loader\/?)([)"])/g + ); + // dont make relative links for excluded loaders + loaderMatches.forEach((match) => { + if (!excludedLoaders.includes(`${match[1]}/${match[2]}`)) { + processingString = processingString.replace( + match[0], + `/loaders/${match[2]}/)` + ); + } + }); + + const pluginMatches = getMatches( + processingString, + /https?:\/\/github.com\/(webpack|webpack-contrib)\/([-A-za-z0-9]+-plugin\/?)([)"])/g + ); + // dont make relative links for excluded loaders + pluginMatches.forEach((match) => { + if (!excludedPlugins.includes(`${match[1]}/${match[2]}`)) { + processingString = processingString.replace( + match[0], + `/plugins/${match[2]}/)` + ); + } + }); + + return processingString; +} diff --git a/src/utilities/process-readme.test.mjs b/src/utilities/process-readme.test.mjs new file mode 100644 index 000000000000..329aaac05880 --- /dev/null +++ b/src/utilities/process-readme.test.mjs @@ -0,0 +1,31 @@ +import processReadme from './process-readme.mjs'; +describe('processReadme', () => { + const url = + 'https://raw.githubusercontent.com/webpack/html-loader/master/README.md'; + it('links with the site', () => { + const options = { source: url }; + const loaderMDData = + '- [file-loader](https://github.com/webpack/file-loader)'; + const pluginMDData = + '- [eslint-webpack-plugin](https://github.com/webpack-contrib/eslint-webpack-plugin)'; + expect(processReadme(loaderMDData, options)).toEqual( + '- [file-loader](/loaders/file-loader/)' + ); + expect(processReadme(pluginMDData, options)).toEqual( + '- [eslint-webpack-plugin](/plugins/eslint-webpack-plugin/)' + ); + }); + it('links without the site', () => { + const options = { source: url }; + const loaderMDData = + '- [extract-loader](https://github.com/peerigon/extract-loader)'; + const pluginMDData = + '- [html-webpack-plugin](https://github.com/jantimon/html-webpack-plugin)'; + expect(processReadme(loaderMDData, options)).toEqual( + '- [extract-loader](https://github.com/peerigon/extract-loader)' + ); + expect(processReadme(pluginMDData, options)).toEqual( + '- [html-webpack-plugin](https://github.com/jantimon/html-webpack-plugin)' + ); + }); +}); diff --git a/src/utilities/yaml-headmatter.js b/src/utilities/yaml-headmatter.mjs similarity index 74% rename from src/utilities/yaml-headmatter.js rename to src/utilities/yaml-headmatter.mjs index a6f076867b98..dc5a400e2e21 100644 --- a/src/utilities/yaml-headmatter.js +++ b/src/utilities/yaml-headmatter.mjs @@ -1,4 +1,4 @@ -module.exports = function yamlHeadmatter(fields) { +export default function yamlHeadmatter(fields) { let ret = '---\n'; Object.keys(fields).map((field) => {